Skip to content

Commit 1ad55ea

Browse files
IvanNardiIvanNardi
committed
Flow: keep track of "dissectors"
In the flow, we should keep track of state of "dissectors", not "protocols". This way, flow structure doesn't depend anymore on the max number of protocols. This is also the first step into fixing #2136
1 parent 31a8d43 commit 1ad55ea

6 files changed

Lines changed: 73 additions & 68 deletions

File tree

‎src/include/ndpi_api.h‎

Lines changed: 0 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -272,29 +272,6 @@ extern "C" {
272272
*/
273273
void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct);
274274

275-
/**
276-
* Sets a single protocol bitmask
277-
* This function does not increment the index of the callback_buffer
278-
*
279-
* @par label = string for the protocol name
280-
* @par ndpi_struct = the detection module
281-
* @par idx = the index of the callback_buffer
282-
* @par func = function pointer of the protocol search
283-
* @par ndpi_selection_bitmask = the protocol selected bitmask
284-
* @par b_save_bitmask_unknow = if set as "true" save the detection bitmask as unknow
285-
* @par b_add_detection_bitmask = if set as "true" add the protocol bitmask to the detection bitmask
286-
*
287-
*/
288-
void ndpi_set_bitmask_protocol_detection(char *label,
289-
struct ndpi_detection_module_struct *ndpi_struct,
290-
const u_int32_t idx,
291-
u_int16_t ndpi_protocol_id,
292-
void (*func) (struct ndpi_detection_module_struct *,
293-
struct ndpi_flow_struct *flow),
294-
const NDPI_SELECTION_BITMASK_PROTOCOL_SIZE ndpi_selection_bitmask,
295-
u_int8_t b_save_bitmask_unknow,
296-
u_int8_t b_add_detection_bitmask);
297-
298275
/**
299276
* Sets the protocol bitmask2
300277
*

‎src/include/ndpi_define.h.in‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,10 @@
139139
#define NDPI_ISSET_BIT(num, n) (num & (1ULL << ( n )))
140140
#define NDPI_ZERO_BIT(num) num = 0
141141

142+
#define NDPI_DISSECTOR_BITMASK ndpi_dissector_bitmask_struct_t
143+
#define NDPI_DISSECTOR_BITMASK_IS_SET(p, n) NDPI_ISSET(&(p), (n))
144+
#define NDPI_DISSECTOR_BITMASK_SET(p, n) NDPI_SET(&(p), (n) & NDPI_NUM_BITS_MASK)
145+
142146
/* this is a very very tricky macro *g*,
143147
* the compiler will remove all shifts here if the protocol is static...
144148
*/

‎src/include/ndpi_private.h‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,6 @@ struct call_function_struct {
5858
void (*func) (struct ndpi_detection_module_struct *, struct ndpi_flow_struct *flow);
5959
NDPI_SELECTION_BITMASK_PROTOCOL_SIZE ndpi_selection_bitmask;
6060
u_int16_t ndpi_protocol_id;
61-
u_int8_t detection_feature;
6261
};
6362

6463
struct subprotocol_conf_struct {
@@ -608,6 +607,16 @@ struct ndpi_detection_module_struct {
608607

609608
/* Generic */
610609

610+
void ndpi_set_bitmask_protocol_detection(char *label,
611+
struct ndpi_detection_module_struct *ndpi_struct,
612+
const u_int32_t idx,
613+
u_int16_t ndpi_protocol_id,
614+
void (*func) (struct ndpi_detection_module_struct *,
615+
struct ndpi_flow_struct *flow),
616+
const NDPI_SELECTION_BITMASK_PROTOCOL_SIZE ndpi_selection_bitmask,
617+
u_int8_t b_save_bitmask_unknow,
618+
u_int8_t b_add_detection_bitmask);
619+
611620
char *strptime(const char *s, const char *format, struct tm *tm);
612621

613622
u_int8_t iph_is_valid_and_not_fragmented(const struct ndpi_iphdr *iph, const u_int16_t ipsize);

‎src/include/ndpi_typedefs.h‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,12 @@ typedef struct ndpi_protocol_bitmask_struct {
272272
ndpi_ndpi_mask fds_bits[NDPI_NUM_FDS_BITS];
273273
} ndpi_protocol_bitmask_struct_t;
274274

275+
276+
#define NDPI_MAX_NUM_DISSECTORS 288 /* Multiple of 32, i.e. 8 * sizeof(ndpi_ndpi_mask) */
277+
typedef struct ndpi_dissector_bitmask_struct {
278+
ndpi_ndpi_mask fds_bits[NDPI_MAX_NUM_DISSECTORS / (8 * sizeof(ndpi_ndpi_mask))];
279+
} ndpi_dissector_bitmask_struct_t;
280+
275281
struct ndpi_detection_module_struct;
276282

277283
/* NDPI_DEBUG_FUNCTION_PTR (cast) */
@@ -1191,7 +1197,7 @@ typedef struct ndpi_proto_defaults {
11911197
u_int8_t isClearTextProto:1, isAppProtocol:1, _notused:6;
11921198
u_int16_t *subprotocols;
11931199
u_int32_t subprotocol_count;
1194-
u_int16_t protoId, protoIdx;
1200+
u_int16_t protoId, dissector_idx;
11951201
u_int16_t tcp_default_ports[MAX_DEFAULT_PORTS], udp_default_ports[MAX_DEFAULT_PORTS];
11961202
ndpi_protocol_breed_t protoBreed;
11971203
ndpi_protocol_qoe_category_t qoeCategory;
@@ -1630,8 +1636,7 @@ struct ndpi_flow_struct {
16301636
/* **Packet** metadata for flows where monitoring is enabled. It is reset after each packet! */
16311637
struct ndpi_metadata_monitoring *monit;
16321638

1633-
/* protocols which have marked a connection as this connection cannot be protocol XXX, multiple u_int64_t */
1634-
NDPI_PROTOCOL_BITMASK excluded_protocol_bitmask;
1639+
NDPI_DISSECTOR_BITMASK excluded_dissectors_bitmask;
16351640

16361641
/* NDPI_PROTOCOL_BITTORRENT */
16371642
u_int8_t bittorrent_stage; // can be 0 - 255

‎src/lib/ndpi_main.c‎

Lines changed: 47 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -526,7 +526,7 @@ void ndpi_exclude_protocol(struct ndpi_detection_module_struct *ndpi_str, struct
526526
(void)_func;
527527
(void)_line;
528528
#endif
529-
NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, protocol_id);
529+
NDPI_DISSECTOR_BITMASK_SET(flow->excluded_dissectors_bitmask, ndpi_str->proto_defaults[protocol_id].dissector_idx);
530530
}
531531
}
532532

@@ -5733,7 +5733,6 @@ int load_protocols_file_fd(struct ndpi_detection_module_struct *ndpi_str, FILE *
57335733

57345734
/* ******************************************************************** */
57355735

5736-
/* ntop */
57375736
void ndpi_set_bitmask_protocol_detection(char *label, struct ndpi_detection_module_struct *ndpi_str,
57385737
const u_int32_t idx,
57395738
u_int16_t ndpi_protocol_id,
@@ -5742,9 +5741,20 @@ void ndpi_set_bitmask_protocol_detection(char *label, struct ndpi_detection_modu
57425741
const NDPI_SELECTION_BITMASK_PROTOCOL_SIZE ndpi_selection_bitmask,
57435742
u_int8_t b_save_bitmask_unknow, u_int8_t b_add_detection_bitmask) {
57445743
(void)label;
5745-
/*
5746-
Compare specify protocol bitmask with main detection bitmask
5747-
*/
5744+
5745+
5746+
if(idx >= NDPI_MAX_NUM_DISSECTORS) {
5747+
/*
5748+
* You need to increase NDPI_MAX_NUM_DISSECTORS define and recompile everything!
5749+
* Please note that custom protocols are independent from NDPI_MAX_NUM_DISSECTORS, so
5750+
* if you hit this error is because you are already changing the code
5751+
* (adding a new dissector)...
5752+
*/
5753+
NDPI_LOG_ERR(ndpi_str, "[NDPI] Internal Error. Too many dissectors!!\n");
5754+
/* Not sure what to do here...*/
5755+
return;
5756+
}
5757+
57485758
if(is_proto_enabled(ndpi_str, ndpi_protocol_id)) {
57495759
#ifdef NDPI_ENABLE_DEBUG_MESSAGES
57505760
NDPI_LOG_DBG2(ndpi_str,
@@ -5753,7 +5763,7 @@ void ndpi_set_bitmask_protocol_detection(char *label, struct ndpi_detection_modu
57535763
label, idx, ndpi_protocol_id);
57545764
#endif
57555765

5756-
if(ndpi_str->proto_defaults[ndpi_protocol_id].protoIdx != 0) {
5766+
if(ndpi_str->proto_defaults[ndpi_protocol_id].dissector_idx != 0) {
57575767
NDPI_LOG_DBG2(ndpi_str, "[NDPI] Internal error: protocol %s/%u has been already registered\n", label,
57585768
ndpi_protocol_id);
57595769
#ifdef NDPI_ENABLE_DEBUG_MESSAGES
@@ -5766,7 +5776,7 @@ void ndpi_set_bitmask_protocol_detection(char *label, struct ndpi_detection_modu
57665776
Set function and index protocol within proto_default structure for port protocol detection
57675777
and callback_buffer function for DPI protocol detection
57685778
*/
5769-
ndpi_str->proto_defaults[ndpi_protocol_id].protoIdx = idx;
5779+
ndpi_str->proto_defaults[ndpi_protocol_id].dissector_idx = idx;
57705780
ndpi_str->proto_defaults[ndpi_protocol_id].func = ndpi_str->callback_buffer[idx].func = func;
57715781
ndpi_str->callback_buffer[idx].ndpi_protocol_id = ndpi_protocol_id;
57725782

@@ -6588,8 +6598,12 @@ static int ndpi_callback_init(struct ndpi_detection_module_struct *ndpi_str) {
65886598

65896599
ndpi_enabled_callbacks_init(ndpi_str,detection_bitmask,0);
65906600

6591-
/* When the module ends, it is necessary to free the memory ndpi_str->callback_buffer and
6592-
ndpi_str->callback_buffer_tcp_payload */
6601+
NDPI_LOG_DBG(ndpi_str, "Tot num dissectors: %d (TCP: %d, TCP_NO_PAYLOAD: %d, UDP: %d, NO_TCP_UDP: %d\n",
6602+
ndpi_str->callback_buffer_size,
6603+
ndpi_str->callback_buffer_size_tcp_payload,
6604+
ndpi_str->callback_buffer_size_tcp_no_payload,
6605+
ndpi_str->callback_buffer_size_udp,
6606+
ndpi_str->callback_buffer_size_non_tcp_udp);
65936607

65946608
return 0;
65956609
}
@@ -7666,12 +7680,11 @@ static u_int32_t check_ndpi_subprotocols(struct ndpi_detection_module_struct * c
76667680
continue;
76677681
}
76687682

7669-
u_int16_t subproto_index = ndpi_str->proto_defaults[subproto_id].protoIdx;
7683+
u_int16_t subproto_index = ndpi_str->proto_defaults[subproto_id].dissector_idx;
76707684

76717685
if((ndpi_str->callback_buffer[subproto_index].ndpi_selection_bitmask & ndpi_selection_packet) ==
76727686
ndpi_str->callback_buffer[subproto_index].ndpi_selection_bitmask &&
7673-
NDPI_BITMASK_COMPARE(flow->excluded_protocol_bitmask,
7674-
ndpi_str->callback_buffer[subproto_index].excluded_protocol_bitmask) == 0 &&
7687+
!NDPI_DISSECTOR_BITMASK_IS_SET(flow->excluded_dissectors_bitmask, subproto_index) &&
76757688
NDPI_BITMASK_COMPARE(ndpi_str->callback_buffer[subproto_index].detection_bitmask,
76767689
detection_bitmask) != 0) {
76777690
ndpi_str->callback_buffer[subproto_index].func(ndpi_str, flow);
@@ -7688,49 +7701,42 @@ static u_int32_t check_ndpi_detection_func(struct ndpi_detection_module_struct *
76887701
struct ndpi_flow_struct * const flow,
76897702
NDPI_SELECTION_BITMASK_PROTOCOL_SIZE const ndpi_selection_packet,
76907703
struct call_function_struct const * const callback_buffer,
7691-
uint32_t callback_buffer_size,
7692-
int is_tcp_without_payload) {
7704+
uint32_t callback_buffer_size) {
76937705
void *func = NULL;
76947706
u_int32_t num_calls = 0;
76957707
/* First callback is associated to classification by-port,
76967708
if we don't already have a partial classification */
76977709
u_int16_t fast_callback_protocol_id = flow->fast_callback_protocol_id ? flow->fast_callback_protocol_id : flow->guessed_protocol_id;
7698-
u_int16_t proto_index = ndpi_str->proto_defaults[fast_callback_protocol_id].protoIdx;
7699-
u_int16_t proto_id = ndpi_str->proto_defaults[fast_callback_protocol_id].protoId;
7710+
u_int16_t dissector_idx = ndpi_str->proto_defaults[fast_callback_protocol_id].dissector_idx;
7711+
u_int16_t proto_id;
77007712
NDPI_PROTOCOL_BITMASK detection_bitmask;
77017713
u_int32_t a;
77027714

77037715
NDPI_SAVE_AS_BITMASK(detection_bitmask, flow->detected_protocol_stack[0]);
77047716

7705-
if((proto_id != NDPI_PROTOCOL_UNKNOWN) &&
7706-
NDPI_BITMASK_COMPARE(flow->excluded_protocol_bitmask,
7707-
ndpi_str->callback_buffer[proto_index].excluded_protocol_bitmask) == 0 &&
7708-
NDPI_BITMASK_COMPARE(ndpi_str->callback_buffer[proto_index].detection_bitmask, detection_bitmask) != 0 &&
7709-
(ndpi_str->callback_buffer[proto_index].ndpi_selection_bitmask & ndpi_selection_packet) ==
7710-
ndpi_str->callback_buffer[proto_index].ndpi_selection_bitmask)
7711-
{
7712-
if((fast_callback_protocol_id != NDPI_PROTOCOL_UNKNOWN) &&
7713-
(ndpi_str->proto_defaults[fast_callback_protocol_id].func != NULL) &&
7714-
(is_tcp_without_payload == 0 ||
7715-
((ndpi_str->callback_buffer[proto_index].ndpi_selection_bitmask &
7716-
NDPI_SELECTION_BITMASK_PROTOCOL_HAS_PAYLOAD) == 0)))
7717-
{
7718-
ndpi_str->proto_defaults[fast_callback_protocol_id].func(ndpi_str, flow);
7719-
func = ndpi_str->proto_defaults[fast_callback_protocol_id].func;
7720-
num_calls++;
7721-
}
7722-
}
7717+
if(fast_callback_protocol_id != NDPI_PROTOCOL_UNKNOWN &&
7718+
ndpi_str->proto_defaults[fast_callback_protocol_id].func &&
7719+
!NDPI_DISSECTOR_BITMASK_IS_SET(flow->excluded_dissectors_bitmask, dissector_idx) &&
7720+
(ndpi_str->callback_buffer[dissector_idx].ndpi_selection_bitmask & ndpi_selection_packet) ==
7721+
ndpi_str->callback_buffer[dissector_idx].ndpi_selection_bitmask &&
7722+
NDPI_BITMASK_COMPARE(ndpi_str->callback_buffer[dissector_idx].detection_bitmask, detection_bitmask) != 0) {
7723+
ndpi_str->proto_defaults[fast_callback_protocol_id].func(ndpi_str, flow);
7724+
func = ndpi_str->proto_defaults[fast_callback_protocol_id].func;
7725+
num_calls++;
7726+
}
77237727

77247728
if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN)
77257729
{
77267730
/* TODO: optimize as today we're doing a linear scan */
77277731

77287732
for (a = 0; a < callback_buffer_size; a++) {
7733+
proto_id = callback_buffer[a].ndpi_protocol_id;
7734+
dissector_idx = ndpi_str->proto_defaults[proto_id].dissector_idx;
7735+
77297736
if((func != callback_buffer[a].func) &&
77307737
(callback_buffer[a].ndpi_selection_bitmask & ndpi_selection_packet) ==
77317738
callback_buffer[a].ndpi_selection_bitmask &&
7732-
NDPI_BITMASK_COMPARE(flow->excluded_protocol_bitmask,
7733-
callback_buffer[a].excluded_protocol_bitmask) == 0 &&
7739+
!NDPI_DISSECTOR_BITMASK_IS_SET(flow->excluded_dissectors_bitmask, dissector_idx) &&
77347740
NDPI_BITMASK_COMPARE(callback_buffer[a].detection_bitmask,
77357741
detection_bitmask) != 0)
77367742
{
@@ -7761,7 +7767,7 @@ u_int32_t check_ndpi_other_flow_func(struct ndpi_detection_module_struct *ndpi_s
77617767
{
77627768
return check_ndpi_detection_func(ndpi_str, flow, *ndpi_selection_packet,
77637769
ndpi_str->callback_buffer_non_tcp_udp,
7764-
ndpi_str->callback_buffer_size_non_tcp_udp, 0);
7770+
ndpi_str->callback_buffer_size_non_tcp_udp);
77657771
}
77667772

77677773
/* ************************************************ */
@@ -7771,7 +7777,7 @@ static u_int32_t check_ndpi_udp_flow_func(struct ndpi_detection_module_struct *n
77717777
NDPI_SELECTION_BITMASK_PROTOCOL_SIZE *ndpi_selection_packet) {
77727778
return check_ndpi_detection_func(ndpi_str, flow, *ndpi_selection_packet,
77737779
ndpi_str->callback_buffer_udp,
7774-
ndpi_str->callback_buffer_size_udp, 0);
7780+
ndpi_str->callback_buffer_size_udp);
77757781
}
77767782

77777783
/* ************************************************ */
@@ -7783,12 +7789,12 @@ static u_int32_t check_ndpi_tcp_flow_func(struct ndpi_detection_module_struct *n
77837789
if(ndpi_str->packet.payload_packet_len != 0) {
77847790
return check_ndpi_detection_func(ndpi_str, flow, *ndpi_selection_packet,
77857791
ndpi_str->callback_buffer_tcp_payload,
7786-
ndpi_str->callback_buffer_size_tcp_payload, 0);
7792+
ndpi_str->callback_buffer_size_tcp_payload);
77877793
} else {
77887794
/* no payload */
77897795
return check_ndpi_detection_func(ndpi_str, flow, *ndpi_selection_packet,
77907796
ndpi_str->callback_buffer_tcp_no_payload,
7791-
ndpi_str->callback_buffer_size_tcp_no_payload, 1);
7797+
ndpi_str->callback_buffer_size_tcp_no_payload);
77927798
}
77937799
}
77947800

@@ -10930,7 +10936,7 @@ const char *ndpi_get_l4_proto_name(ndpi_l4_proto_info proto) {
1093010936
ndpi_l4_proto_info ndpi_get_l4_proto_info(struct ndpi_detection_module_struct *ndpi_struct,
1093110937
u_int16_t ndpi_proto_id) {
1093210938
if(ndpi_struct && ndpi_proto_id < ndpi_struct->ndpi_num_supported_protocols) {
10933-
u_int16_t idx = ndpi_struct->proto_defaults[ndpi_proto_id].protoIdx;
10939+
u_int16_t idx = ndpi_struct->proto_defaults[ndpi_proto_id].dissector_idx;
1093410940
NDPI_SELECTION_BITMASK_PROTOCOL_SIZE bm = ndpi_struct->callback_buffer[idx].ndpi_selection_bitmask;
1093510941

1093610942
if(bm & NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP)

‎windows/src/ndpi_define.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,10 @@
138138
#define NDPI_ISSET_BIT(num, n) (num & (1ULL << ( n )))
139139
#define NDPI_ZERO_BIT(num) num = 0
140140

141+
#define NDPI_DISSECTOR_BITMASK ndpi_dissector_bitmask_struct_t
142+
#define NDPI_DISSECTOR_BITMASK_IS_SET(p, n) NDPI_ISSET(&(p), (n))
143+
#define NDPI_DISSECTOR_BITMASK_SET(p, n) NDPI_SET(&(p), (n) & NDPI_NUM_BITS_MASK)
144+
141145
/* this is a very very tricky macro *g*,
142146
* the compiler will remove all shifts here if the protocol is static...
143147
*/

0 commit comments

Comments
 (0)