Skip to content

Commit 5d2ac9f

Browse files
jasonishjasonish
committed
smtp: add firewall progress states
Add minimal SMTP progress states to support envelope validation before moving to data. Update SMTP, file and email keywords to hook into the appropriate states. Purposefully kept minimal for now as to not break the current idea of an SMTP transaction, which is probably not ideal for firewall mode. Ticket: OISF#8393
1 parent e7e837c commit 5d2ac9f

4 files changed

Lines changed: 144 additions & 29 deletions

File tree

‎src/app-layer-smtp.c‎

Lines changed: 96 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,50 @@ static const char *SMTPGetFrameNameById(const uint8_t frame_id)
186186
return name;
187187
}
188188

189+
static SCEnumCharMap smtp_state_client_table[] = {
190+
{ "request_started", SMTP_REQUEST_STARTED },
191+
{ "request_data", SMTP_REQUEST_DATA },
192+
{ "request_complete", SMTP_REQUEST_COMPLETE },
193+
{ NULL, -1 },
194+
};
195+
196+
static SCEnumCharMap smtp_state_server_table[] = {
197+
{ "response_started", SMTP_RESPONSE_STARTED },
198+
{ "response_data", SMTP_RESPONSE_DATA },
199+
{ "response_complete", SMTP_RESPONSE_COMPLETE },
200+
{ NULL, -1 },
201+
};
202+
203+
static int SMTPStateGetStateIdByName(const char *name, const uint8_t direction)
204+
{
205+
SCEnumCharMap *map =
206+
direction == STREAM_TOSERVER ? smtp_state_client_table : smtp_state_server_table;
207+
int id = SCMapEnumNameToValue(name, map);
208+
if (id < 0) {
209+
return -1;
210+
}
211+
return id;
212+
}
213+
214+
static const char *SMTPStateGetStateNameById(const int id, const uint8_t direction)
215+
{
216+
SCEnumCharMap *map =
217+
direction == STREAM_TOSERVER ? smtp_state_client_table : smtp_state_server_table;
218+
return SCMapEnumValueToName(id, map);
219+
}
220+
221+
static inline void SMTPSetProgressTS(SMTPTransaction *tx, uint8_t progress)
222+
{
223+
if (tx != NULL && tx->progress_ts < progress)
224+
tx->progress_ts = progress;
225+
}
226+
227+
static inline void SMTPSetProgressTC(SMTPTransaction *tx, uint8_t progress)
228+
{
229+
if (tx != NULL && tx->progress_tc < progress)
230+
tx->progress_tc = progress;
231+
}
232+
189233
typedef struct SMTPThreadCtx_ {
190234
MpmThreadCtx *smtp_mpm_thread_ctx;
191235
PrefilterRuleStore *pmq;
@@ -665,6 +709,8 @@ static int SMTPInsertCommandIntoCommandBuffer(uint8_t command, SMTPState *state)
665709
return 0;
666710
}
667711

712+
static inline void SMTPTransactionComplete(SMTPState *state);
713+
668714
static int SMTPProcessCommandBDAT(SMTPState *state, const SMTPLine *line)
669715
{
670716
SCEnter();
@@ -677,6 +723,9 @@ static int SMTPProcessCommandBDAT(SMTPState *state, const SMTPLine *line)
677723
SCReturnInt(-1);
678724
} else if (state->bdat_chunk_idx == state->bdat_chunk_len) {
679725
state->parser_state &= ~SMTP_PARSER_STATE_COMMAND_DATA_MODE;
726+
if (state->bdat_last) {
727+
SMTPTransactionComplete(state);
728+
}
680729
}
681730

682731
SCReturnInt(0);
@@ -948,6 +997,7 @@ static int SMTPProcessReply(
948997
}
949998
} else if (IsReplyToCommand(state, SMTP_COMMAND_DATA)) {
950999
if (reply_code == SMTP_REPLY_354) {
1000+
SMTPSetProgressTC(state->curr_tx, SMTP_RESPONSE_DATA);
9511001
/* Next comes the mail for the DATA command in toserver direction */
9521002
state->parser_state |= SMTP_PARSER_STATE_COMMAND_DATA_MODE;
9531003
} else {
@@ -958,6 +1008,8 @@ static int SMTPProcessReply(
9581008
}
9591009
SMTPSetEvent(state, SMTP_DECODER_EVENT_DATA_COMMAND_REJECTED);
9601010
}
1011+
} else if (IsReplyToCommand(state, SMTP_COMMAND_BDAT)) {
1012+
SMTPSetProgressTC(state->curr_tx, SMTP_RESPONSE_DATA);
9611013
} else if (IsReplyToCommand(state, SMTP_COMMAND_RSET)) {
9621014
if (reply_code == SMTP_REPLY_250 && state->curr_tx &&
9631015
!(state->parser_state & SMTP_PARSER_STATE_PARSING_MULTILINE_REPLY)) {
@@ -1009,18 +1061,41 @@ static int SMTPParseCommandBDAT(SMTPState *state, const SMTPLine *line)
10091061
/* decoder event */
10101062
return -1;
10111063
}
1064+
1065+
const int chunk_len_start = i;
1066+
while (i < line->len) {
1067+
if (line->buf[i] == ' ') {
1068+
break;
1069+
}
1070+
i++;
1071+
}
1072+
10121073
// copy in temporary null-terminated buffer for conversion
10131074
char strbuf[24];
10141075
int len = 23;
1015-
if (line->len - i < len) {
1016-
len = line->len - i;
1076+
if (i - chunk_len_start < len) {
1077+
len = i - chunk_len_start;
10171078
}
1018-
memcpy(strbuf, line->buf + i, len);
1079+
memcpy(strbuf, line->buf + chunk_len_start, len);
10191080
strbuf[len] = '\0';
10201081
if (ByteExtractStringUint32(&state->bdat_chunk_len, 10, 0, strbuf) < 0) {
10211082
/* decoder event */
10221083
return -1;
10231084
}
1085+
state->bdat_chunk_idx = 0;
1086+
state->bdat_last = false;
1087+
1088+
while (i < line->len && line->buf[i] == ' ') {
1089+
i++;
1090+
}
1091+
if (i < line->len) {
1092+
if ((line->len - i) == 4 && SCMemcmpLowercase("last", line->buf + i, 4) == 0) {
1093+
state->bdat_last = true;
1094+
} else {
1095+
/* decoder event */
1096+
return -1;
1097+
}
1098+
}
10241099

10251100
return 0;
10261101
}
@@ -1193,6 +1268,7 @@ static int SMTPProcessRequest(
11931268
state->current_command = SMTP_COMMAND_STARTTLS;
11941269
} else if (line->len >= 4 && SCMemcmpLowercase("data", line->buf, 4) == 0) {
11951270
state->current_command = SMTP_COMMAND_DATA;
1271+
SMTPSetProgressTS(tx, SMTP_REQUEST_DATA);
11961272
if (state->curr_tx->is_data) {
11971273
// We did not receive a confirmation from server
11981274
// And now client sends a next DATA
@@ -1233,7 +1309,15 @@ static int SMTPProcessRequest(
12331309
SCReturnInt(-1);
12341310
}
12351311
state->current_command = SMTP_COMMAND_BDAT;
1236-
state->parser_state |= SMTP_PARSER_STATE_COMMAND_DATA_MODE;
1312+
SMTPSetProgressTS(tx, SMTP_REQUEST_DATA);
1313+
if (state->bdat_chunk_len == 0) {
1314+
state->parser_state &= ~SMTP_PARSER_STATE_COMMAND_DATA_MODE;
1315+
if (state->bdat_last) {
1316+
SMTPTransactionComplete(state);
1317+
}
1318+
} else {
1319+
state->parser_state |= SMTP_PARSER_STATE_COMMAND_DATA_MODE;
1320+
}
12371321
} else if (line->len >= 4 && ((SCMemcmpLowercase("helo", line->buf, 4) == 0) ||
12381322
SCMemcmpLowercase("ehlo", line->buf, 4) == 0)) {
12391323
r = SMTPParseCommandHELO(state, line);
@@ -1256,6 +1340,7 @@ static int SMTPProcessRequest(
12561340
} else if (line->len >= 4 && SCMemcmpLowercase("rset", line->buf, 4) == 0) {
12571341
// Resets chunk index in case of connection reuse
12581342
state->bdat_chunk_idx = 0;
1343+
state->bdat_last = false;
12591344
state->current_command = SMTP_COMMAND_RSET;
12601345
} else {
12611346
state->current_command = SMTP_COMMAND_OTHER_CMD;
@@ -1809,7 +1894,9 @@ static void *SMTPStateGetTx(void *state, uint64_t id)
18091894
static int SMTPStateGetAlstateProgress(void *vtx, uint8_t direction)
18101895
{
18111896
SMTPTransaction *tx = vtx;
1812-
return tx->done;
1897+
if (direction & STREAM_TOSERVER)
1898+
return tx->done ? SMTP_REQUEST_COMPLETE : tx->progress_ts;
1899+
return tx->done ? SMTP_RESPONSE_COMPLETE : tx->progress_tc;
18131900
}
18141901

18151902
static AppLayerGetFileState SMTPGetTxFiles(void *txv, uint8_t direction)
@@ -1912,9 +1999,12 @@ void RegisterSMTPParsers(void)
19121999
AppLayerParserRegisterGetTxIterator(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetTxIterator);
19132000
AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetTxData);
19142001
AppLayerParserRegisterStateDataFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetStateData);
1915-
AppLayerParserRegisterStateProgressCompletionStatus(ALPROTO_SMTP, 1, 1);
2002+
AppLayerParserRegisterStateProgressCompletionStatus(
2003+
ALPROTO_SMTP, SMTP_REQUEST_COMPLETE, SMTP_RESPONSE_COMPLETE);
19162004
AppLayerParserRegisterGetFrameFuncs(
19172005
IPPROTO_TCP, ALPROTO_SMTP, SMTPGetFrameIdByName, SMTPGetFrameNameById);
2006+
AppLayerParserRegisterGetStateFuncs(
2007+
IPPROTO_TCP, ALPROTO_SMTP, SMTPStateGetStateIdByName, SMTPStateGetStateNameById);
19182008
} else {
19192009
SCLogInfo("Parser disabled for %s protocol. Protocol detection still on.", proto_name);
19202010
}

‎src/app-layer-smtp.h‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,18 @@ typedef struct SMTPString_ {
7070
TAILQ_ENTRY(SMTPString_) next;
7171
} SMTPString;
7272

73+
enum SMTPRequestProgress {
74+
SMTP_REQUEST_STARTED = 0,
75+
SMTP_REQUEST_DATA = 1,
76+
SMTP_REQUEST_COMPLETE = 2,
77+
};
78+
79+
enum SMTPResponseProgress {
80+
SMTP_RESPONSE_STARTED = 0,
81+
SMTP_RESPONSE_DATA = 1,
82+
SMTP_RESPONSE_COMPLETE = 2,
83+
};
84+
7385
typedef struct SMTPTransaction_ {
7486
/** id of this tx, starting at 0 */
7587
uint64_t tx_id;
@@ -78,6 +90,10 @@ typedef struct SMTPTransaction_ {
7890

7991
/** the tx is complete and can be logged and cleaned */
8092
bool done;
93+
/** to-server firewall progress state. */
94+
uint8_t progress_ts;
95+
/** to-client firewall progress state. */
96+
uint8_t progress_tc;
8197
/** the tx has seen a DATA command */
8298
// another DATA command within the same context
8399
// will trigger an app-layer event.
@@ -134,6 +150,8 @@ typedef struct SMTPState_ {
134150
uint32_t bdat_chunk_len;
135151
/** bdat chunk idx */
136152
uint32_t bdat_chunk_idx;
153+
/** bdat chunk is the final message chunk */
154+
bool bdat_last;
137155

138156
/* the request commands are store here and the reply handler uses these
139157
* stored command in the buffer to match the reply(ies) with the command */

‎src/detect-email.c‎

Lines changed: 28 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -260,80 +260,87 @@ void DetectEmailRegister(void)
260260
kw.Setup = DetectMimeEmailFromSetup;
261261
kw.flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER;
262262
SCDetectHelperKeywordRegister(&kw);
263-
g_mime_email_from_buffer_id = SCDetectHelperBufferMpmRegister(
264-
"email.from", "MIME EMAIL FROM", ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailFromData);
263+
g_mime_email_from_buffer_id =
264+
SCDetectHelperBufferProgressMpmRegister("email.from", "MIME EMAIL FROM", ALPROTO_SMTP,
265+
STREAM_TOSERVER, GetMimeEmailFromData, SMTP_REQUEST_DATA);
265266

266267
kw.name = "email.subject";
267268
kw.desc = "'Subject' field from an email";
268269
kw.url = "/rules/email-keywords.html#email-subject";
269270
kw.Setup = DetectMimeEmailSubjectSetup;
270271
kw.flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER;
271272
SCDetectHelperKeywordRegister(&kw);
272-
g_mime_email_subject_buffer_id = SCDetectHelperBufferMpmRegister("email.subject",
273-
"MIME EMAIL SUBJECT", ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailSubjectData);
273+
g_mime_email_subject_buffer_id =
274+
SCDetectHelperBufferProgressMpmRegister("email.subject", "MIME EMAIL SUBJECT",
275+
ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailSubjectData, SMTP_REQUEST_DATA);
274276

275277
kw.name = "email.to";
276278
kw.desc = "'To' field from an email";
277279
kw.url = "/rules/email-keywords.html#email-to";
278280
kw.Setup = DetectMimeEmailToSetup;
279281
kw.flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER;
280282
SCDetectHelperKeywordRegister(&kw);
281-
g_mime_email_to_buffer_id = SCDetectHelperBufferMpmRegister(
282-
"email.to", "MIME EMAIL TO", ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailToData);
283+
g_mime_email_to_buffer_id = SCDetectHelperBufferProgressMpmRegister("email.to", "MIME EMAIL TO",
284+
ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailToData, SMTP_REQUEST_DATA);
283285

284286
kw.name = "email.cc";
285287
kw.desc = "'Cc' field from an email";
286288
kw.url = "/rules/email-keywords.html#email-cc";
287289
kw.Setup = DetectMimeEmailCcSetup;
288290
kw.flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER;
289291
SCDetectHelperKeywordRegister(&kw);
290-
g_mime_email_cc_buffer_id = SCDetectHelperBufferMpmRegister(
291-
"email.cc", "MIME EMAIL CC", ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailCcData);
292+
g_mime_email_cc_buffer_id = SCDetectHelperBufferProgressMpmRegister("email.cc", "MIME EMAIL CC",
293+
ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailCcData, SMTP_REQUEST_DATA);
292294

293295
kw.name = "email.date";
294296
kw.desc = "'Date' field from an email";
295297
kw.url = "/rules/email-keywords.html#email-date";
296298
kw.Setup = DetectMimeEmailDateSetup;
297299
kw.flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER;
298300
SCDetectHelperKeywordRegister(&kw);
299-
g_mime_email_date_buffer_id = SCDetectHelperBufferMpmRegister(
300-
"email.date", "MIME EMAIL DATE", ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailDateData);
301+
g_mime_email_date_buffer_id =
302+
SCDetectHelperBufferProgressMpmRegister("email.date", "MIME EMAIL DATE", ALPROTO_SMTP,
303+
STREAM_TOSERVER, GetMimeEmailDateData, SMTP_REQUEST_DATA);
301304

302305
kw.name = "email.message_id";
303306
kw.desc = "'Message-Id' field from an email";
304307
kw.url = "/rules/email-keywords.html#email-message-id";
305308
kw.Setup = DetectMimeEmailMessageIdSetup;
306309
kw.flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER;
307310
SCDetectHelperKeywordRegister(&kw);
308-
g_mime_email_message_id_buffer_id = SCDetectHelperBufferMpmRegister("email.message_id",
309-
"MIME EMAIL Message-Id", ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailMessageIdData);
311+
g_mime_email_message_id_buffer_id =
312+
SCDetectHelperBufferProgressMpmRegister("email.message_id", "MIME EMAIL Message-Id",
313+
ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailMessageIdData, SMTP_REQUEST_DATA);
310314

311315
kw.name = "email.x_mailer";
312316
kw.desc = "'X-Mailer' field from an email";
313317
kw.url = "/rules/email-keywords.html#email-x-mailer";
314318
kw.Setup = DetectMimeEmailXMailerSetup;
315319
kw.flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER;
316320
SCDetectHelperKeywordRegister(&kw);
317-
g_mime_email_x_mailer_buffer_id = SCDetectHelperBufferMpmRegister("email.x_mailer",
318-
"MIME EMAIL X-Mailer", ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailXMailerData);
321+
g_mime_email_x_mailer_buffer_id =
322+
SCDetectHelperBufferProgressMpmRegister("email.x_mailer", "MIME EMAIL X-Mailer",
323+
ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailXMailerData, SMTP_REQUEST_DATA);
319324

320325
kw.name = "email.url";
321326
kw.desc = "'Url' extracted from an email";
322327
kw.url = "/rules/email-keywords.html#email-url";
323328
kw.Setup = DetectMimeEmailUrlSetup;
324329
kw.flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER | SIGMATCH_INFO_MULTI_BUFFER;
325330
SCDetectHelperKeywordRegister(&kw);
326-
g_mime_email_url_buffer_id = SCDetectHelperMultiBufferMpmRegister(
327-
"email.url", "MIME EMAIL URL", ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailUrlData);
331+
g_mime_email_url_buffer_id =
332+
SCDetectHelperMultiBufferProgressMpmRegister("email.url", "MIME EMAIL URL",
333+
ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailUrlData, SMTP_REQUEST_DATA);
328334

329335
kw.name = "email.received";
330336
kw.desc = "'Received' field from an email";
331337
kw.url = "/rules/email-keywords.html#email-received";
332338
kw.Setup = DetectMimeEmailReceivedSetup;
333339
kw.flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER | SIGMATCH_INFO_MULTI_BUFFER;
334340
SCDetectHelperKeywordRegister(&kw);
335-
g_mime_email_received_buffer_id = SCDetectHelperMultiBufferMpmRegister("email.received",
336-
"MIME EMAIL RECEIVED", ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailReceivedData);
341+
g_mime_email_received_buffer_id =
342+
SCDetectHelperMultiBufferProgressMpmRegister("email.received", "MIME EMAIL RECEIVED",
343+
ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailReceivedData, SMTP_REQUEST_DATA);
337344

338345
if (!MimeBodyMd5IsDisabled()) {
339346
// do not register the keyword if explicitly disabled
@@ -343,10 +350,9 @@ void DetectEmailRegister(void)
343350
kw.Setup = DetectMimeEmailBodyMd5Setup;
344351
kw.flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER;
345352
DETECT_EMAIL_BODY_MD5 = SCDetectHelperKeywordRegister(&kw);
346-
// We do not need a progress because SMTP tx has only progress 0 or 1
347-
// even if we have a MimeSmtpMd5State enumeration
348-
g_mime_email_body_md5_buffer_id = SCDetectHelperBufferMpmRegister("email.body_md5",
349-
"MIME EMAIL BODY MD5", ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailBodyMd5Data);
353+
g_mime_email_body_md5_buffer_id =
354+
SCDetectHelperBufferProgressMpmRegister("email.body_md5", "MIME EMAIL BODY MD5",
355+
ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailBodyMd5Data, SMTP_REQUEST_DATA);
350356
DetectBufferTypeRegisterValidateCallback("email.body_md5", DetectMd5ValidateCallback);
351357
}
352358
}

‎src/detect-file-data.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,8 @@ DetectFileHandlerProtocol al_protocols[ALPROTO_WITHFILES_MAX] = {
9393
.direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT,
9494
.progress_tc = HTTP2ProgData,
9595
.progress_ts = HTTP2ProgData },
96-
{ .alproto = ALPROTO_SMTP, .direction = SIG_FLAG_TOSERVER }, { .alproto = ALPROTO_UNKNOWN }
96+
{ .alproto = ALPROTO_SMTP, .direction = SIG_FLAG_TOSERVER, .progress_ts = SMTP_REQUEST_DATA },
97+
{ .alproto = ALPROTO_UNKNOWN }
9798
};
9899

99100
void DetectFileRegisterProto(

0 commit comments

Comments
 (0)