Added compresensive testing. This builds where agent V2 left off, and adds additional assertions and validations to ensure of expected final output

Author: kamalchaturvedi

internal/collector/securityviolationsprocessor/processor_test.go

@@ -0,0 +1 @@
+N/A,443,10.0.1.50,false,POST,security_policy_01,HTTPS,blocked,403,N/A,N/A::N/A,{SQL Injection Signatures}::{SQL Injection CVE-2021-1234},8080,N/A,9876543210123456789,N/A,4,2-api.example.com:2-/api/users,N/A,REJECTED,SECURITY_WAF_VIOLATION,Attack signature detected,<?xml version='1.0' encoding='UTF-8'?><BAD_MSG><violation_masks><block>414000000200c00-3a03030c30000072-8000000000000000-0</block><alarm>475f0ffcbbd0fea-befbf35cb000007e-f400000000000000-0</alarm><learn>0-0-0-0</learn><staging>0-0-0-0</staging></violation_masks><request-violations><violation><viol_index>42</viol_index><viol_name>VIOL_ATTACK_SIGNATURE</viol_name><context>parameter</context><context_data><param_data><name>id</name><value>MScgT1IgJzEnPScx</value></param_data></context_data><sig_data><sig_id>200001475</sig_id><blocking_mask>7</blocking_mask><kw_data><buffer>MScgT1IgJzEnPScx</buffer><offset>0</offset><length>15</length></kw_data></sig_data><sig_data><sig_id>200001476</sig_id><blocking_mask>7</blocking_mask><kw_data><buffer>MScgT1IgJzEnPScx</buffer><offset>5</offset><length>10</length></kw_data></sig_data></violation></request-violations></BAD_MSG>,PostmanRuntime,HTTP Library,7.29.0,N/A,Trusted Bot,N/A,N/A,HTTP/2.0,/api/users,POST /api/users HTTP/2.0\\r\\nHost: api.example.com\\r\\nContent-Type: application/json\\r\\n\\r\\n{"id":"1' OR '1'='1"}

internal/collector/securityviolationsprocessor/testdata/csv_sql_injection_parameter_signatures.log.txt

@@ -0,0 +1 @@
+N/A,80,127.0.0.1,false,GET,nms_app_protect_default_policy,HTTP,blocked,0,N/A,N/A::N/A,{High Accuracy Signatures;Cross Site Scripting Signatures}::{High Accuracy Signatures; Cross Site Scripting Signatures},56064,N/A,5377540117854870581,N/A,5,1-localhost:1-/,N/A,REJECTED,SECURITY_WAF_VIOLATION,Illegal meta character in URL::Attack signature detected::Violation Rating Threat detected::Bot Client Detected,<?xml version='1.0' encoding='UTF-8'?><BAD_MSG><violation_masks><block>414000000200c00-3a03030c30000072-8000000000000000-0</block><alarm>475f0ffcbbd0fea-befbf35cb000007e-f400000000000000-0</alarm><learn>0-0-0-0</learn><staging>0-0-0-0</staging></violation_masks><request-violations><violation><viol_index>42</viol_index><viol_name>VIOL_ATTACK_SIGNATURE</viol_name><context>url</context><sig_data><sig_id>200000099</sig_id><blocking_mask>3</blocking_mask><kw_data><buffer>Lzw+PHNjcmlwdD4=</buffer><offset>3</offset><length>7</length></kw_data></sig_data><sig_data><sig_id>200000093</sig_id><blocking_mask>3</blocking_mask><kw_data><buffer>Lzw+PHNjcmlwdD4=</buffer><offset>4</offset><length>7</length></kw_data></sig_data></violation><violation><viol_index>26</viol_index><viol_name>VIOL_URL_METACHAR</viol_name><uri>Lzw+PHNjcmlwdD4=</uri><metachar_index>60</metachar_index><wildcard_entity>*</wildcard_entity><staging>0</staging></violation><violation><viol_index>26</viol_index><viol_name>VIOL_URL_METACHAR</viol_name><uri>Lzw+PHNjcmlwdD4=</uri><metachar_index>62</metachar_index><wildcard_entity>*</wildcard_entity><staging>0</staging></violation><violation><viol_index>122</viol_index><viol_name>VIOL_BOT_CLIENT</viol_name></violation><violation><viol_index>93</viol_index><viol_name>VIOL_RATING_THREAT</viol_name></violation></request-violations></BAD_MSG>,curl,HTTP Library,N/A,N/A,Untrusted Bot,N/A,N/A,HTTP/1.1,/<><script>,GET /<><script> HTTP/1.1\\r\\nHost: localhost\\r\\nUser-Agent: curl/7.81.0\\r\\nAccept: */*\\r\\n\\r\\n

internal/collector/securityviolationsprocessor/testdata/csv_url_violations_bot_client.log.txt

@@ -0,0 +1 @@
+N/A,80,127.0.0.1,,GET,app_protect_default_policy,HTTP,blocked,0,Critical,::,{Cross Site Scripting Signatures;High Accuracy Signatures}::{Cross Site Scripting Signatures;High Accuracy Signatures},61478,HTTP protocol compliance failed:Host header contains IP address::HTTP protocol compliance failed:Evasion technique,4355056874564592513,campaign1::campaign2,5,1-localhost:1-/,N/A,REJECTED,SECURITY_WAF_VIOLATION,HTTP protocol compliance failed::Illegal meta character in value::Attack signature detected::Violation Rating Threat detected::Bot Client Detected,<?xml version='1.0' encoding='UTF-8'?><BAD_MSG><violation_masks><block>410000000200c00-3a03030c30000072-8000000000000000-0</block><alarm>477f0ffcbbd0fea-befbf35cb000007e-8000000000000000-0</alarm><learn>0-20-0-0</learn><staging>0-0-0-0</staging></violation_masks><request-violations><violation><viol_index>42</viol_index><viol_name>VIOL_ATTACK_SIGNATURE</viol_name><context>parameter</context><parameter_data><value_error/><enforcement_level>global</enforcement_level><name>YQ==</name><auto_detected_type>alpha-numeric</auto_detected_type><value>PHNjcmlwdD4=</value><location>query</location><param_name_pattern>*</param_name_pattern><staging>0</staging></parameter_data><staging>0</staging><sig_data><sig_id>200001475</sig_id><blocking_mask>3</blocking_mask><kw_data><buffer>YT08c2NyaXB0Pg==</buffer><offset>3</offset><length>7</length></kw_data></sig_data><sig_data><sig_id>200000098</sig_id><blocking_mask>3</blocking_mask><kw_data><buffer>YT08c2NyaXB0Pg==</buffer><offset>2</offset><length>7</length></kw_data></sig_data></violation><violation><viol_index>14</viol_index><viol_name>VIOL_HTTP_PROTOCOL</viol_name><http_sanity_checks_status>2048</http_sanity_checks_status><http_sub_violation_status>2048</http_sub_violation_status><http_sub_violation>SG9zdCBoZWFkZXIgd2l0aCBJUCB2YWx1ZTogMTAuMTQ2LjE3OS4xMTk=</http_sub_violation></violation><violation><viol_index>24</viol_index><viol_name>VIOL_PARAMETER_VALUE_METACHAR</viol_name><parameter_data><value_error/><enforcement_level>global</enforcement_level><name>YQ==</name><auto_detected_type>alpha-numeric</auto_detected_type><value>PHNjcmlwdD4=</value><location>query</location></parameter_data><wildcard_entity>*</wildcard_entity><staging>0</staging><language_type>4</language_type><metachar_index>60</metachar_index><metachar_index>62</metachar_index></violation></request-violations></BAD_MSG>,curl,HTTP Library,N/A,N/A,Untrusted Bot,N/A,N/A,HTTP/1.1,/,GET /?a=<script> HTTP/1.1\r\nHost: 127.0.0.1\r\nUser-Agent: curl/7.64.1\r\nAccept: */*\r\n\r\n

internal/collector/securityviolationsprocessor/testdata/expanded_nap_waf.log.txt

@@ -0,0 +1 @@
+not a syslog line

internal/collector/securityviolationsprocessor/testdata/invalid_syslog_plain_text.log.txt

@@ -0,0 +1 @@
+N/A,80,127.0.0.1,,GET,app_protect_default_policy,HTTP,blocked,0,Critical,::,{Cross Site Scripting Signatures;High Accuracy Signatures}::{Cross Site Scripting Signatures;High Accuracy Signatures},61478,HTTP protocol compliance failed:Host header contains IP address::HTTP protocol compliance failed:Evasion technique,5543056874564592516,campaign1::campaign2,5,1-localhost:1-/,N/A,REJECTED,SECURITY_WAF_VIOLATION,HTTP protocol compliance failed::Illegal meta character in value::Attack signature detected::Violation Rating Threat detected::Bot Client Detected,<?xml version='1.0' encoding='UTF-8'?><BAD_MSG><violation_masks><block>410000000200c00-3a03030c30000072-8000000000000000-0</block><alarm>477f0ffcbbd0fea-befbf35cb000007e-8000000000000000-0</alarm><learn>0-20-0-0</learn><staging>0-0-0-0</staging></violation_masks><request-violations><violation><viol_index>42</viol_index><viol_name>VIOL_ATTACK_SIGNATURE</viol_name><context>parameter</context><parameter_data><value_error/><enforcement_level>global</enforcement_level><name>YQ==</name><auto_detected_type>alpha-numeric</auto_detected_type><value>PHNjcmlwdD4=</value><location>query</location><param_name_pattern>*</param_name_pattern><staging>0</staging></parameter_data><staging>0</staging><sig_data><sig_id>200001475</sig_id><blocking_mask>3</blocking_mask><kw_data><buffer>YT08c2NyaXB0Pg==</buffer><offset>3</offset><length>7</length></kw_data></sig_data><sig_data><sig_id>200000098</sig_id><blocking_mask>3</blocking_mask><kw_data><buffer>YT08c2NyaXB0Pg==</buffer><offset>2</offset><length>7</length></kw_data></sig_data></violation><violation><viol_index>14</viol_index><viol_name>VIOL_HTTP_PROTOCOL</viol_name><http_sanity_checks_status>2048</http_sanity_checks_status><http_sub_violation_status>2048</http_sub_violation_status><http_sub_violation>SG9zdCBoZWFkZXIgd2l0aCBJUCB2YWx1ZTogMTAuMTQ2LjE3OS4xMTk=</http_sub_violation></violation><violation><viol_index>24</viol_index><viol_name>VIOL_PARAMETER_VALUE_METACHAR</viol_name><parameter_data><value_error/><enforcement_level>global</enforcement_level><name>YQ==</name><auto_detected_type>alpha-numeric</auto_detected_type><value>PHNjcmlwdD4=</value><location>query</location></parameter_data><wildcard_entity>*</wildcard_entity><staging>0</staging><language_type>4</language_type><metachar_index>60</metachar_index><metachar_index>62</metachar_index></violation></request-violations></BAD_MSG>,curl,HTTP Library,N/A,N/A,Untrusted Bot,N/A,N/A,HTTP/1.1,/

internal/collector/securityviolationsprocessor/testdata/syslog_logline_less_fields.log.txt

@@ -0,0 +1 @@
+N/A,80,127.0.0.1,,GET,app_protect_default_policy,HTTP,blocked,0,Critical,::,{Cross Site Scripting Signatures;High Accuracy Signatures}::{Cross Site Scripting Signatures;High Accuracy Signatures},61478,HTTP protocol compliance failed:Host header contains IP address::HTTP protocol compliance failed:Evasion technique,5543056874564592517,campaign1::campaign2,5,1-localhost:1-/,N/A,REJECTED,SECURITY_WAF_VIOLATION,HTTP protocol compliance failed::Illegal meta character in value::Attack signature detected::Violation Rating Threat detected::Bot Client Detected,<?xml version='1.0' encoding='UTF-8'?><BAD_MSG><violation_masks><block>410000000200c00-3a03030c30000072-8000000000000000-0</block><alarm>477f0ffcbbd0fea-befbf35cb000007e-8000000000000000-0</alarm><learn>0-20-0-0</learn><staging>0-0-0-0</staging></violation_masks><request-violations><violation><viol_index>42</viol_index><viol_name>VIOL_ATTACK_SIGNATURE</viol_name><context>parameter</context><parameter_data><value_error/><enforcement_level>global</enforcement_level><name>YQ==</name><auto_detected_type>alpha-numeric</auto_detected_type><value>PHNjcmlwdD4=</value><location>query</location><param_name_pattern>*</param_name_pattern><staging>0</staging></parameter_data><staging>0</staging><sig_data><sig_id>200001475</sig_id><blocking_mask>3</blocking_mask><kw_data><buffer>YT08c2NyaXB0Pg==</buffer><offset>3</offset><length>7</length></kw_data></sig_data><sig_data><sig_id>200000098</sig_id><blocking_mask>3</blocking_mask><kw_data><buffer>YT08c2NyaXB0Pg==</buffer><offset>2</offset><length>7</length></kw_data></sig_data></violation><violation><viol_index>14</viol_index><viol_name>VIOL_HTTP_PROTOCOL</viol_name><http_sanity_checks_status>2048</http_sanity_checks_status><http_sub_violation_status>2048</http_sub_violation_status><http_sub_violation>SG9zdCBoZWFkZXIgd2l0aCBJUCB2YWx1ZTogMTAuMTQ2LjE3OS4xMTk=</http_sub_violation></violation><violation><viol_index>24</viol_index><viol_name>VIOL_PARAMETER_VALUE_METACHAR</viol_name><parameter_data><value_error/><enforcement_level>global</enforcement_level><name>YQ==</name><auto_detected_type>alpha-numeric</auto_detected_type><value>PHNjcmlwdD4=</value><location>query</location></parameter_data><wildcard_entity>*</wildcard_entity><staging>0</staging><language_type>4</language_type><metachar_index>60</metachar_index><metachar_index>62</metachar_index></violation></request-violations></BAD_MSG>,curl,HTTP Library,N/A,N/A,Untrusted Bot,N/A,N/A,HTTP/1.1,/,GET /?a=<script> HTTP/1.1\r\nHost: 127.0.0.1\r\nUser-Agent: curl/7.64.1\r\nAccept: */*\r\n\r\n,extra_field

internal/collector/securityviolationsprocessor/testdata/syslog_logline_more_fields.log.txt

@@ -0,0 +1 @@
+<34>Oct 11 22:14:15 mymachine su: 'su root' failed for lonvick on /dev/pts/8

internal/collector/securityviolationsprocessor/testdata/syslog_non_app_protect.log.txt

@@ -0,0 +1 @@
+N/A,80,127.0.0.1,,GET,app_protect_default_policy,HTTP,blocked,0,Critical,::,{Cross Site Scripting Signatures;High Accuracy Signatures}::{Cross Site Scripting Signatures;High Accuracy Signatures},61478,HTTP protocol compliance failed:Host header contains IP address::HTTP protocol compliance failed:Evasion technique,4355056874564592513,campaign1::campaign2,5,1-localhost:1-/,N/A,REJECTED,SECURITY_WAF_VIOLATION,HTTP protocol compliance failed::Illegal meta character in value::Attack signature detected::Violation Rating Threat detected::Bot Client Detected,<?xml version='1.0' encoding='UTF-8'?><BAD_MSG><violation_masks><block>410000000200c00-3a03030c30000072-8000000000000000-0</block><alarm>477f0ffcbbd0fea-befbf35cb000007e-8000000000000000-0</alarm><learn>0-20-0-0</learn><staging>0-0-0-0</staging></violation_masks><request-violations><violation><viol_index>42</viol_index><viol_name>VIOL_ATTACK_SIGNATURE</viol_name><context>parameter</context><parameter_data><value_error/><enforcement_level>global</enforcement_level><name>YQ==</name><auto_detected_type>alpha-numeric</auto_detected_type><value>PHNjcmlwdD4=</value><location>query</location><param_name_pattern>*</param_name_pattern><staging>0</staging></parameter_data><staging>0</staging><sig_data><sig_id>200001475</sig_id><blocking_mask>3</blocking_mask><kw_data><buffer>YT08c2NyaXB0Pg==</buffer><offset>3</offset><length>7</length></kw_data></sig_data><sig_data><sig_id>200000098</sig_id><blocking_mask>3</blocking_mask><kw_data><buffer>YT08c2NyaXB0Pg==</buffer><offset>2</offset><length>7</length></kw_data></sig_data></violation><violation><viol_index>14</viol_index><viol_name>VIOL_HTTP_PROTOCOL</viol_name><http_sanity_checks_status>2048</http_sanity_checks_status><http_sub_violation_status>2048</http_sub_violation_status><http_sub_violation>SG9zdCBoZWFkZXIgd2l0aCBJUCB2YWx1ZTogMTAuMTQ2LjE3OS4xMTk=</http_sub_violation></violation><violation><viol_index>24</viol_index><viol_name>VIOL_PARAMETER_VALUE_METACHAR</viol_name><parameter_data><value_error/><enforcement_level>global</enforcement_level><name>YQ==</name><auto_detected_type>alpha-numeric</auto_detected_type><value>PHNjcmlwdD4=</value><location>query</location></parameter_data><wildcard_entity>*</wildcard_entity><staging>0</staging><language_type>4</language_type><metachar_index>60</metachar_index><metachar_index>62</metachar_index></violation></request-violations></BAD_MSG>,curl,HTTP Library,N/A,N/A,Untrusted Bot,N/A,N/A,HTTP/1.1,/with%2Ccomma,GET /with%2Ccomma HTTP/1.1\r\nHost: 10.146.183.68\r\nConnection: keep-alive\r\nCache-Control: max-age=0\r\nUpgrade-Insecure-Requests: 1\r\nUser-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML%2C like Gecko) Chrome/104.0.0.0 Safari/537.36\r\nAccept: text/html%2Capplication/xhtml+xml%2Capplication/xml;q=0.9%2Cimage/avif%2Cimage/webp%2Cimage/apng%2C*/*;q=0.8%2Capplication/signed-exchange;v=b3;q=0.9\r\nAccept-Encoding: gzip%2C deflate\r\nAccept-Language: en-US%2Cen;q=0.9\r\n\r\n

internal/collector/securityviolationsprocessor/testdata/uri_request_contain_escaped_comma.log.txt

@@ -0,0 +1 @@
+N/A,80,127.0.0.1,,GET,app_protect_default_policy,HTTP,blocked,0,Critical,::,{Cross Site Scripting Signatures;High Accuracy Signatures}::{Cross Site Scripting Signatures;High Accuracy Signatures},61478,HTTP protocol compliance failed:Host header contains IP address::HTTP protocol compliance failed:Evasion technique,3255056874564592514,campaign1::campaign2,5,1-localhost:1-/,N/A,REJECTED,SECURITY_WAF_VIOLATION,HTTP protocol compliance failed::Illegal meta character in value::Attack signature detected::Violation Rating Threat detected::Bot Client Detected,<?xml version='1.0' ?> <BAD_MSG> <violation_masks> <block>410000000200c00-3a03030c30000072-8000000000000000-0</block> <alarm>477f0ffcbbd0fea-befbf35cb000007e-8000000000000000-0</alarm> <learn>0-2-0-0</learn> <staging>0-0-0-0</staging> </violation_masks> <request-violations><violation><viol_index>18</viol_index><viol_name>VIOL_COOKIE_LENGTH</viol_name><cookie_len>28</cookie_len><cookie_len_limit>10</cookie_len_limit><cookie>Q29va2llOiBkZmRmZGZkZmRmPWRmZGZkZmRmOw==</cookie></violation></request-violations> </BAD_MSG>,curl,HTTP Library,N/A,N/A,Untrusted Bot,N/A,N/A,HTTP/1.1,/,GET /?a=<script> HTTP/1.1\r\nHost: 127.0.0.1\r\nUser-Agent: curl/7.64.1\r\nAccept: */*\r\n\r\n