update dev branch

assemblyline-models/src/datastore/tagging.rs

@@ -46,7 +46,7 @@ impl From<String> for TagValue {
 
 // MARK: Tag Processors
 #[derive(Debug)]
-enum TagProcessor {
+pub enum TagProcessor {
     // Generic strings
     String,
     Uppercase,
@@ -935,7 +935,25 @@ fn network_tag_parsing() {
     assert_eq!(proc.apply(json!("www.google.com")), Err(json!("www.google.com")));
     assert_eq!(proc.apply(json!("www.GooGle.com")), Err(json!("www.GooGle.com")));
     assert_eq!(proc.apply(json!("172.0.0.1")), Ok(json!("172.0.0.1")));
+    assert_eq!(proc.apply(json!("0.0.0.0")), Ok(json!("0.0.0.0")));
+    assert_eq!(proc.apply(json!("127.0.0.0")), Ok(json!("127.0.0.0")));
+    assert_eq!(proc.apply(json!("127.0.10.200")), Ok(json!("127.0.10.200")));
+    assert_eq!(proc.apply(json!("255.255.255.255")), Ok(json!("255.255.255.255")));
     assert_eq!(proc.apply(json!("1234:5678:9ABC:0000:0000:1234:5678:9abc")), Ok(json!("1234:5678:9ABC:0000:0000:1234:5678:9ABC")));
+    // we don't want abbrivated, octal, hex, or padded ip addresses
+    assert_eq!(proc.apply(json!("172.1")), Err(json!("172.1")));
+    assert_eq!(proc.apply(json!("256.0.0.1")), Err(json!("256.0.0.1")));
+    assert_eq!(proc.apply(json!("0.256.0.1")), Err(json!("0.256.0.1")));
+    assert_eq!(proc.apply(json!("0.0.256.1")), Err(json!("0.0.256.1")));
+    assert_eq!(proc.apply(json!("0.0.0.256")), Err(json!("0.0.0.256")));
+    assert_eq!(proc.apply(json!("172.0x1.0.1")), Err(json!("172.0x1.0.1")));
+    assert_eq!(proc.apply(json!("172.01.0.1")), Err(json!("172.01.0.1")));
+    assert_eq!(proc.apply(json!("172.1.0.00000000001")), Err(json!("172.1.0.00000000001")));
+    assert_eq!(proc.apply(json!("0.0.0.")), Err(json!("0.0.0.")));
+    assert_eq!(proc.apply(json!("0.0.0.0.")), Err(json!("0.0.0.0.")));
+    assert_eq!(proc.apply(json!(".0.0.0")), Err(json!(".0.0.0")));
+    assert_eq!(proc.apply(json!(".0.0.0.0")), Err(json!(".0.0.0.0")));
+
 
     let proc = TagProcessor::UNCPath;
     assert_eq!(proc.apply(json!("www.google.com")), Err(json!("www.google.com")));

assemblyline-models/src/types/strings.rs

@@ -40,7 +40,7 @@ impl std::ops::Deref for ServiceName {
 impl<'de> Deserialize<'de> for ServiceName {
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
-        D: serde::Deserializer<'de> 
+        D: serde::Deserializer<'de>
     {
         struct Visitor {}
 
@@ -81,17 +81,17 @@ impl From<&str> for ServiceName {
 }
 
 /// A string that maps to a keyword field in elasticsearch.
-/// 
+///
 /// This is the default behaviour for a String in a mapped struct, the only reason
 /// to use this over a standard String is cases where the 'mapping' field has been overwritten
 /// by a container and the more explicit 'mapping' this provided is needed to reassert
 /// the keyword type.
-/// 
+///
 /// Example:
 ///         #[metadata(store=false, mapping="flattenedobject")]
 ///         pub safelisted_tags: HashMap<String, Vec<Keyword>>,
-/// 
-/// In that example, if the inner Keyword was String the entire HashMap would have its 
+///
+/// In that example, if the inner Keyword was String the entire HashMap would have its
 /// mapping set to 'flattenedobject', the inner Keyword more explicitly overrides this.
 #[derive(Debug, Serialize, Deserialize, Described, Clone, PartialEq, Eq, PartialOrd, Ord)]
 #[metadata_type(ElasticMeta)]
@@ -238,8 +238,8 @@ impl Text {
 #[derive(Debug, thiserror::Error)]
 #[error("Could not process {original} as a {name}: {error}")]
 pub struct ValidationError {
-    original: String, 
-    name: &'static str, 
+    original: String,
+    name: &'static str,
     error: String
 }
 
@@ -371,11 +371,11 @@ pub fn check_domain(data: &str) -> Result<String, DomainError> {
                 return Err(DomainError::IlligalCharacter)
                 // raise ValueError(f"[{self.name or self.parent_name}] '{segment}' in '{value}' "
                 //                     f"includes a Unicode character that can not be normalized to '{segment_norm}'.")
-            } 
+            }
             normalized_parts.push(segment_norm);
         }
     }
-    
+
     let mut domain = normalized_parts.join(".");
 
     static VALIDATION_REGEX: LazyLock<Regex> = LazyLock::new(||{
@@ -391,7 +391,7 @@ pub fn check_domain(data: &str) -> Result<String, DomainError> {
     }
 
     if domain.contains("@") {
-        return Err(DomainError::IlligalCharacter)   
+        return Err(DomainError::IlligalCharacter)
     }
 
     if let Some((_, tld)) = domain.rsplit_once(".") {
@@ -411,7 +411,7 @@ pub fn check_domain(data: &str) -> Result<String, DomainError> {
     }
 
     Err(DomainError::InvalidTLD)
-} 
+}
 
 // def is_valid_domain(domain: str) -> bool:
 //     if "@" in domain:
@@ -461,14 +461,14 @@ fn system_local_tld() -> Vec<String> {
 fn find_top_level_domains() -> &'static HashSet<String> {
     static TLDS: LazyLock<HashSet<String>> = LazyLock::new(|| {
         use super::net_static::TLDS_ALPHA_BY_DOMAIN;
-        let mut combined_tlds = HashSet::<String>::new(); 
+        let mut combined_tlds = HashSet::<String>::new();
         combined_tlds.extend(TLDS_ALPHA_BY_DOMAIN.iter().map(|s|s.to_string()));
 
         for d in TLDS_SPECIAL_BY_DOMAIN {
             if !d.contains(".") {
                 combined_tlds.insert(d.to_owned());
             }
-        } 
+        }
 
         for tld in system_local_tld() {
             let tld = tld.trim_matches('.').to_uppercase();
@@ -497,11 +497,11 @@ pub type Domain = ValidatedString<DomainValidator>;
 
 #[test]
 fn internationalized_domains() {
-    assert_eq!(check_domain("ουτοπία.δπθ.gr").unwrap(), "ουτοπία.δπθ.gr"); 
+    assert_eq!(check_domain("ουτοπία.δπθ.gr").unwrap(), "ουτοπία.δπθ.gr");
     assert_eq!(check_domain("xn--kxae4bafwg.xn--pxaix.gr").unwrap(), "ουτοπία.δπθ.gr");
-    assert_eq!(check_domain("site.XN--W4RS40L").unwrap(), "site.嘉里"); 
-    assert!(check_domain("ουτοπία.δπθ.g").is_err()); 
-    assert!(check_domain("ουτοπία..gr").is_err()); 
+    assert_eq!(check_domain("site.XN--W4RS40L").unwrap(), "site.嘉里");
+    assert!(check_domain("ουτοπία.δπθ.g").is_err());
+    assert!(check_domain("ουτοπία..gr").is_err());
     assert!(check_domain("xn--kxae4bafwg.xn--pxaix.g").is_err());
     assert!(check_domain("xn--kxae4bafwg.xn--xaix.gr").is_err());
 }
@@ -596,8 +596,8 @@ pub type Uri = ValidatedString<UriValidator>;
 // pub type UriPath = String;
 
 // MARK: IP
+const IPV4_REGEX: &str = r"(?:(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.){3}(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])";
 
-const IPV4_REGEX: &str = r"(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)";
 const IPV6_REGEX: &str = concat!(
     r"(?:(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|",
     r"(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|",

assemblyline-server/src/constants.rs

@@ -19,6 +19,7 @@ pub(crate) const DISPATCH_TASK_HASH: &str = "dispatch-active-submissions";
 pub(crate) const SCALER_TIMEOUT_QUEUE: &str = "scaler-timeout-queue";
 pub(crate) const SERVICE_STATE_HASH: &str = "service-stasis-table";
 pub(crate) const SERVICE_QUEUE_PREFIX: &str = "service-queue-";
+pub(crate) const SERVICE_API_KEY_HASH: &str = "dynamic-service-keys";
 
 /// Take the name of a service, and provide the queue name to send tasks to that service.
 pub fn service_queue_name(service: &str) -> String {
@@ -130,3 +131,12 @@ pub const SERVICE_STAGE_KEY: &str = "service-stage";
 //     'critical': 500,
 //     'high': 100,
 // }
+
+
+/// An entry stored in redis in the SERVICE_API_KEY_HASH hash
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct ServiceApiKeyConfig {
+    pub key: String,
+    pub allow_registry_writing: bool,
+    pub expiry: chrono::DateTime<chrono::Utc>
+}

assemblyline-server/src/identify/default.magic

@@ -24,6 +24,10 @@
 # Open XML files with Microsoft Word
 0   string
 >0  search/0x100 =<?mso-application\ progid="Word.Document"?>    custom: document/office/word
+# MSBuild Project Files
+0      string
+>0     search/0x40    \<Project
+>>&0   search/0x40    http://schemas.microsoft.com/developer/msbuild    custom: code/xml/msbuild
 # VBE files
 0 string #@~^
 >&0 regex/9 \^[^=]{6}== custom: code/vbe

assemblyline-server/src/postprocessing/parsing.rs

@@ -6,12 +6,12 @@ use chrono::{DateTime, Utc, Duration, NaiveDate, NaiveDateTime, NaiveTime};
 use nom::{IResult, Parser};
 use nom::branch::alt;
 use nom::bytes::complete::{tag, take_while, escaped_transform, is_not, take_while1, tag_no_case, is_a};
-use nom::character::complete::{multispace0, alphanumeric1, one_of};
-use nom::combinator::{map, map_opt, map_res, opt, value};
+use nom::character::complete::{alphanumeric1, multispace0, multispace1, one_of};
+use nom::combinator::{eof, map, map_opt, map_res, opt, peek, value};
 use nom::error::ParseError;
-use nom::multi::{separated_list1, count, many1};
+use nom::multi::{count, many_till, many1, separated_list1};
 use nom::number::complete::double;
-use nom::sequence::{delimited, pair};
+use nom::sequence::{delimited, pair, terminated};
 
 use super::search::{Query, PrefixOperator, StringQuery, FieldQuery, RangeBound, RangeTerm, RangeQuery, DateExpression, DateUnit, NumberQuery};
 use super::ParsingError;
@@ -86,9 +86,9 @@ fn not_operator(input: &str) -> IResult<&str, ()> {
 fn atom(input: &str) -> IResult<&str, Query> {
     // println!("atom: {input}");
     alt((
-        delimited(ws(tag("(")), expression, ws(tag(")"))), 
+        delimited(ws(tag("(")), expression, ws(tag(")"))),
         exists,
-        field, 
+        field,
         term
     )).parse(input)
 }
@@ -133,7 +133,7 @@ fn prefix_operator(input: &str) -> IResult<&str, PrefixOperator> {
 fn is_special(value: char) -> bool {
     matches!(value, '_' | '-')
 }
-fn simple_term(input: &str) -> IResult<&str, String> {
+fn primitive_simple_term(input: &str) -> IResult<&str, String> {
     // println!("simple_term: {input}");
     map_res(escaped_transform(
         alt((alphanumeric1, take_while1(is_special))),
@@ -170,18 +170,33 @@ fn simple_term(input: &str) -> IResult<&str, String> {
     }).parse(input)
 }
 
+fn simple_term(input: &str) -> IResult<&str, String> {
+    // println!("simple_term: {input}");
+    terminated(primitive_simple_term, end_term).parse(input)
+}
+
 fn pattern_term(input: &str) -> IResult<&str, regex::Regex> {
-    // println!("pattern_term: {input}");
-    map_res(many1(alt((
-        map(tag("*"), |_|{String::from(".*")}),
-        map(tag("?"), |_|{String::from(".")}),
-        map(simple_term, |row|{regex::escape(&row)}),
-    ))), |parts|{
+    map_res(many_till(
+        alt((
+            map(tag("*"), |_|{String::from(".*")}),
+            map(tag("?"), |_|{String::from(".")}),
+            map(primitive_simple_term, |row|{regex::escape(&row)}),
+        )),
+        end_term
+    ), |(parts, _)|{
         let pattern = parts.join("");
         regex::Regex::new(&pattern)
     }).parse(input)
 }
 
+fn end_term(input: &str) -> IResult<&str, ()> {
+    alt((
+        map(peek(tag(")")), |_| ()),
+        map(eof, |_|()),
+        map(multispace1, |_| ()),
+    )).parse(input)
+}
+
 // phrase_term: ESCAPED_STRING
 fn phrase_term(input: &str) -> IResult<&str, String> {
     quoted_string(input)

assemblyline-server/src/postprocessing/search.rs

@@ -52,7 +52,7 @@ fn get_full_text_field_names() -> &'static Vec<Vec<String>> {
                     struct_metadata::Kind::Mapping(_, _) => {},
                     _ => {}
                 }
-                break 
+                break
             }
         }
 
@@ -673,7 +673,7 @@ fn check_field_type(_root: &str, tail: &[String], kind: &struct_metadata::Kind<E
                 0 => false,
                 1 => true,
                 _ => check_field_type(&tail[1], &tail[2..], &inner.kind)
-            }            
+            }
         },
 
         // Recurse into the inner type for these
@@ -683,7 +683,7 @@ fn check_field_type(_root: &str, tail: &[String], kind: &struct_metadata::Kind<E
             check_field_type(_root, tail, &kind.kind)
         }
 
-        // all remaining types are scalar, accept the field so long as its not trying 
+        // all remaining types are scalar, accept the field so long as its not trying
         // to select a subfield of a scalar
         _ => {
             tail.is_empty()

assemblyline-server/src/postprocessing/tests.rs

@@ -50,6 +50,9 @@ fn test_simple_filters() {
     assert!(!fltr.test(&sub).unwrap());
 
     // Try a prefix operator and wildcard matches
+    let _ = parse("max_score: >100 AND NOT results: virus").unwrap();
+    let _ = parse("max_score: >100 AND NOT results: virus*").unwrap();
+    let _ = parse("max_score: >100 AND NOT results: *virus").unwrap();
     let fltr = parse("max_score: >100 AND NOT results: *virus*").unwrap();
     assert_eq!(fltr.cache_safe(), CacheAvailabilityStatus::ErrorUsesForbiddenFields(vec!["results".to_owned()]));
 
@@ -96,6 +99,10 @@ fn test_simple_filters() {
     assert_eq!(fltr.cache_safe(), CacheAvailabilityStatus::Ok);
     assert!(fltr.test(&sub).unwrap());
 
+    let fltr = parse("metadata.stuff: big-*").unwrap();
+    assert_eq!(fltr.cache_safe(), CacheAvailabilityStatus::Ok);
+    assert!(fltr.test(&sub).unwrap());
+
     let fltr = parse("metadata.stuff: big\\-bad").unwrap();
     assert_eq!(fltr.cache_safe(), CacheAvailabilityStatus::Ok);
     assert!(fltr.test(&sub).unwrap());
@@ -299,15 +306,15 @@ async fn test_hook() {
     use assemblyline_models::datastore::submission::Submission;
 
     let (port, mut hits) = run_server().await;
-    
+
     let action = PostprocessAction{
         enabled: true,
         run_on_completed: true,
         filter: "metadata.do_hello: *".to_string(),
         webhook: Some(Webhook {
             uri: format!("http://localhost:{port}"),
             headers: vec![NamedValue{
-                name: "care-of".to_string(), 
+                name: "care-of".to_string(),
                 value: "assemblyline".to_string()
             }],
             password: None,
@@ -338,14 +345,14 @@ async fn test_hook() {
     {
         let mut sub: Submission = rand::rng().random();
         sub.metadata.insert("ok".to_string(), "bad".into());
-        worker.process(&sub, Default::default(), false).await.unwrap(); 
+        worker.process(&sub, Default::default(), false).await.unwrap();
     }
 
     {
         let mut sub: Submission = rand::rng().random();
         sub.metadata.insert("ok".to_string(), "good".into());
         sub.metadata.insert("do_hello".to_string(), "yes".into());
-        worker.process(&sub, Default::default(), false).await.unwrap(); 
+        worker.process(&sub, Default::default(), false).await.unwrap();
     }
 
     let (headers, body) = tokio::time::timeout(std::time::Duration::from_secs(3), hits.recv()).await.unwrap().unwrap();
@@ -360,11 +367,11 @@ async fn test_hook() {
 #[test]
 fn test_webhook_match() {
     let webhook_first = json!({
-        "uri": "http://api.interface.website"        
+        "uri": "http://api.interface.website"
     });
 
     let webhook_second = json!({
-        "uri": "http://api.interface.website",        
+        "uri": "http://api.interface.website",
         "headers": [{
             "name": "APIKEY",
             "value": "1111111111111111111111111111111111111111111111111111111111111111"