Add support for other regex engines (#26)

This add support for other regex engines (regex and fancy-regex), which helps users where the compilation requirements of onig may be onerous.

Note that fancy-regex is quite complete, but performs 4x to 8x slower than onig in nearly all cases. This feature is only recommended for specific cases.

Author: mmastrac

.github/workflows/ci.yml

@@ -30,4 +30,8 @@ jobs:
           toolchain: stable
       - uses: actions-rs/cargo@v1
         with:
-          command: test
+          command: test
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --no-default-features --features fancy-regex

Cargo.toml

@@ -17,8 +17,20 @@ build = "build.rs"
 edition = "2021"
 rust-version = "1.56"
 
+[features]
+default = ["onig"]
+
+onig = []
+regex = ["dep:regex"]
+fancy-regex = ["dep:fancy-regex"]
+
 [dependencies]
+# The default regex engine. Use default-feature = false to disable it.
 onig = { version = "6.5", default-features = false }
+# The Rust regex library. Does not support backtracking, so many patterns are unusable.
+regex = { version = "1", optional = true, default-features = false, features = ["std", "unicode", "perf", "perf-dfa-full"] }
+# A more complete Rust regex library supporting backtracking.
+fancy-regex = { version = "0.14", optional = true, default-features = false, features = ["std", "unicode", "perf"] }
 
 [build-dependencies]
 glob = "0.3"

README.md

@@ -48,7 +48,35 @@ be passed freely around. For performance reasons the `Match` returned is bound t
 them close together or clone/copy out the containing results as needed.
 
 ## Further Information
-This library depends on [onig](https://crates.io/crates/onig) for its regex execution, which itself is a Rust binding for the powerful [Oniguruma](https://github.com/kkos/oniguruma) regex library. If in doubt why a specific regex doesn't work, this is the best place to look for more information what patterns are supported and how to use advanced features.
+
+This library supports multiple regex engines through feature flags. By default,
+it uses [onig](https://crates.io/crates/onig), which is a Rust binding for the
+powerful [Oniguruma](https://github.com/kkos/oniguruma) regex library. You can
+also use the standard Rust regex engine or fancy-regex by enabling the
+respective features:
+
+The default engine, and at this time the most performant one, is `onig`:
+
+```toml
+[dependencies]
+grok = { version = "2.0", features = ["onig"] }
+```
+
+The `fancy-regex` engine is a more complete Rust regex library supporting
+backtracking:
+
+```toml
+[dependencies]
+grok = { version = "2.0", default-features = false, features = ["fancy-regex"] }
+```
+
+The `regex` engine is supported, but it does not support backtracking, so many
+patterns are unusable. This is not recommended for most use cases:
+
+```toml
+[dependencies]
+grok = { version = "2.0", default-features = false, features = ["regex"] }
+```
 
 ## License
 `grok` is distributed under the terms of the Apache License (Version 2.0). 

patterns/firewalls.pattern

@@ -83,9 +83,19 @@ CISCOFW713172 Group = %{GREEDYDATA:group}, IP = %{IP:src_ip}, Automatic NAT Dete
 CISCOFW733100 \[\s*%{DATA:drop_type}\s*\] drop %{DATA:drop_rate_id} exceeded. Current burst rate is %{INT:drop_rate_current_burst} per second, max configured rate is %{INT:drop_rate_max_burst}; Current average rate is %{INT:drop_rate_current_avg} per second, max configured rate is %{INT:drop_rate_max_avg}; Cumulative total count is %{INT:drop_total_count}
 #== End Cisco ASA ==
 
+IPTABLES_TCP_FLAGS (CWR |ECE |URG |ACK |PSH |RST |SYN |FIN )*
+IPTABLES_TCP_PART (?:SEQ=%{INT:[iptables][tcp][seq]:int}\s+)?(?:ACK=%{INT:[iptables][tcp][ack]:int}\s+)?WINDOW=%{INT:[iptables][tcp][window]:int}\s+RES=0x%{BASE16NUM:[iptables][tcp_reserved_bits]}\s+%{IPTABLES_TCP_FLAGS:[iptables][tcp][flags]}
+
+IPTABLES4_FRAG (?:(?<= )(?:CE|DF|MF))*
+IPTABLES4_PART SRC=%{IPV4:[source][ip]}\s+DST=%{IPV4:[destination][ip]}\s+LEN=(?:%{INT:[iptables][length]:int})?\s+TOS=(?:0|0x%{BASE16NUM:[iptables][tos]})?\s+PREC=(?:0x%{BASE16NUM:[iptables][precedence_bits]})?\s+TTL=(?:%{INT:[iptables][ttl]:int})?\s+ID=(?:%{INT:[iptables][id]})?\s+(?:%{IPTABLES4_FRAG:[iptables][fragment_flags]})?(?:\s+FRAG: %{INT:[iptables][fragment_offset]:int})?
+IPTABLES6_PART SRC=%{IPV6:[source][ip]}\s+DST=%{IPV6:[destination][ip]}\s+LEN=(?:%{INT:[iptables][length]:int})?\s+TC=(?:0|0x%{BASE16NUM:[iptables][tos]})?\s+HOPLIMIT=(?:%{INT:[iptables][ttl]:int})?\s+FLOWLBL=(?:%{INT:[iptables][flow_label]})?
+
+IPTABLES IN=(?:%{NOTSPACE:[observer][ingress][interface][name]})?\s+OUT=(?:%{NOTSPACE:[observer][egress][interface][name]})?\s+(?:MAC=(?:%{COMMONMAC:[destination][mac]})?(?::%{COMMONMAC:[source][mac]})?(?::[A-Fa-f0-9]{2}:[A-Fa-f0-9]{2})?\s+)?(:?%{IPTABLES4_PART}|%{IPTABLES6_PART}).*?PROTO=(?:%{WORD:[network][transport]})?\s+SPT=(?:%{INT:[source][port]:int})?\s+DPT=(?:%{INT:[destination][port]:int})?\s+(?:%{IPTABLES_TCP_PART})?
+
 # Shorewall firewall logs
-SHOREWALL (%{SYSLOGTIMESTAMP:timestamp}) (%{WORD:nf_host}) kernel:.*Shorewall:(%{WORD:nf_action1})?:(%{WORD:nf_action2})?.*IN=(%{USERNAME:nf_in_interface})?.*(OUT= *MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?|OUT=%{USERNAME:nf_out_interface}).*SRC=(%{IPV4:nf_src_ip}).*DST=(%{IPV4:nf_dst_ip}).*LEN=(%{WORD:nf_len}).?*TOS=(%{WORD:nf_tos}).?*PREC=(%{WORD:nf_prec}).?*TTL=(%{INT:nf_ttl}).?*ID=(%{INT:nf_id}).?*PROTO=(%{WORD:nf_protocol}).?*SPT=(%{INT:nf_src_port}?.*DPT=%{INT:nf_dst_port}?.*)
+SHOREWALL (?:%{SYSLOGTIMESTAMP:timestamp}) (?:%{WORD:[observer][hostname]}) .*Shorewall:(?:%{WORD:[shorewall][firewall][type]})?:(?:%{WORD:[shorewall][firewall][action]})?.*%{IPTABLES}
 #== End Shorewall
 #== SuSE Firewall 2 ==
-SFW2 ((%{SYSLOGTIMESTAMP})|(%{TIMESTAMP_ISO8601}))\s*%{HOSTNAME}\s*kernel\S+\s*%{NAGIOSTIME}\s*SFW2\-INext\-%{NOTSPACE:nf_action}\s*IN=%{USERNAME:nf_in_interface}.*OUT=((\s*%{USERNAME:nf_out_interface})|(\s*))MAC=((%{COMMONMAC:nf_dst_mac}:%{COMMONMAC:nf_src_mac})|(\s*)).*SRC=%{IP:nf_src_ip}\s*DST=%{IP:nf_dst_ip}.*PROTO=%{WORD:nf_protocol}((.*SPT=%{INT:nf_src_port}.*DPT=%{INT:nf_dst_port}.*)|())
+SFW2_LOG_PREFIX SFW2\-INext\-%{NOTSPACE:[suse][firewall][action]}
+SFW2 ((?:%{SYSLOGTIMESTAMP:timestamp})|(?:%{TIMESTAMP_ISO8601:timestamp}))\s*%{HOSTNAME:[observer][hostname]}.*?%{SFW2_LOG_PREFIX:[suse][firewall][log_prefix]}\s*%{IPTABLES}
 #== End SuSE ==

src/fancy_regex.rs

@@ -0,0 +1,116 @@
+use crate::Error;
+use fancy_regex::{Captures, Regex};
+use std::collections::{btree_map, BTreeMap, HashMap};
+
+/// The `Pattern` represents a compiled regex, ready to be matched against arbitrary text.
+#[derive(Debug)]
+pub struct FancyRegexPattern {
+    regex: Regex,
+    names: BTreeMap<String, usize>,
+}
+
+impl FancyRegexPattern {
+    /// Creates a new pattern from a raw regex string and an alias map to identify the
+    /// fields properly.
+    pub(crate) fn new(regex: &str, alias: &HashMap<String, String>) -> Result<Self, Error> {
+        match Regex::new(regex) {
+            Ok(r) => Ok({
+                let mut names = BTreeMap::new();
+                for (i, name) in r.capture_names().enumerate() {
+                    if let Some(name) = name {
+                        let name = match alias.iter().find(|&(_k, v)| *v == name) {
+                            Some(item) => item.0.clone(),
+                            None => String::from(name),
+                        };
+                        names.insert(name, i);
+                    }
+                }
+                Self { regex: r, names }
+            }),
+            Err(e) => Err(Error::RegexCompilationFailed(format!(
+                "Regex compilation failed: {e:?}:\n{regex}"
+            ))),
+        }
+    }
+
+    /// Matches this compiled `Pattern` against the text and returns the matches.
+    pub fn match_against<'a>(&'a self, text: &'a str) -> Option<FancyRegexMatches<'a>> {
+        self.regex.captures(text).ok().flatten().and_then(|caps| {
+            Some(FancyRegexMatches {
+                captures: caps,
+                pattern: self,
+            })
+        })
+    }
+
+    /// Returns all names this `Pattern` captures.
+    pub fn capture_names(&self) -> impl Iterator<Item = &str> {
+        self.names.keys().map(|s| s.as_str())
+    }
+}
+
+/// The `Matches` represent matched results from a `Pattern` against a provided text.
+#[derive(Debug)]
+pub struct FancyRegexMatches<'a> {
+    captures: Captures<'a>,
+    pattern: &'a FancyRegexPattern,
+}
+
+impl<'a> FancyRegexMatches<'a> {
+    /// Gets the value for the name (or) alias if found, `None` otherwise.
+    pub fn get(&self, name_or_alias: &str) -> Option<&str> {
+        self.pattern
+            .names
+            .get(name_or_alias)
+            .and_then(|&idx| self.captures.get(idx))
+            .map(|m| m.as_str())
+    }
+
+    /// Returns the number of matches.
+    pub fn len(&self) -> usize {
+        self.pattern.names.len()
+    }
+
+    /// Returns true if there are no matches, false otherwise.
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    /// Returns a tuple of key/value with all the matches found.
+    ///
+    /// Note that if no match is found, the value is empty.
+    pub fn iter(&'a self) -> FancyRegexMatchesIter<'a> {
+        FancyRegexMatchesIter {
+            captures: &self.captures,
+            names: self.pattern.names.iter(),
+        }
+    }
+}
+
+impl<'a> IntoIterator for &'a FancyRegexMatches<'a> {
+    type Item = (&'a str, &'a str);
+    type IntoIter = FancyRegexMatchesIter<'a>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.iter()
+    }
+}
+
+/// An `Iterator` over all matches, accessible via `Matches`.
+pub struct FancyRegexMatchesIter<'a> {
+    captures: &'a Captures<'a>,
+    names: btree_map::Iter<'a, String, usize>,
+}
+
+impl<'a> Iterator for FancyRegexMatchesIter<'a> {
+    type Item = (&'a str, &'a str);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        for (k, &v) in self.names.by_ref() {
+            if let Some(m) = self.captures.get(v) {
+                return Some((k.as_str(), m.as_str()));
+            }
+        }
+        None
+    }
+}