xor-zero number(0)

Author: marirs

CHANGELOG.md

@@ -3,6 +3,34 @@
 All notable changes to **capa** are documented here.
 This project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.5.2] — xor-zero number(0), regex /i fast path, rule pre-pruning
+
+### Fixed — feature extraction parity
+
+- **`xor reg, reg` now emits `Number(0)`**
+  Self-XOR is the canonical register-zeroing idiom; previously
+  `extract_insn_nzxor_characteristic_features` returned an empty feature set
+  for it, so rules matching on `number: 0` at the produced register missed the
+  pattern entirely.
+
+### Performance
+
+- **Pre-prune rules with unsatisfiable global-feature constraints**
+  New `RuleSet::filter_rules_by_meta_features` walks
+  each rule's statement AST and discards rules whose `os`/`arch`/`format`
+  constraints can't be met by the binary's globals, before the per-function
+  evaluation loop. Transitive dependencies of kept rules are preserved.
+- **Regex `/i` fast path via lowercased-string lookup**
+  Pure-literal patterns ending in `/i` now do an O(1) `HashSet` lookup
+  against the lowercased feature values instead of compiling and running a
+  regex. Builds on the 0.4.2 lowercase canonicalization.
+- **String pre-filter at file load** New
+  `RuleSet::prepare_for_file` populates an `impossible_string_rule_names` set
+  by checking each rule's required strings against the file's string blob.
+  Impossible rules are skipped during per-function matching unless their scope
+  contains `Characteristic("stack string")` (since stack-built strings aren't
+  in the file's string set).
+
 ## [0.5.1] — Mach-O closeout: zero `Unknown` rows, stub-VA API resolution, iOS distinction
 
 Closes every `Unknown` placeholder in the 0.5.0 Mach-O security

Cargo.toml

@@ -1,32 +1,15 @@
 [package]
 name = "capa"
-version = "0.5.1"
+version = "0.5.2"
 description = "File capability extractor."
 authors = ["Marirs <marirs@gmail.com>", "Andrey Mnatsakanov <andrey@3ig.kiev.ua>", "Jorge Alejandro Durán Royo<jorge.duran.royo@gmail.com>"]
 keywords = ["capa", "fce", "capability", "aslr", "reverse"]
 readme = "README.md"
 license-file = "LICENSE"
 repository = "https://github.com/marirs/capa-rs"
 homepage = "https://github.com/marirs/capa-rs"
-# Drop `examples` from `exclude` so the [[example]] target below ships
-# with the crate (was inconsistent: example target referenced an
-# excluded path). `data/` stays excluded — those are test fixtures.
-# `flirt-sigs/` stays excluded — 70 MB of FLIRT .sig files would
-# blow past crates.io's 10 MB upload limit. The GitHub release ships
-# them as a separate `flirt-sigs.tar.gz` artifact alongside the CLI
-# binaries; source builds (`git clone`) get them automatically since
-# the directory lives in the repo.
-# `scripts/` (0.5.0) stays excluded — the only file is
-# `gen_com_tables.py`, the Python generator that produced
-# `src/rules/com_db.rs` (the 29k-entry COM GUID database, included!
-# at build time). The generated table ships in src/; the generator
-# itself is only useful for regenerating from a newer upstream
-# Python capa, which crates.io consumers don't need.
 exclude = ["data", "flirt-sigs", "scripts"]
 edition = "2024"
-# 0.3.21: bumped from 1.85 → 1.95 because smda 0.5.x sets MSRV 1.95
-# (needed for some 2024-edition features it leans on). Pure-Rust dep
-# tree below — no aws-lc / openssl / native-tls.
 rust-version = "1.95"
 
 [dependencies]
@@ -37,26 +20,12 @@ petgraph = "0.8"
 regex = "1.11.1"
 fancy-regex = "0.18"
 serde = { version = "1", features = ["derive"] }
-# 0.5.1: smda 0.6.4 → 0.6.5. Picks up the Mach-O closeout:
-# `__TEXT,__stubs` walker via `LC_DYSYMTAB.indirectsymoff` so
-# direct `bl _stub` calls resolve to their API names (0.6.4 only
-# covered the GOT-slot indirect form, missing the most common
-# ARM64 PIC call shape), and `MachoArchPreference` plumbed through
-# `BinaryInfo` so post-parse passes honour the slice preference
-# set at parse time. `0.6.5` is the minimum; semver-compatible
-# with future 0.6.x.
 smda = "0.6.5"
 thiserror = "2.0.12"
 walkdir = "2.5.0"
 yaml-rust = "0.4.5"
 goblin = { version = "0.10.0", features = ["alloc"] }
 maplit = "1.0.2"
-# 0.4.2: dnfile 0.4 → 0.5 picks up `Send + Sync` supertrait bounds on
-# the public trait family (MDTableTrait, MDTableRowTrait,
-# MDTableRowTraitT, CodedIndex) so `DnPe<'_>` — and the capa-rs
-# `dnfile::Extractor<'_>` that wraps it — can cross thread
-# boundaries. Required by the rayon-parallel function loop in
-# `find_capabilities`. No runtime behaviour change.
 dnfile = "0.5"
 lazy_static = "1.5.0"
 parking_lot = "0.12.3"
@@ -66,49 +35,12 @@ memmap2 = "0.9.5"
 scroll = "0.13"
 once_cell = "1.21.1"
 dynamic-loader-cache = "0.2.2"
-# (0.5.1) Pure-Rust PLIST parser for Mach-O code-signing
-# entitlements blob (CS_EmbeddedEntitlements, magic 0xfade7171).
-# Used by security/macho.rs to detect ALLOW-JIT
-# (com.apple.security.cs.allow-jit). No openssl / aws-lc /
-# native-tls in the transitive tree — matches the project's
-# pure-Rust stance.
 plist = { version = "1", default-features = false }
-# 0.3.21 introduced ouroboros as a self-referential wrapper around
-# `buf: Vec<u8>` + `DisassemblyReport<'_>` so the public Extractor::new
-# could keep its 0.3.20 signature. 0.4.0 completes the zero-copy
-# refactor — `Extractor<'a>` takes `&'a [u8]` from the caller and
-# borrows the smda/dnfile reports directly. The ouroboros dep is gone.
-# 0.3.21: iced-x86 is already in the transitive dep tree via smda.
-# Pulling it in directly lets the smda extractor compare instructions
-# against typed `Mnemonic` / `FlowControl` enums instead of re-formatting
-# strings — the same hot-path perf win smda itself got from the iced
-# decoder swap. Match the feature set smda uses to keep monomorphisation
-# overhead identical.
 iced-x86 = { version = "1", default-features = false, features = ["std", "decoder", "intel", "instr_info"] }
-# 0.4.3: fast-flirt is our pure-Rust FLIRT engine — Apache-2.0,
-# zero-copy arena + multi-level prefix trie matcher. capa-rs uses it
-# to identify statically-linked library functions (MSVC CRT, ATL/MFC,
-# OpenSSL, zlib, boost, libcurl, lua, protobuf, DirectX, Intel libs,
-# etc.) and exclude them from the user-facing capability output via
-# the existing `lib: true` rule-skip path. Always compiled in —
-# `AnalyzeBuilder::signatures` is the opt-in (no signatures path →
-# no FLIRT, identical behaviour to pre-0.4.3).
-#
-# Three direct deps (`miniz_oxide`, `smallvec`, `thiserror`); ~240×
-# faster on `matches()` and ~70% less resident memory than the
-# upstream-FLIRT engine we briefly piloted on.
-fast-flirt = "0.2.2"
-# 0.4.2: rayon for inter-function parallelism in `find_capabilities`.
-# Each function's analysis is pure — reads the extractor, evaluates
-# rules, returns matches. Parallelising the outer loop gives ~4-8×
-# end-to-end speedup on multi-core machines for binaries with more
-# than a handful of functions. Also used in `rules::get_rules` to
-# parallelise YAML parsing across the rule corpus.
 rayon = "1.10"
 
 [dev-dependencies]
 clap = { version = "4.5", features = ["cargo", "derive"] }
-
 prettytable-rs = "0.10.0"
 
 [lib]

src/extractor/smda.rs

@@ -1005,6 +1005,17 @@ impl<'data> Extractor<'data> {
             let rn = (a.opcode >> 5) & 0x1f;
             let rm = (a.opcode >> 16) & 0x1f;
             if rn == rm {
+                // 0.5.2 (upstream parity #2997): `eor xd, xn, xn` is the
+                // AArch64 zeroing idiom — same shape as x86 `xor eax, eax`.
+                // Emit Number(0) at the instruction so rules matching on
+                // `number: 0` see the produced value, rather than dropping
+                // the case silently (pre-0.5.2 behaviour).
+                res.push((
+                    crate::rules::features::Feature::Number(
+                        crate::rules::features::NumberFeature::new(f.bitness, &0_i128, "")?,
+                    ),
+                    insn.offset,
+                ));
                 return Ok(res);
             }
             // Security-cookie filter is x86-specific (stack-canary
@@ -1031,6 +1042,16 @@ impl<'data> Extractor<'data> {
         if let Some(o) = insn.format_operands() {
             let operands: Vec<String> = o.split(',').map(|s| s.trim().to_string()).collect();
             if operands[0] == operands[1] {
+                // 0.5.2 (upstream parity #2997): `xor eax, eax` (and the SSE
+                // / packed variants Xorpd/Xorps/Pxor) zero the destination
+                // register. Emit Number(0) for rules matching the produced
+                // value, instead of dropping the case silently.
+                res.push((
+                    crate::rules::features::Feature::Number(
+                        crate::rules::features::NumberFeature::new(f.bitness, &0_i128, "")?,
+                    ),
+                    insn.offset,
+                ));
                 return Ok(res);
             }
         }
@@ -1928,3 +1949,100 @@ pub(crate) fn classify_macho_os(buf: &[u8]) -> Result<Os> {
         _ => Ok(Os::MACOS),
     }
 }
+
+// 0.5.2 (upstream parity #2997): integration test for the
+// `xor reg, reg` → `Number(0)` correctness fix. This is the first
+// extractor-level test in the crate — placed inline rather than in
+// `tests/` so we keep crate-internal access to the `Mnemonic` import,
+// the `SmdaExtractor::report()` accessor, and the public
+// `extract_insn_nzxor_characteristic_features` method without
+// having to widen any visibility.
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::rules::features::{CharacteristicFeature, Feature, NumberFeature};
+
+    /// Walks every instruction in `data/Demo64.dll`, finds any
+    /// `xor reg, reg` (or `xorpd`/`xorps`/`pxor`) where both operands
+    /// are the same register, and asserts that
+    /// `extract_insn_nzxor_characteristic_features` emits `Number(0)`
+    /// at the instruction and does NOT emit `Characteristic("nzxor")`
+    /// — mirroring the upstream Python parity-test pattern from
+    /// `mandiant/capa#2997`.
+    ///
+    /// Fails loudly if no self-XOR site is discovered, so we can't pass
+    /// vacuously on a binary that happens to contain none.
+    #[test]
+    fn upstream_parity_2997_xor_self_emits_number_zero() {
+        let path = "data/Demo64.dll";
+        let bytes = std::fs::read(path).unwrap_or_else(|e| {
+            panic!("test fixture missing: {path}: {e}");
+        });
+        let extractor = Extractor::new(path, false, false, &bytes).expect("smda parse Demo64.dll");
+
+        let report = extractor.report();
+        let functions = report.get_functions().expect("smda get_functions");
+
+        // PartialEq on NumberFeature / CharacteristicFeature is value-only
+        // (see `impl PartialEq for NumberFeature` in src/rules/features.rs),
+        // so the bitness / description we construct here are irrelevant for
+        // the assertion — only the value matters.
+        let want_zero =
+            Feature::Number(NumberFeature::new(64, &0_i128, "").expect("NumberFeature::new"));
+        let want_nzxor = Feature::Characteristic(
+            CharacteristicFeature::new("nzxor", "").expect("CharacteristicFeature::new"),
+        );
+
+        let mut sites_checked = 0_usize;
+        for smda_func in functions.values() {
+            let Ok(blocks) = smda_func.get_blocks() else {
+                continue;
+            };
+            for instrs in blocks.values() {
+                for insn in instrs {
+                    // Mirror the extractor's own self-XOR detection so the
+                    // test exercises the exact branch the fix added a
+                    // `Number(0)` push to.
+                    if !matches!(
+                        insn.mnemonic_enum(),
+                        Mnemonic::Xor | Mnemonic::Xorpd | Mnemonic::Xorps | Mnemonic::Pxor
+                    ) {
+                        continue;
+                    }
+                    let Some(operand_str) = insn.format_operands() else {
+                        continue;
+                    };
+                    let parts: Vec<&str> = operand_str.split(',').map(|s| s.trim()).collect();
+                    if parts.len() < 2 || parts[0] != parts[1] {
+                        continue;
+                    }
+
+                    sites_checked += 1;
+                    let res = extractor
+                        .extract_insn_nzxor_characteristic_features(smda_func, insn)
+                        .expect("extract_insn_nzxor_characteristic_features");
+
+                    assert!(
+                        res.iter().any(|(f, _)| f == &want_zero),
+                        "self-XOR at {:#x} did not emit Number(0); features were {:?}",
+                        insn.offset,
+                        res.iter().map(|(f, _)| f).collect::<Vec<_>>(),
+                    );
+                    assert!(
+                        res.iter().all(|(f, _)| f != &want_nzxor),
+                        "self-XOR at {:#x} incorrectly tagged as nzxor",
+                        insn.offset,
+                    );
+                }
+            }
+        }
+
+        assert!(
+            sites_checked > 0,
+            "no `xor reg, reg` site found in {path} — \
+             the test cannot verify the fix; switch the fixture to a \
+             binary that contains at least one self-XOR.",
+        );
+        eprintln!("upstream parity #2997: verified {sites_checked} self-XOR site(s) in {path}");
+    }
+}

src/extractor/smda.rs:1982:13

@@ -1126,6 +1126,23 @@ fn find_capabilities(
 )> {
     use rayon::prelude::*;
 
+    // 0.5.2 (upstream parity mandiant/capa#2929): pre-prune rules whose
+    // global-feature constraints (os/arch/format) can't be satisfied by
+    // this binary, *before* the per-function matching loop. Eliminates
+    // those rules from every per-function / per-basic-block /
+    // per-instruction evaluation downstream.
+    //
+    // Globals are constant per binary, so this is a one-shot up-front
+    // call that pays off across thousands of per-function evaluations.
+    // The owned filtered `RuleSet` is held in a local so all
+    // `&rules::Rule` references taken later point into it.
+    let mut globals_map: HashMap<crate::rules::features::Feature, Vec<u64>> = HashMap::new();
+    for (feat, va) in extractor.extract_global_features()? {
+        globals_map.entry(feat).or_default().push(va);
+    }
+    let pruned_ruleset = ruleset.filter_rules_by_meta_features(&globals_map)?;
+    let ruleset: &rules::RuleSet = &pruned_ruleset;
+
     let mut all_function_matches: HashMap<&rules::Rule, Vec<(u64, (bool, Vec<u64>))>> =
         HashMap::new();
     let mut all_bb_matches: HashMap<&rules::Rule, Vec<(u64, (bool, Vec<u64>))>> = HashMap::new();

src/lib.rs

@@ -109,6 +109,17 @@ pub enum Feature {
 }
 
 impl Feature {
+    /// 0.5.2 (upstream parity #2929): is this a "global" feature —
+    /// one that's constant per binary (OS, architecture, file
+    /// format), determined once from headers, and therefore
+    /// suitable for pre-pruning rules whose constraints can't be
+    /// satisfied?
+    ///
+    /// Mirrors Python capa's `capa.features.common.is_global_feature`.
+    pub fn is_global_feature(&self) -> bool {
+        matches!(self, Feature::Os(_) | Feature::Arch(_) | Feature::Format(_))
+    }
+
     pub fn new(t: RuleFeatureType, value: &Value, description: &str) -> Result<Feature> {
         // let readpro = "property/read".to_string();
 
@@ -1530,6 +1541,11 @@ impl ArchFeature {
         }
         Ok((false, vec![]))
     }
+
+    /// 0.5.2 (upstream parity #2929): see `OsFeature::value`.
+    pub fn value(&self) -> &str {
+        &self.value
+    }
 }
 
 impl Hash for ArchFeature {
@@ -1689,6 +1705,13 @@ impl OsFeature {
         }
         Ok((false, vec![]))
     }
+
+    /// 0.5.2 (upstream parity #2929): expose the canonicalised
+    /// (lowercased) OS string so the `filter_rules_by_meta_features`
+    /// pre-prune walker can detect the `os: any` wildcard.
+    pub fn value(&self) -> &str {
+        &self.value
+    }
 }
 
 impl Hash for OsFeature {
@@ -1744,6 +1767,11 @@ impl FormatFeature {
         }
         Ok((false, vec![]))
     }
+
+    /// 0.5.2 (upstream parity #2929): see `OsFeature::value`.
+    pub fn value(&self) -> &str {
+        &self.value
+    }
 }
 
 impl Hash for FormatFeature {