From a8e69413c73bec26402cd9f409207e0e99f5f74c Mon Sep 17 00:00:00 2001 From: Jon Stewart Date: Fri, 21 Mar 2025 12:32:33 -0400 Subject: [PATCH 1/6] configure should check for existence of yara_x_capi and link against it if enabled and present --- configure.ac | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/configure.ac b/configure.ac index 28874d9b..686e101f 100644 --- a/configure.ac +++ b/configure.ac @@ -216,6 +216,39 @@ if test x"$lightgrep" == x"yes"; then LDFLAGS="$LDFLAGS `$PKG_CONFIG --libs-only-L --libs-only-other lightgrep`" fi +################################################################ +## yara-x support +## +AC_ARG_ENABLE([yara], + AS_HELP_STRING([--disable-yara], [Disable YARA scanning]), + [], + [AC_DEFINE(USE_YARA, 1, [Use YARA scanning]) yara="yes"]) +if test x"$yara" == x"yes"; then + m4_ifndef([PKG_CHECK_MODULES], + [AC_MSG_ERROR([pkg-config autoconf macros are missing; try installing pkgconfig])]) + + if test x"$mingw" == x"yes" ; then + # get static flags when cross-compiling with mingw + PKG_CONFIG="$PKG_CONFIG --static" + else + # pkg-config doesn't look in /usr/local/lib on some systems + if test x"$PKG_CONFIG_PATH" != x; then + export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/lib/pkgconfig + else + export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig + fi + fi + + m4_ifdef([PKG_CHECK_MODULES], + [PKG_CHECK_MODULES([yara], [yara_x_capi])]) + + AC_DEFINE([HAVE_LIBYARA_X_CAPI], 1, [Define to 1 if you have libyara_x_capi.]) + + CPPFLAGS="$CPPFLAGS $yara_CFLAGS" + LIBS="$LIBS `$PKG_CONFIG --libs-only-l yara_x_capi`" + LDFLAGS="$LDFLAGS `$PKG_CONFIG --libs-only-L --libs-only-other yara_x_capi`" +fi + ################################################################ ## LIBEWF support From f079e26481f25eb115b04532f029f5c420cdd205 Mon Sep 17 00:00:00 2001 From: Jon Stewart Date: Fri, 21 Mar 2025 14:18:32 -0400 Subject: [PATCH 2/6] create a stub scanner for scan_yarax, with the config symbol being HAVE_YARAX --- configure.ac | 2 +- src/Makefile.am | 1 + src/bulk_extractor_scanners.h | 3 +++ src/scan_yarax.cpp | 26 ++++++++++++++++++++++++++ 4 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 src/scan_yarax.cpp diff --git a/configure.ac b/configure.ac index 686e101f..66782a92 100644 --- a/configure.ac +++ b/configure.ac @@ -242,7 +242,7 @@ if test x"$yara" == x"yes"; then m4_ifdef([PKG_CHECK_MODULES], [PKG_CHECK_MODULES([yara], [yara_x_capi])]) - AC_DEFINE([HAVE_LIBYARA_X_CAPI], 1, [Define to 1 if you have libyara_x_capi.]) + AC_DEFINE([HAVE_YARAX], 1, [Define to 1 if you have libyara_x_capi.]) CPPFLAGS="$CPPFLAGS $yara_CFLAGS" LIBS="$LIBS `$PKG_CONFIG --libs-only-l yara_x_capi`" diff --git a/src/Makefile.am b/src/Makefile.am index 62d85038..55898050 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -103,6 +103,7 @@ scanners_builtin = \ scan_winprefetch.cpp \ scan_wordlist.cpp scan_wordlist.h \ scan_xor.cpp \ + scan_yarax.cpp \ scan_zip.cpp \ pcap_writer.cpp \ pcap_writer.h diff --git a/src/bulk_extractor_scanners.h b/src/bulk_extractor_scanners.h index 1ac62e1b..d6b1ecd4 100644 --- a/src/bulk_extractor_scanners.h +++ b/src/bulk_extractor_scanners.h @@ -60,6 +60,9 @@ SCANNER(winpe) SCANNER(winprefetch) SCANNER(wordlist) SCANNER(xor) +#ifdef HAVE_YARAX +SCANNER(yarax) +#endif SCANNER(zip) diff --git a/src/scan_yarax.cpp b/src/scan_yarax.cpp new file mode 100644 index 00000000..5dbcd31b --- /dev/null +++ b/src/scan_yarax.cpp @@ -0,0 +1,26 @@ +/** + * Plugin: scan_yarax + * Purpose: Run yara-x rules against raw pages + * Reference: https://virustotal.github.io/yara-x/ + **/ + +#include + +#include "config.h" +#include "be20_api/scanner_params.h" + +#ifdef HAVE_YARAX +extern "C" void scan_yarax(scanner_params &sp); + +void scan_yarax(scanner_params &sp) { + sp.check_version(); + if (sp.phase == scanner_params::PHASE_INIT){ + sp.info->set_name("yarax"); + sp.info->author = "Jon Stewart"; + sp.info->description = "Scans for yara-x rule matches in raw pages"; + sp.info->scanner_version = "1.0"; + return; + } +} +#endif + From af88d0e71ed945f98e4a8594a6a8dc6af9c2c490 Mon Sep 17 00:00:00 2001 From: Jon Stewart Date: Sat, 22 Mar 2025 10:28:31 -0400 Subject: [PATCH 3/6] basic scanning of sbufs with yara_x implemented. This uses a hard-coded rule that is unlikely to generate matches and it also does not use feature-recorders to record any matches. But it does get called and pass tests with the address_sanitizer. I will need to think of some ways to unit test this. --- src/scan_yarax.cpp | 78 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/src/scan_yarax.cpp b/src/scan_yarax.cpp index 5dbcd31b..71867314 100644 --- a/src/scan_yarax.cpp +++ b/src/scan_yarax.cpp @@ -10,7 +10,38 @@ #include "be20_api/scanner_params.h" #ifdef HAVE_YARAX -extern "C" void scan_yarax(scanner_params &sp); +extern "C" { +#include + +void scan_yarax(scanner_params &sp); +} + +auto YARA_RULE = R"( +rule test_rule { + strings: + $a = "asdfoiqweofiuxkbjvlkjdlksdjflaisjflkvlkfvnbzkdfjasoei;lzkxcvnkaslfkjqlkj" + condition: + $a +} +)"; + +using rules_ptr = std::unique_ptr; + +rules_ptr& getYaraRules() { + static rules_ptr rules(nullptr, yrx_rules_destroy); + return rules; +} + +void yara_callback(const YRX_RULE* rule, void*) { + const uint8_t* ruleID = nullptr; + size_t idLen = 0; + if (yrx_rule_identifier(rule, &ruleID, &idLen) != SUCCESS) { + std::cerr << "Failed to get rule ID in yara_callback" << std::endl; + return; + } + std::string_view ruleIDView(reinterpret_cast(ruleID), idLen); + std::cout << "Matched rule: " << ruleIDView << std::endl; +} void scan_yarax(scanner_params &sp) { sp.check_version(); @@ -21,6 +52,51 @@ void scan_yarax(scanner_params &sp) { sp.info->scanner_version = "1.0"; return; } + else if (sp.phase == scanner_params::PHASE_INIT2) { + YRX_RULES* rules = nullptr; + YRX_RESULT result = yrx_compile(YARA_RULE, &rules); + if (result != SUCCESS) { + std::cerr << "Failed to compile yara-x rule: " << yrx_last_error() << std::endl; + return; + } + else { + getYaraRules().reset(rules); + } + } + else if (sp.phase == scanner_params::PHASE_SCAN) { + rules_ptr& rules = getYaraRules(); + if (!rules) { + return; + } + YRX_SCANNER* scannerRawPtr = nullptr; + YRX_RESULT result = yrx_scanner_create(rules.get(), &scannerRawPtr); + if (result != SUCCESS) { + std::cerr << "Failed to create yara-x scanner: " << yrx_last_error() << std::endl; + return; + } + std::unique_ptr scanner(scannerRawPtr, yrx_scanner_destroy); + + result = yrx_scanner_on_matching_rule(scanner.get(), yara_callback, nullptr); + if (result != SUCCESS) { + std::cerr << "Failed to set yara-x callback: " << yrx_last_error() << std::endl; + return; + } + + // Iterate the sbuf 4KB at a time and scan these pages individually + const sbuf_t *sbuf = sp.sbuf; + const size_t blockSize = 4096; + const uint8_t* curBlock = sbuf->get_buf(); + const uint8_t* end = curBlock + sbuf->pagesize; + while (curBlock < end) { + const size_t len = std::min(blockSize, static_cast(end - curBlock)); + result = yrx_scanner_scan(scanner.get(), curBlock, len); + if (result != SUCCESS) { + std::cerr << "Failed to scan yara-x: " << yrx_last_error() << std::endl; + return; + } + curBlock += len; + } + } } #endif From bbd5cdf1578b01aaaeca371d8edc15bb1a86e645 Mon Sep 17 00:00:00 2001 From: Jon Stewart Date: Sat, 22 Mar 2025 16:58:03 -0400 Subject: [PATCH 4/6] Create a yara-x feature recorder and write the names of matching rules with the block/forensic path of the data block that triggered the rule. --- src/scan_yarax.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/scan_yarax.cpp b/src/scan_yarax.cpp index 71867314..862ce56d 100644 --- a/src/scan_yarax.cpp +++ b/src/scan_yarax.cpp @@ -19,7 +19,7 @@ void scan_yarax(scanner_params &sp); auto YARA_RULE = R"( rule test_rule { strings: - $a = "asdfoiqweofiuxkbjvlkjdlksdjflaisjflkvlkfvnbzkdfjasoei;lzkxcvnkaslfkjqlkj" + $a = "test" condition: $a } @@ -32,7 +32,12 @@ rules_ptr& getYaraRules() { return rules; } -void yara_callback(const YRX_RULE* rule, void*) { +struct YaraCallbackData { + feature_recorder &Recorder; + pos0_t Pos; +}; + +void yara_callback(const YRX_RULE* rule, void* userData) { const uint8_t* ruleID = nullptr; size_t idLen = 0; if (yrx_rule_identifier(rule, &ruleID, &idLen) != SUCCESS) { @@ -40,7 +45,8 @@ void yara_callback(const YRX_RULE* rule, void*) { return; } std::string_view ruleIDView(reinterpret_cast(ruleID), idLen); - std::cout << "Matched rule: " << ruleIDView << std::endl; + const YaraCallbackData* data = static_cast(userData); + data->Recorder.write(data->Pos, std::string(ruleIDView), ""); } void scan_yarax(scanner_params &sp) { @@ -50,6 +56,7 @@ void scan_yarax(scanner_params &sp) { sp.info->author = "Jon Stewart"; sp.info->description = "Scans for yara-x rule matches in raw pages"; sp.info->scanner_version = "1.0"; + sp.info->feature_defs.push_back(feature_recorder_def("yara-x")); return; } else if (sp.phase == scanner_params::PHASE_INIT2) { @@ -76,7 +83,9 @@ void scan_yarax(scanner_params &sp) { } std::unique_ptr scanner(scannerRawPtr, yrx_scanner_destroy); - result = yrx_scanner_on_matching_rule(scanner.get(), yara_callback, nullptr); + YaraCallbackData callbackData{sp.named_feature_recorder("yara-x"), sp.sbuf->pos0}; + + result = yrx_scanner_on_matching_rule(scanner.get(), yara_callback, &callbackData); if (result != SUCCESS) { std::cerr << "Failed to set yara-x callback: " << yrx_last_error() << std::endl; return; From 7f33588c7ce6b3dd9cf7cc89375f0d084e7c6e0e Mon Sep 17 00:00:00 2001 From: Jon Stewart Date: Sun, 23 Mar 2025 10:35:54 -0400 Subject: [PATCH 5/6] Report on the block size offset (4KB-aligned) of a rule match, not just the much larger bulk_extractor page offset (typically 16MB). --- src/scan_yarax.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/scan_yarax.cpp b/src/scan_yarax.cpp index 862ce56d..0168316e 100644 --- a/src/scan_yarax.cpp +++ b/src/scan_yarax.cpp @@ -34,7 +34,8 @@ rules_ptr& getYaraRules() { struct YaraCallbackData { feature_recorder &Recorder; - pos0_t Pos; + pos0_t PagePos; + size_t Offset; }; void yara_callback(const YRX_RULE* rule, void* userData) { @@ -46,7 +47,7 @@ void yara_callback(const YRX_RULE* rule, void* userData) { } std::string_view ruleIDView(reinterpret_cast(ruleID), idLen); const YaraCallbackData* data = static_cast(userData); - data->Recorder.write(data->Pos, std::string(ruleIDView), ""); + data->Recorder.write(data->PagePos.shift(data->Offset), std::string(ruleIDView), ""); } void scan_yarax(scanner_params &sp) { @@ -83,7 +84,7 @@ void scan_yarax(scanner_params &sp) { } std::unique_ptr scanner(scannerRawPtr, yrx_scanner_destroy); - YaraCallbackData callbackData{sp.named_feature_recorder("yara-x"), sp.sbuf->pos0}; + YaraCallbackData callbackData{sp.named_feature_recorder("yara-x"), sp.sbuf->pos0, 0}; result = yrx_scanner_on_matching_rule(scanner.get(), yara_callback, &callbackData); if (result != SUCCESS) { @@ -104,6 +105,7 @@ void scan_yarax(scanner_params &sp) { return; } curBlock += len; + callbackData.Offset += len; } } } From 6293ef438fc57cb0a3cb74161b6d8ef42753a7e2 Mon Sep 17 00:00:00 2001 From: Jon Stewart Date: Tue, 25 Mar 2025 21:55:17 -0400 Subject: [PATCH 6/6] Added a new --yara_x_rules CLI option for specifying a directory of .yar files containing rules to be loaded. Uses these for scanning 4KB chunks at a time. Added a semi-duplicative second scanning pass over the sbuf in one go. Will need to check out various rule bases and see what makes the most sense (and an option for control). But I was able to get a number of hits on known malware --- src/be20_api | 2 +- src/bulk_extractor.cpp | 5 ++++ src/scan_yarax.cpp | 61 +++++++++++++++++++++++++++++++++++++++--- 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/src/be20_api b/src/be20_api index efa4d698..7fcc332a 160000 --- a/src/be20_api +++ b/src/be20_api @@ -1 +1 @@ -Subproject commit efa4d69821ca2323f5b75de0e5b51b76758d3696 +Subproject commit 7fcc332ab36e5b191cdfbe69dffc4ed1ce8ff6e2 diff --git a/src/bulk_extractor.cpp b/src/bulk_extractor.cpp index 5912cbb9..24183b53 100644 --- a/src/bulk_extractor.cpp +++ b/src/bulk_extractor.cpp @@ -254,6 +254,7 @@ int bulk_extractor_main( std::ostream &cout, std::ostream &cerr, int argc,char * ("V,version", "Display PACKAGE_VERSION (currently) " PACKAGE_VERSION) ("w,stop_list", "file to read stop list from", cxxopts::value()) ("Y,scan", "specify [-end] of area on disk to scan", cxxopts::value()) + ("yara_x_rules", "directory containing yara-x rules", cxxopts::value()) ("z,page_start", "specify a starting page number", cxxopts::value()) ("Z,zap", "wipe the output directory (recursively) before starting") ("0,no_notify", "disable real-time notification") @@ -328,6 +329,10 @@ int bulk_extractor_main( std::ostream &cout, std::ostream &cerr, int argc,char * cfg.opt_recurse = result.count( "recurse" ); + try { + sc.yara_x_rules_path = result["yara_x_rules"].as(); + } catch ( cxxopts::option_has_no_value_exception &e ) { } + try { for ( const auto &it : result["set"].as>() ) { std::vector kv = split( it,'='); diff --git a/src/scan_yarax.cpp b/src/scan_yarax.cpp index 0168316e..591cc5ed 100644 --- a/src/scan_yarax.cpp +++ b/src/scan_yarax.cpp @@ -5,6 +5,7 @@ **/ #include +#include #include "config.h" #include "be20_api/scanner_params.h" @@ -25,7 +26,14 @@ rule test_rule { } )"; +std::string readFile(const std::string &filename) { + std::ifstream infile(filename, std::ios::binary | std::ios::in); + + return std::string(std::istreambuf_iterator(infile), std::istreambuf_iterator()); +} + using rules_ptr = std::unique_ptr; +using compiler_ptr = std::unique_ptr; rules_ptr& getYaraRules() { static rules_ptr rules(nullptr, yrx_rules_destroy); @@ -38,6 +46,48 @@ struct YaraCallbackData { size_t Offset; }; +void addYaraFile(const std::string &filename, YRX_COMPILER* compiler) { + std::string rule = readFile(filename); + YRX_RESULT result = yrx_compiler_add_source(compiler, rule.c_str()); + if (result != SUCCESS) { + std::cerr << "Failed to add yara-x file " << filename << ": " << yrx_last_error() << std::endl; + } + else { + // std::cerr << "Added yara-x file " << filename << std::endl; + } +} + +compiler_ptr gatherRules(const std::filesystem::path& rulesPath) { + compiler_ptr compiler(nullptr, yrx_compiler_destroy); + + YRX_COMPILER* compilerRawPtr = nullptr; + YRX_RESULT result = yrx_compiler_create(0, &compilerRawPtr); + if (result != SUCCESS) { + std::cerr << "Failed to create yara-x compiler: " << yrx_last_error() << std::endl; + return compiler; + } + compiler.reset(compilerRawPtr); + + size_t numFiles = 0; + + if (std::filesystem::is_regular_file(rulesPath) && rulesPath.extension() == ".yar") { + addYaraFile(rulesPath, compiler.get()); + ++numFiles; + } + else if (std::filesystem::is_directory(rulesPath)) { + // std::cerr << "Scanning directory " << rulesPath << " for .yar files" << std::endl; + for (const auto& entry : std::filesystem::recursive_directory_iterator(rulesPath)) { + // std::cerr << "Found " << entry.path() << " with extension " << entry.path().extension() << std::endl; + if (entry.is_regular_file() && entry.path().extension() == ".yar") { + addYaraFile(entry.path().string(), compiler.get()); + ++numFiles; + } + } + } + // std::cerr << "Added " << numFiles << " .yar files" << std::endl; + return compiler; +} + void yara_callback(const YRX_RULE* rule, void* userData) { const uint8_t* ruleID = nullptr; size_t idLen = 0; @@ -61,9 +111,9 @@ void scan_yarax(scanner_params &sp) { return; } else if (sp.phase == scanner_params::PHASE_INIT2) { - YRX_RULES* rules = nullptr; - YRX_RESULT result = yrx_compile(YARA_RULE, &rules); - if (result != SUCCESS) { + compiler_ptr compiler = gatherRules(std::filesystem::path(sp.sc.yara_x_rules_path)); + YRX_RULES* rules = yrx_compiler_build(compiler.get()); + if (rules == nullptr) { std::cerr << "Failed to compile yara-x rule: " << yrx_last_error() << std::endl; return; } @@ -107,6 +157,11 @@ void scan_yarax(scanner_params &sp) { curBlock += len; callbackData.Offset += len; } + result = yrx_scanner_scan(scanner.get(), sbuf->get_buf(), sbuf->bufsize); + if (result != SUCCESS) { + std::cerr << "Failed to scan yara-x on whole sbuf: " << yrx_last_error() << std::endl; + return; + } } } #endif