diff --git a/configure.ac b/configure.ac index 28874d9b..66782a92 100644 --- a/configure.ac +++ b/configure.ac @@ -216,6 +216,39 @@ if test x"$lightgrep" == x"yes"; then LDFLAGS="$LDFLAGS `$PKG_CONFIG --libs-only-L --libs-only-other lightgrep`" fi +################################################################ +## yara-x support +## +AC_ARG_ENABLE([yara], + AS_HELP_STRING([--disable-yara], [Disable YARA scanning]), + [], + [AC_DEFINE(USE_YARA, 1, [Use YARA scanning]) yara="yes"]) +if test x"$yara" == x"yes"; then + m4_ifndef([PKG_CHECK_MODULES], + [AC_MSG_ERROR([pkg-config autoconf macros are missing; try installing pkgconfig])]) + + if test x"$mingw" == x"yes" ; then + # get static flags when cross-compiling with mingw + PKG_CONFIG="$PKG_CONFIG --static" + else + # pkg-config doesn't look in /usr/local/lib on some systems + if test x"$PKG_CONFIG_PATH" != x; then + export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/lib/pkgconfig + else + export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig + fi + fi + + m4_ifdef([PKG_CHECK_MODULES], + [PKG_CHECK_MODULES([yara], [yara_x_capi])]) + + AC_DEFINE([HAVE_YARAX], 1, [Define to 1 if you have libyara_x_capi.]) + + CPPFLAGS="$CPPFLAGS $yara_CFLAGS" + LIBS="$LIBS `$PKG_CONFIG --libs-only-l yara_x_capi`" + LDFLAGS="$LDFLAGS `$PKG_CONFIG --libs-only-L --libs-only-other yara_x_capi`" +fi + ################################################################ ## LIBEWF support diff --git a/src/Makefile.am b/src/Makefile.am index 62d85038..55898050 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -103,6 +103,7 @@ scanners_builtin = \ scan_winprefetch.cpp \ scan_wordlist.cpp scan_wordlist.h \ scan_xor.cpp \ + scan_yarax.cpp \ scan_zip.cpp \ pcap_writer.cpp \ pcap_writer.h diff --git a/src/be20_api b/src/be20_api index efa4d698..7fcc332a 160000 --- a/src/be20_api +++ b/src/be20_api @@ -1 +1 @@ -Subproject commit efa4d69821ca2323f5b75de0e5b51b76758d3696 +Subproject commit 7fcc332ab36e5b191cdfbe69dffc4ed1ce8ff6e2 diff --git a/src/bulk_extractor.cpp b/src/bulk_extractor.cpp index 5912cbb9..24183b53 100644 --- a/src/bulk_extractor.cpp +++ b/src/bulk_extractor.cpp @@ -254,6 +254,7 @@ int bulk_extractor_main( std::ostream &cout, std::ostream &cerr, int argc,char * ("V,version", "Display PACKAGE_VERSION (currently) " PACKAGE_VERSION) ("w,stop_list", "file to read stop list from", cxxopts::value()) ("Y,scan", "specify [-end] of area on disk to scan", cxxopts::value()) + ("yara_x_rules", "directory containing yara-x rules", cxxopts::value()) ("z,page_start", "specify a starting page number", cxxopts::value()) ("Z,zap", "wipe the output directory (recursively) before starting") ("0,no_notify", "disable real-time notification") @@ -328,6 +329,10 @@ int bulk_extractor_main( std::ostream &cout, std::ostream &cerr, int argc,char * cfg.opt_recurse = result.count( "recurse" ); + try { + sc.yara_x_rules_path = result["yara_x_rules"].as(); + } catch ( cxxopts::option_has_no_value_exception &e ) { } + try { for ( const auto &it : result["set"].as>() ) { std::vector kv = split( it,'='); diff --git a/src/bulk_extractor_scanners.h b/src/bulk_extractor_scanners.h index 1ac62e1b..d6b1ecd4 100644 --- a/src/bulk_extractor_scanners.h +++ b/src/bulk_extractor_scanners.h @@ -60,6 +60,9 @@ SCANNER(winpe) SCANNER(winprefetch) SCANNER(wordlist) SCANNER(xor) +#ifdef HAVE_YARAX +SCANNER(yarax) +#endif SCANNER(zip) diff --git a/src/scan_yarax.cpp b/src/scan_yarax.cpp new file mode 100644 index 00000000..591cc5ed --- /dev/null +++ b/src/scan_yarax.cpp @@ -0,0 +1,168 @@ +/** + * Plugin: scan_yarax + * Purpose: Run yara-x rules against raw pages + * Reference: https://virustotal.github.io/yara-x/ + **/ + +#include +#include + +#include "config.h" +#include "be20_api/scanner_params.h" + +#ifdef HAVE_YARAX +extern "C" { +#include + +void scan_yarax(scanner_params &sp); +} + +auto YARA_RULE = R"( +rule test_rule { + strings: + $a = "test" + condition: + $a +} +)"; + +std::string readFile(const std::string &filename) { + std::ifstream infile(filename, std::ios::binary | std::ios::in); + + return std::string(std::istreambuf_iterator(infile), std::istreambuf_iterator()); +} + +using rules_ptr = std::unique_ptr; +using compiler_ptr = std::unique_ptr; + +rules_ptr& getYaraRules() { + static rules_ptr rules(nullptr, yrx_rules_destroy); + return rules; +} + +struct YaraCallbackData { + feature_recorder &Recorder; + pos0_t PagePos; + size_t Offset; +}; + +void addYaraFile(const std::string &filename, YRX_COMPILER* compiler) { + std::string rule = readFile(filename); + YRX_RESULT result = yrx_compiler_add_source(compiler, rule.c_str()); + if (result != SUCCESS) { + std::cerr << "Failed to add yara-x file " << filename << ": " << yrx_last_error() << std::endl; + } + else { + // std::cerr << "Added yara-x file " << filename << std::endl; + } +} + +compiler_ptr gatherRules(const std::filesystem::path& rulesPath) { + compiler_ptr compiler(nullptr, yrx_compiler_destroy); + + YRX_COMPILER* compilerRawPtr = nullptr; + YRX_RESULT result = yrx_compiler_create(0, &compilerRawPtr); + if (result != SUCCESS) { + std::cerr << "Failed to create yara-x compiler: " << yrx_last_error() << std::endl; + return compiler; + } + compiler.reset(compilerRawPtr); + + size_t numFiles = 0; + + if (std::filesystem::is_regular_file(rulesPath) && rulesPath.extension() == ".yar") { + addYaraFile(rulesPath, compiler.get()); + ++numFiles; + } + else if (std::filesystem::is_directory(rulesPath)) { + // std::cerr << "Scanning directory " << rulesPath << " for .yar files" << std::endl; + for (const auto& entry : std::filesystem::recursive_directory_iterator(rulesPath)) { + // std::cerr << "Found " << entry.path() << " with extension " << entry.path().extension() << std::endl; + if (entry.is_regular_file() && entry.path().extension() == ".yar") { + addYaraFile(entry.path().string(), compiler.get()); + ++numFiles; + } + } + } + // std::cerr << "Added " << numFiles << " .yar files" << std::endl; + return compiler; +} + +void yara_callback(const YRX_RULE* rule, void* userData) { + const uint8_t* ruleID = nullptr; + size_t idLen = 0; + if (yrx_rule_identifier(rule, &ruleID, &idLen) != SUCCESS) { + std::cerr << "Failed to get rule ID in yara_callback" << std::endl; + return; + } + std::string_view ruleIDView(reinterpret_cast(ruleID), idLen); + const YaraCallbackData* data = static_cast(userData); + data->Recorder.write(data->PagePos.shift(data->Offset), std::string(ruleIDView), ""); +} + +void scan_yarax(scanner_params &sp) { + sp.check_version(); + if (sp.phase == scanner_params::PHASE_INIT){ + sp.info->set_name("yarax"); + sp.info->author = "Jon Stewart"; + sp.info->description = "Scans for yara-x rule matches in raw pages"; + sp.info->scanner_version = "1.0"; + sp.info->feature_defs.push_back(feature_recorder_def("yara-x")); + return; + } + else if (sp.phase == scanner_params::PHASE_INIT2) { + compiler_ptr compiler = gatherRules(std::filesystem::path(sp.sc.yara_x_rules_path)); + YRX_RULES* rules = yrx_compiler_build(compiler.get()); + if (rules == nullptr) { + std::cerr << "Failed to compile yara-x rule: " << yrx_last_error() << std::endl; + return; + } + else { + getYaraRules().reset(rules); + } + } + else if (sp.phase == scanner_params::PHASE_SCAN) { + rules_ptr& rules = getYaraRules(); + if (!rules) { + return; + } + YRX_SCANNER* scannerRawPtr = nullptr; + YRX_RESULT result = yrx_scanner_create(rules.get(), &scannerRawPtr); + if (result != SUCCESS) { + std::cerr << "Failed to create yara-x scanner: " << yrx_last_error() << std::endl; + return; + } + std::unique_ptr scanner(scannerRawPtr, yrx_scanner_destroy); + + YaraCallbackData callbackData{sp.named_feature_recorder("yara-x"), sp.sbuf->pos0, 0}; + + result = yrx_scanner_on_matching_rule(scanner.get(), yara_callback, &callbackData); + if (result != SUCCESS) { + std::cerr << "Failed to set yara-x callback: " << yrx_last_error() << std::endl; + return; + } + + // Iterate the sbuf 4KB at a time and scan these pages individually + const sbuf_t *sbuf = sp.sbuf; + const size_t blockSize = 4096; + const uint8_t* curBlock = sbuf->get_buf(); + const uint8_t* end = curBlock + sbuf->pagesize; + while (curBlock < end) { + const size_t len = std::min(blockSize, static_cast(end - curBlock)); + result = yrx_scanner_scan(scanner.get(), curBlock, len); + if (result != SUCCESS) { + std::cerr << "Failed to scan yara-x: " << yrx_last_error() << std::endl; + return; + } + curBlock += len; + callbackData.Offset += len; + } + result = yrx_scanner_scan(scanner.get(), sbuf->get_buf(), sbuf->bufsize); + if (result != SUCCESS) { + std::cerr << "Failed to scan yara-x on whole sbuf: " << yrx_last_error() << std::endl; + return; + } + } +} +#endif +