From f010bfeffc4f7795c353cf2b6f5d03abfb9ee546 Mon Sep 17 00:00:00 2001 From: Parth Malhotra <28601533+parthmalhotra@users.noreply.github.com> Date: Wed, 30 Oct 2024 16:29:21 +0530 Subject: [PATCH 1/2] Added Awesome Search Queries intigration (-cpe) and Passive Wordpress plugin and theme detection (-wp, -wordpress) flag --- common/httpx/option.go | 2 + go.mod | 5 +- go.sum | 2 + runner/awesome_queries.go | 141 ++++++++++++++++++++++++++++++++++++++ runner/options.go | 15 +++- runner/runner.go | 95 +++++++++++++++++++++++++ runner/types.go | 4 ++ runner/wordpress.go | 121 ++++++++++++++++++++++++++++++++ 8 files changed, 381 insertions(+), 4 deletions(-) create mode 100644 runner/awesome_queries.go create mode 100644 runner/wordpress.go diff --git a/common/httpx/option.go b/common/httpx/option.go index 6ef619193..e8939567b 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -50,6 +50,7 @@ type Options struct { CDNCheckClient *cdncheck.Client Protocol Proto Trace bool + AwesomeSearchQueries bool } // DefaultOptions contains the default options @@ -71,6 +72,7 @@ var DefaultOptions = Options{ VHostStripHTML: false, VHostSimilarityRatio: 85, DefaultUserAgent: "httpx - Open-source project (github.com/projectdiscovery/httpx)", + AwesomeSearchQueries: false, } func (options *Options) parseCustomCookies() { diff --git a/go.mod b/go.mod index a0df1c2f2..b2b8cd249 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/projectdiscovery/httpx -go 1.21 +go 1.21.3 + +toolchain go1.23.2 require ( github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 @@ -52,6 +54,7 @@ require ( require ( github.com/go-viper/mapstructure/v2 v2.1.0 + github.com/projectdiscovery/awesome-search-queries v0.0.0-20241030094221-9fa3c0933578 github.com/weppos/publicsuffix-go v0.30.2 ) diff --git a/go.sum b/go.sum index 937b2a506..c167854fa 100644 --- a/go.sum +++ b/go.sum @@ -226,6 +226,8 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kID2iwsDqI= github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60= +github.com/projectdiscovery/awesome-search-queries v0.0.0-20241030094221-9fa3c0933578 h1:PSpd8NNjmDUgfbhgB/49HZqXQ+7DJRnMR1TVyEus7PM= +github.com/projectdiscovery/awesome-search-queries v0.0.0-20241030094221-9fa3c0933578/go.mod h1:nSovPcipgSx/EzAefF+iCfORolkKAuodiRWL3RCGHOM= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/snk0cQ3kXE= diff --git a/runner/awesome_queries.go b/runner/awesome_queries.go new file mode 100644 index 000000000..9102fc141 --- /dev/null +++ b/runner/awesome_queries.go @@ -0,0 +1,141 @@ +package runner + +import ( + "encoding/json" + "strings" + + awesomesearchqueries "github.com/projectdiscovery/awesome-search-queries" +) + +type AwesomeQuery struct { + Name string `json:"name"` + Vendor string `json:"vendor"` + Type string `json:"type"` + Engines []Engines `json:"engines"` +} + +type Engines struct { + Platform string `json:"platform"` + Queries []string `json:"queries"` +} + +type AwesomeSearchMaps struct { + aqTitle map[string][]ProductVendor + aqBody map[string][]ProductVendor + aqFavicon map[string][]ProductVendor +} + +type ProductVendor struct { + Product string + Vendor string +} + +func LoadAwesomeQueries() (*AwesomeSearchMaps, error) { + data, err := awesomesearchqueries.GetQueries() + if err != nil { + return nil, err + } + + var queries []AwesomeQuery + if err := json.Unmarshal(data, &queries); err != nil { + return nil, err + } + + maps := &AwesomeSearchMaps{ + aqTitle: make(map[string][]ProductVendor), + aqBody: make(map[string][]ProductVendor), + aqFavicon: make(map[string][]ProductVendor), + } + + for _, query := range queries { + pv := ProductVendor{ + Product: query.Name, + Vendor: query.Vendor, + } + + for _, engine := range query.Engines { + for _, q := range engine.Queries { + switch engine.Platform { + case "shodan": + if strings.HasPrefix(q, "http.html:") { + maps.aqBody[extractQuery(q, "http.html:")] = append(maps.aqBody[extractQuery(q, "http.html:")], pv) + } else if strings.HasPrefix(q, "http.title:") { + maps.aqTitle[extractQuery(q, "http.title:")] = append(maps.aqTitle[extractQuery(q, "http.title:")], pv) + } else if strings.HasPrefix(q, "http.favicon.hash:") { + maps.aqFavicon[extractQuery(q, "http.favicon.hash:")] = append(maps.aqFavicon[extractQuery(q, "http.favicon.hash:")], pv) + } + case "fofa": + if strings.HasPrefix(q, "body=") { + maps.aqBody[extractQuery(q, "body=")] = append(maps.aqBody[extractQuery(q, "body=")], pv) + } else if strings.HasPrefix(q, "title=") { + maps.aqTitle[extractQuery(q, "title=")] = append(maps.aqTitle[extractQuery(q, "title=")], pv) + } else if strings.HasPrefix(q, "icon_hash=") { + maps.aqFavicon[extractQuery(q, "icon_hash=")] = append(maps.aqFavicon[extractQuery(q, "icon_hash=")], pv) + } + case "google": + if strings.HasPrefix(q, "intext:") { + maps.aqBody[extractQuery(q, "intext:")] = append(maps.aqBody[extractQuery(q, "intext:")], pv) + } else if strings.HasPrefix(q, "intitle:") { + maps.aqTitle[extractQuery(q, "intitle:")] = append(maps.aqTitle[extractQuery(q, "intitle:")], pv) + } + } + } + } + } + + return maps, nil +} + +func extractQuery(query string, prefix string) string { + q := strings.TrimPrefix(query, prefix) + return strings.Trim(q, "\"") +} + +func (a *AwesomeSearchMaps) FindMatches(result *Result) ([]ProductVendor, bool) { + var matches []ProductVendor + matchMap := make(map[string]bool) + + if result.Title != "" { + for title, pvs := range a.aqTitle { + if strings.Contains(strings.ToLower(result.Title), strings.ToLower(title)) { + for _, pv := range pvs { + key := pv.Product + pv.Vendor + if !matchMap[key] { + matches = append(matches, pv) + matchMap[key] = true + } + } + } + } + } + + if result.ResponseBody != "" { + for body, pvs := range a.aqBody { + if strings.Contains(strings.ToLower(result.ResponseBody), strings.ToLower(body)) { + for _, pv := range pvs { + key := pv.Product + pv.Vendor + if !matchMap[key] { + matches = append(matches, pv) + matchMap[key] = true + } + } + } + } + } + + if result.FavIconMMH3 != "" { + for favicon, pvs := range a.aqFavicon { + if result.FavIconMMH3 == favicon { + for _, pv := range pvs { + key := pv.Product + pv.Vendor + if !matchMap[key] { + matches = append(matches, pv) + matchMap[key] = true + } + } + } + } + } + + return matches, len(matches) > 0 +} diff --git a/runner/options.go b/runner/options.go index c09944d18..ca1f7f3bb 100644 --- a/runner/options.go +++ b/runner/options.go @@ -335,9 +335,11 @@ type Options struct { Trace bool // Optional pre-created objects to reduce allocations - Wappalyzer *wappalyzer.Wappalyze - Networkpolicy *networkpolicy.NetworkPolicy - CDNCheckClient *cdncheck.Client + Wappalyzer *wappalyzer.Wappalyze + Networkpolicy *networkpolicy.NetworkPolicy + CDNCheckClient *cdncheck.Client + AwesomeSearchQueries bool + WordPress bool } // ParseOptions parses the command line options for application @@ -369,6 +371,7 @@ func ParseOptions() *Options { flagSet.DynamicVarP(&options.ResponseBodyPreviewSize, "body-preview", "bp", 100, "display first N characters of response body"), flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "display server name"), flagSet.BoolVarP(&options.TechDetect, "tech-detect", "td", false, "display technology in use based on wappalyzer dataset"), + flagSet.BoolVar(&options.AwesomeSearchQueries, "cpe", false, "display product and vendor information based on awesome search queries"), flagSet.BoolVar(&options.OutputMethod, "method", false, "display http request method"), flagSet.BoolVar(&options.OutputWebSocket, "websocket", false, "display server using websocket"), flagSet.BoolVar(&options.OutputIP, "ip", false, "display host ip"), @@ -377,6 +380,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.Asn, "asn", false, "display host asn information"), flagSet.DynamicVar(&options.OutputCDN, "cdn", "true", "display cdn/waf in use"), flagSet.BoolVar(&options.Probe, "probe", false, "display probe status"), + flagSet.BoolVarP(&options.WordPress, "wordpress", "wp", false, "display WordPress themes and plugins"), ) flagSet.CreateGroup("headless", "Headless", @@ -623,6 +627,11 @@ func ParseOptions() *Options { gologger.Fatal().Msgf("%s\n", err) } + // Enable WordPress detection for JSON output + if options.JSONOutput { + options.WordPress = true + } + return options } diff --git a/runner/runner.go b/runner/runner.go index 4b8061507..9a53216c2 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -90,6 +90,8 @@ type Runner struct { pHashClusters []pHashCluster simHashes gcache.Cache[uint64, struct{}] // Include simHashes for efficient duplicate detection httpApiEndpoint *Server + awesomeQueries *AwesomeSearchMaps + wpData *WordPressData } func (r *Runner) HTTPX() *httpx.HTTPX { @@ -375,6 +377,26 @@ func New(options *Options) (*Runner, error) { }() } + if options.JSONOutput || options.AwesomeSearchQueries { + aq, err := LoadAwesomeQueries() + if err != nil { + gologger.Warning().Msgf("Could not load awesome search queries: %s", err) + } else { + runner.awesomeQueries = aq + } + } + + if options.WordPress { + wpData, err := NewWordPressData() + if err != nil { + return nil, err + } + if err := wpData.LoadData(); err != nil { + gologger.Warning().Msgf("Could not load WordPress data: %s", err) + } + runner.wpData = wpData + } + return runner, nil } @@ -2225,6 +2247,74 @@ retry: } } + // Add awesome queries check here, before creating the result struct + var product, vendor, cpe string + if r.awesomeQueries != nil { + tempResult := Result{ + Title: title, + ResponseBody: string(resp.Data), + FavIconMMH3: faviconMMH3, + } + if matches, found := r.awesomeQueries.FindMatches(&tempResult); found && len(matches) > 0 { + product = matches[0].Product + vendor = matches[0].Vendor + cpe = fmt.Sprintf("cpe:2.3:a:%s:%s:*:*:*:*:*:*:*:*", strings.ToLower(vendor), strings.ToLower(product)) + + // Update the builder string for CLI output + if r.options.AwesomeSearchQueries { + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Magenta(cpe).String()) + } else { + builder.WriteString(cpe) + } + builder.WriteString("] [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Magenta(vendor).String()) + } else { + builder.WriteString(vendor) + } + builder.WriteString("] [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Magenta(product).String()) + } else { + builder.WriteString(product) + } + builder.WriteString("]") + } + } + } + + var wpInfo *WordPressInfo + if r.wpData != nil { + wpInfo = r.wpData.ExtractInfo(string(resp.Data)) + if wpInfo != nil { + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + if len(wpInfo.Plugins) > 0 { + builder.WriteString(aurora.Magenta(fmt.Sprintf("WP Plugins: %s", strings.Join(wpInfo.Plugins, ","))).String()) + } + if len(wpInfo.Themes) > 0 { + if len(wpInfo.Plugins) > 0 { + builder.WriteString("] [") + } + builder.WriteString(aurora.Magenta(fmt.Sprintf("WP Themes: %s", strings.Join(wpInfo.Themes, ","))).String()) + } + } else { + if len(wpInfo.Plugins) > 0 { + builder.WriteString(fmt.Sprintf("WP Plugins: %s", strings.Join(wpInfo.Plugins, ","))) + } + if len(wpInfo.Themes) > 0 { + if len(wpInfo.Plugins) > 0 { + builder.WriteString("] [") + } + builder.WriteString(fmt.Sprintf("WP Themes: %s", strings.Join(wpInfo.Themes, ","))) + } + } + builder.WriteString("]") + } + } + result := Result{ Timestamp: time.Now(), Request: request, @@ -2286,6 +2376,10 @@ retry: RequestRaw: requestDump, Response: resp, FaviconData: faviconData, + Product: product, + Vendor: vendor, + CPE: cpe, + WordPress: wpInfo, } if resp.BodyDomains != nil { result.Fqdns = resp.BodyDomains.Fqdns @@ -2294,6 +2388,7 @@ retry: if r.options.Trace { result.Trace = req.TraceInfo } + return result } diff --git a/runner/types.go b/runner/types.go index 7bdb3a9c6..cc1a186e7 100644 --- a/runner/types.go +++ b/runner/types.go @@ -100,6 +100,10 @@ type Result struct { Response *httpx.Response `json:"-" csv:"-" mapstructure:"-"` FaviconData []byte `json:"-" csv:"-" mapstructure:"-"` Trace *retryablehttp.TraceInfo `json:"trace,omitempty" csv:"trace" mapstructure:"trace"` + Product string `json:"product,omitempty" csv:"product"` + Vendor string `json:"vendor,omitempty" csv:"vendor"` + WordPress *WordPressInfo `json:"wordpress,omitempty" csv:"wordpress"` + CPE string `json:"cpe,omitempty" csv:"cpe"` } type Trace struct { diff --git a/runner/wordpress.go b/runner/wordpress.go new file mode 100644 index 000000000..9627975f9 --- /dev/null +++ b/runner/wordpress.go @@ -0,0 +1,121 @@ +package runner + +import ( + "bufio" + "bytes" + "regexp" + "strings" + "sync" + + awesomesearchqueries "github.com/projectdiscovery/awesome-search-queries" +) + +type WordPressInfo struct { + Plugins []string `json:"plugins,omitempty"` + Themes []string `json:"themes,omitempty"` +} + +type WordPressData struct { + pluginsMap map[string]struct{} + themesMap map[string]struct{} + sync.Once + pluginRegex *regexp.Regexp + themeRegex *regexp.Regexp +} + +func NewWordPressData() (*WordPressData, error) { + wp := &WordPressData{ + pluginsMap: make(map[string]struct{}), + themesMap: make(map[string]struct{}), + } + + var err error + wp.pluginRegex, err = regexp.Compile(`/wp-content/plugins/([^/]+)/`) + if err != nil { + return nil, err + } + + wp.themeRegex, err = regexp.Compile(`/wp-content/themes/([^/]+)/`) + if err != nil { + return nil, err + } + + return wp, nil +} + +func (w *WordPressData) LoadData() error { + var err error + w.Do(func() { + // Load plugins + pluginsData, err := awesomesearchqueries.GetWordPressPlugins() + if err != nil { + return + } + if err = w.loadFromBytes(pluginsData, w.pluginsMap); err != nil { + return + } + + // Load themes + themesData, err := awesomesearchqueries.GetWordPressThemes() + if err != nil { + return + } + if err = w.loadFromBytes(themesData, w.themesMap); err != nil { + return + } + }) + return err +} + +func (w *WordPressData) loadFromBytes(data []byte, dataMap map[string]struct{}) error { + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line != "" { + dataMap[line] = struct{}{} + } + } + return scanner.Err() +} + +func (w *WordPressData) ExtractInfo(body string) *WordPressInfo { + var info WordPressInfo + + // Extract and validate plugins + if matches := w.pluginRegex.FindAllStringSubmatch(body, -1); len(matches) > 0 { + seenPlugins := make(map[string]struct{}) + for _, match := range matches { + if len(match) > 1 { + plugin := match[1] + if _, exists := seenPlugins[plugin]; !exists { + if _, valid := w.pluginsMap[plugin]; valid { + info.Plugins = append(info.Plugins, plugin) + seenPlugins[plugin] = struct{}{} + } + } + } + } + } + + // Extract and validate themes + if matches := w.themeRegex.FindAllStringSubmatch(body, -1); len(matches) > 0 { + seenThemes := make(map[string]struct{}) + for _, match := range matches { + if len(match) > 1 { + theme := match[1] + if _, exists := seenThemes[theme]; !exists { + if _, valid := w.themesMap[theme]; valid { + info.Themes = append(info.Themes, theme) + seenThemes[theme] = struct{}{} + } + } + } + } + } + + if len(info.Plugins) == 0 && len(info.Themes) == 0 { + return nil + } + + return &info +} From e0fdb4139e2cf408a91f1653d3d85f7312837f96 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Wed, 30 Oct 2024 17:24:08 +0530 Subject: [PATCH 2/2] go mod clean up --- go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.mod b/go.mod index b2b8cd249..1adf3073f 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,6 @@ module github.com/projectdiscovery/httpx go 1.21.3 -toolchain go1.23.2 - require ( github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 github.com/PuerkitoBio/goquery v1.8.1