Added Awesome Search Queries (-cpe) and Passive Wordpress Plugins and Themes ( -wp, -wordpress ) flags

common/httpx/option.go

@@ -50,6 +50,7 @@ type Options struct {
 	CDNCheckClient            *cdncheck.Client
 	Protocol                  Proto
 	Trace                     bool
+	AwesomeSearchQueries      bool
 }
 
 // DefaultOptions contains the default options
@@ -71,6 +72,7 @@ var DefaultOptions = Options{
 	VHostStripHTML:           false,
 	VHostSimilarityRatio:     85,
 	DefaultUserAgent:         "httpx - Open-source project (github.com/projectdiscovery/httpx)",
+	AwesomeSearchQueries:     false,
 }
 
 func (options *Options) parseCustomCookies() {

go.mod

@@ -1,6 +1,6 @@
 module github.com/projectdiscovery/httpx
 
-go 1.21
+go 1.21.3
 
 require (
 	github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057
@@ -52,6 +52,7 @@ require (
 
 require (
 	github.com/go-viper/mapstructure/v2 v2.1.0
+	github.com/projectdiscovery/awesome-search-queries v0.0.0-20241030094221-9fa3c0933578
 	github.com/weppos/publicsuffix-go v0.30.2
 )
 

go.sum

@@ -226,6 +226,8 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
 github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kID2iwsDqI=
 github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60=
+github.com/projectdiscovery/awesome-search-queries v0.0.0-20241030094221-9fa3c0933578 h1:PSpd8NNjmDUgfbhgB/49HZqXQ+7DJRnMR1TVyEus7PM=
+github.com/projectdiscovery/awesome-search-queries v0.0.0-20241030094221-9fa3c0933578/go.mod h1:nSovPcipgSx/EzAefF+iCfORolkKAuodiRWL3RCGHOM=
 github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ=
 github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
 github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/snk0cQ3kXE=

runner/awesome_queries.go

@@ -0,0 +1,141 @@
+package runner
+
+import (
+	"encoding/json"
+	"strings"
+
+	awesomesearchqueries "github.com/projectdiscovery/awesome-search-queries"
+)
+
+type AwesomeQuery struct {
+	Name    string    `json:"name"`
+	Vendor  string    `json:"vendor"`
+	Type    string    `json:"type"`
+	Engines []Engines `json:"engines"`
+}
+
+type Engines struct {
+	Platform string   `json:"platform"`
+	Queries  []string `json:"queries"`
+}
+
+type AwesomeSearchMaps struct {
+	aqTitle   map[string][]ProductVendor
+	aqBody    map[string][]ProductVendor
+	aqFavicon map[string][]ProductVendor
+}
+
+type ProductVendor struct {
+	Product string
+	Vendor  string
+}
+
+func LoadAwesomeQueries() (*AwesomeSearchMaps, error) {
+	data, err := awesomesearchqueries.GetQueries()
+	if err != nil {
+		return nil, err
+	}
+
+	var queries []AwesomeQuery
+	if err := json.Unmarshal(data, &queries); err != nil {
+		return nil, err
+	}
+
+	maps := &AwesomeSearchMaps{
+		aqTitle:   make(map[string][]ProductVendor),
+		aqBody:    make(map[string][]ProductVendor),
+		aqFavicon: make(map[string][]ProductVendor),
+	}
+
+	for _, query := range queries {
+		pv := ProductVendor{
+			Product: query.Name,
+			Vendor:  query.Vendor,
+		}
+
+		for _, engine := range query.Engines {
+			for _, q := range engine.Queries {
+				switch engine.Platform {
+				case "shodan":
+					if strings.HasPrefix(q, "http.html:") {
+						maps.aqBody[extractQuery(q, "http.html:")] = append(maps.aqBody[extractQuery(q, "http.html:")], pv)
+					} else if strings.HasPrefix(q, "http.title:") {
+						maps.aqTitle[extractQuery(q, "http.title:")] = append(maps.aqTitle[extractQuery(q, "http.title:")], pv)
+					} else if strings.HasPrefix(q, "http.favicon.hash:") {
+						maps.aqFavicon[extractQuery(q, "http.favicon.hash:")] = append(maps.aqFavicon[extractQuery(q, "http.favicon.hash:")], pv)
+					}
+				case "fofa":
+					if strings.HasPrefix(q, "body=") {
+						maps.aqBody[extractQuery(q, "body=")] = append(maps.aqBody[extractQuery(q, "body=")], pv)
+					} else if strings.HasPrefix(q, "title=") {
+						maps.aqTitle[extractQuery(q, "title=")] = append(maps.aqTitle[extractQuery(q, "title=")], pv)
+					} else if strings.HasPrefix(q, "icon_hash=") {
+						maps.aqFavicon[extractQuery(q, "icon_hash=")] = append(maps.aqFavicon[extractQuery(q, "icon_hash=")], pv)
+					}
+				case "google":
+					if strings.HasPrefix(q, "intext:") {
+						maps.aqBody[extractQuery(q, "intext:")] = append(maps.aqBody[extractQuery(q, "intext:")], pv)
+					} else if strings.HasPrefix(q, "intitle:") {
+						maps.aqTitle[extractQuery(q, "intitle:")] = append(maps.aqTitle[extractQuery(q, "intitle:")], pv)
+					}
+				}
+			}
+		}
+	}
+
+	return maps, nil
+}
+
+func extractQuery(query string, prefix string) string {
+	q := strings.TrimPrefix(query, prefix)
+	return strings.Trim(q, "\"")
+}
+
+func (a *AwesomeSearchMaps) FindMatches(result *Result) ([]ProductVendor, bool) {
+	var matches []ProductVendor
+	matchMap := make(map[string]bool)
+
+	if result.Title != "" {
+		for title, pvs := range a.aqTitle {
+			if strings.Contains(strings.ToLower(result.Title), strings.ToLower(title)) {
+				for _, pv := range pvs {
+					key := pv.Product + pv.Vendor
+					if !matchMap[key] {
+						matches = append(matches, pv)
+						matchMap[key] = true
+					}
+				}
+			}
+		}
+	}
+
+	if result.ResponseBody != "" {
+		for body, pvs := range a.aqBody {
+			if strings.Contains(strings.ToLower(result.ResponseBody), strings.ToLower(body)) {
+				for _, pv := range pvs {
+					key := pv.Product + pv.Vendor
+					if !matchMap[key] {
+						matches = append(matches, pv)
+						matchMap[key] = true
+					}
+				}
+			}
+		}
+	}
+
+	if result.FavIconMMH3 != "" {
+		for favicon, pvs := range a.aqFavicon {
+			if result.FavIconMMH3 == favicon {
+				for _, pv := range pvs {
+					key := pv.Product + pv.Vendor
+					if !matchMap[key] {
+						matches = append(matches, pv)
+						matchMap[key] = true
+					}
+				}
+			}
+		}
+	}
+
+	return matches, len(matches) > 0
+}

runner/options.go

@@ -335,9 +335,11 @@ type Options struct {
 	Trace bool
 
 	// Optional pre-created objects to reduce allocations
-	Wappalyzer     *wappalyzer.Wappalyze
-	Networkpolicy  *networkpolicy.NetworkPolicy
-	CDNCheckClient *cdncheck.Client
+	Wappalyzer           *wappalyzer.Wappalyze
+	Networkpolicy        *networkpolicy.NetworkPolicy
+	CDNCheckClient       *cdncheck.Client
+	AwesomeSearchQueries bool
+	WordPress            bool
 }
 
 // ParseOptions parses the command line options for application
@@ -369,6 +371,7 @@ func ParseOptions() *Options {
 		flagSet.DynamicVarP(&options.ResponseBodyPreviewSize, "body-preview", "bp", 100, "display first N characters of response body"),
 		flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "display server name"),
 		flagSet.BoolVarP(&options.TechDetect, "tech-detect", "td", false, "display technology in use based on wappalyzer dataset"),
+		flagSet.BoolVar(&options.AwesomeSearchQueries, "cpe", false, "display product and vendor information based on awesome search queries"),
 		flagSet.BoolVar(&options.OutputMethod, "method", false, "display http request method"),
 		flagSet.BoolVar(&options.OutputWebSocket, "websocket", false, "display server using websocket"),
 		flagSet.BoolVar(&options.OutputIP, "ip", false, "display host ip"),
@@ -377,6 +380,7 @@ func ParseOptions() *Options {
 		flagSet.BoolVar(&options.Asn, "asn", false, "display host asn information"),
 		flagSet.DynamicVar(&options.OutputCDN, "cdn", "true", "display cdn/waf in use"),
 		flagSet.BoolVar(&options.Probe, "probe", false, "display probe status"),
+		flagSet.BoolVarP(&options.WordPress, "wordpress", "wp", false, "display WordPress themes and plugins"),
 	)
 
 	flagSet.CreateGroup("headless", "Headless",
@@ -623,6 +627,11 @@ func ParseOptions() *Options {
 		gologger.Fatal().Msgf("%s\n", err)
 	}
 
+	// Enable WordPress detection for JSON output
+	if options.JSONOutput {
+		options.WordPress = true
+	}
+
 	return options
 }
 

runner/runner.go

@@ -90,6 +90,8 @@ type Runner struct {
 	pHashClusters      []pHashCluster
 	simHashes          gcache.Cache[uint64, struct{}] // Include simHashes for efficient duplicate detection
 	httpApiEndpoint    *Server
+	awesomeQueries     *AwesomeSearchMaps
+	wpData             *WordPressData
 }
 
 func (r *Runner) HTTPX() *httpx.HTTPX {
@@ -375,6 +377,26 @@ func New(options *Options) (*Runner, error) {
 		}()
 	}
 
+	if options.JSONOutput || options.AwesomeSearchQueries {
+		aq, err := LoadAwesomeQueries()
+		if err != nil {
+			gologger.Warning().Msgf("Could not load awesome search queries: %s", err)
+		} else {
+			runner.awesomeQueries = aq
+		}
+	}
+
+	if options.WordPress {
+		wpData, err := NewWordPressData()
+		if err != nil {
+			return nil, err
+		}
+		if err := wpData.LoadData(); err != nil {
+			gologger.Warning().Msgf("Could not load WordPress data: %s", err)
+		}
+		runner.wpData = wpData
+	}
+
 	return runner, nil
 }
 
@@ -2225,6 +2247,74 @@ retry:
 		}
 	}
 
+	// Add awesome queries check here, before creating the result struct
+	var product, vendor, cpe string
+	if r.awesomeQueries != nil {
+		tempResult := Result{
+			Title:        title,
+			ResponseBody: string(resp.Data),
+			FavIconMMH3:  faviconMMH3,
+		}
+		if matches, found := r.awesomeQueries.FindMatches(&tempResult); found && len(matches) > 0 {
+			product = matches[0].Product
+			vendor = matches[0].Vendor
+			cpe = fmt.Sprintf("cpe:2.3:a:%s:%s:*:*:*:*:*:*:*:*", strings.ToLower(vendor), strings.ToLower(product))
+
+			// Update the builder string for CLI output
+			if r.options.AwesomeSearchQueries {
+				builder.WriteString(" [")
+				if !scanopts.OutputWithNoColor {
+					builder.WriteString(aurora.Magenta(cpe).String())
+				} else {
+					builder.WriteString(cpe)
+				}
+				builder.WriteString("] [")
+				if !scanopts.OutputWithNoColor {
+					builder.WriteString(aurora.Magenta(vendor).String())
+				} else {
+					builder.WriteString(vendor)
+				}
+				builder.WriteString("] [")
+				if !scanopts.OutputWithNoColor {
+					builder.WriteString(aurora.Magenta(product).String())
+				} else {
+					builder.WriteString(product)
+				}
+				builder.WriteString("]")
+			}
+		}
+	}
+
+	var wpInfo *WordPressInfo
+	if r.wpData != nil {
+		wpInfo = r.wpData.ExtractInfo(string(resp.Data))
+		if wpInfo != nil {
+			builder.WriteString(" [")
+			if !scanopts.OutputWithNoColor {
+				if len(wpInfo.Plugins) > 0 {
+					builder.WriteString(aurora.Magenta(fmt.Sprintf("WP Plugins: %s", strings.Join(wpInfo.Plugins, ","))).String())
+				}
+				if len(wpInfo.Themes) > 0 {
+					if len(wpInfo.Plugins) > 0 {
+						builder.WriteString("] [")
+					}
+					builder.WriteString(aurora.Magenta(fmt.Sprintf("WP Themes: %s", strings.Join(wpInfo.Themes, ","))).String())
+				}
+			} else {
+				if len(wpInfo.Plugins) > 0 {
+					builder.WriteString(fmt.Sprintf("WP Plugins: %s", strings.Join(wpInfo.Plugins, ",")))
+				}
+				if len(wpInfo.Themes) > 0 {
+					if len(wpInfo.Plugins) > 0 {
+						builder.WriteString("] [")
+					}
+					builder.WriteString(fmt.Sprintf("WP Themes: %s", strings.Join(wpInfo.Themes, ",")))
+				}
+			}
+			builder.WriteString("]")
+		}
+	}
+
 	result := Result{
 		Timestamp:        time.Now(),
 		Request:          request,
@@ -2286,6 +2376,10 @@ retry:
 		RequestRaw:        requestDump,
 		Response:          resp,
 		FaviconData:       faviconData,
+		Product:           product,
+		Vendor:            vendor,
+		CPE:               cpe,
+		WordPress:         wpInfo,
 	}
 	if resp.BodyDomains != nil {
 		result.Fqdns = resp.BodyDomains.Fqdns
@@ -2294,6 +2388,7 @@ retry:
 	if r.options.Trace {
 		result.Trace = req.TraceInfo
 	}
+
 	return result
 }
 

runner/types.go

@@ -100,6 +100,10 @@ type Result struct {
 	Response           *httpx.Response               `json:"-" csv:"-" mapstructure:"-"`
 	FaviconData        []byte                        `json:"-" csv:"-" mapstructure:"-"`
 	Trace              *retryablehttp.TraceInfo      `json:"trace,omitempty" csv:"trace"  mapstructure:"trace"`
+	Product            string                        `json:"product,omitempty" csv:"product"`
+	Vendor             string                        `json:"vendor,omitempty" csv:"vendor"`
+	WordPress          *WordPressInfo                `json:"wordpress,omitempty" csv:"wordpress"`
+	CPE                string                        `json:"cpe,omitempty" csv:"cpe"`
 }
 
 type Trace struct {