add option to hide responses of a certain content-length

Author: liamg

cmd/scout/url.go

@@ -22,6 +22,7 @@ var filename string
 var headers []string
 var extensions = []string{"php", "htm", "html", "txt"}
 var enableSpidering bool
+var ignoredLengths []int
 
 var urlCmd = &cobra.Command{
 	Use:   "url [url]",
@@ -80,6 +81,7 @@ var urlCmd = &cobra.Command{
 
 		options := []scan.URLOption{
 			scan.WithPositiveStatusCodes(intStatusCodes),
+			scan.WithNegativeLengths(ignoredLengths),
 			scan.WithTargetURL(*parsedURL),
 			scan.WithResultChan(resultChan),
 			scan.WithBusyChan(busyChan),
@@ -124,7 +126,7 @@ var urlCmd = &cobra.Command{
 
 		go func() {
 			for result := range resultChan {
-				importantOutputChan <- tml.Sprintf("<blue>[</blue><yellow>%d</yellow><blue>]</blue> %s\n", result.StatusCode, result.URL.String())
+				importantOutputChan <- tml.Sprintf("<blue>[</blue><yellow>%d</yellow><blue>]</blue> <blue>[</blue><yellow>%d</yellow><blue>]</blue> %s\n", result.StatusCode, result.Size, result.URL.String())
 			}
 			close(waitChan)
 		}()
@@ -197,6 +199,7 @@ func init() {
 	urlCmd.Flags().StringSliceVarP(&extensions, "extensions", "x", extensions, "File extensions to detect.")
 	urlCmd.Flags().StringSliceVarP(&headers, "header", "H", headers, "Extra header to send with requests (can be specified multiple times).")
 	urlCmd.Flags().BoolVarP(&enableSpidering, "spider", "s", enableSpidering, "Spider links within page content")
+	urlCmd.Flags().IntSliceVarP(&ignoredLengths, "hide-lengths", "l", ignoredLengths, "Hide results with these content lengths")
 
 	rootCmd.AddCommand(urlCmd)
 }

pkg/scan/url_options.go

@@ -31,6 +31,13 @@ func WithPositiveStatusCodes(codes []int) URLOption {
 	}
 }
 
+// WithNegativeLengths provides lengths which should be ignored
+func WithNegativeLengths(lengths []int) URLOption {
+	return func(s *URLScanner) {
+		s.negativeLengths = lengths
+	}
+}
+
 func WithTimeout(timeout time.Duration) URLOption {
 	return func(s *URLScanner) {
 		s.timeout = timeout
@@ -104,4 +111,5 @@ func WithMethod(method string) URLOption {
 type URLResult struct {
 	URL        url.URL
 	StatusCode int
+	Size       int
 }

pkg/scan/url_scanner.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -43,6 +44,7 @@ type URLScanner struct {
 	jobsLoaded          int32
 	proxy               *url.URL
 	method              string
+	negativeLengths     []int
 }
 
 type URLJob struct {
@@ -334,9 +336,22 @@ func (scanner *URLScanner) checkURL(job URLJob) *URLResult {
 					_, _ = io.Copy(ioutil.Discard, resp.Body)
 				}
 
+				var size int
+				contentLength := resp.Header.Get("Content-Length")
+				if contentLength != "" {
+					size, _ = strconv.Atoi(contentLength)
+				}
+
+				for _, length := range scanner.negativeLengths {
+					if length == size {
+						return nil
+					}
+				}
+
 				result = &URLResult{
 					StatusCode: code,
 					URL:        *parsedURL,
+					Size:       size,
 				}
 
 				break