Add more params to web/spider

Author: mozi

lib/ronin/recon/builtin/web/spider.rb

@@ -45,6 +45,30 @@ class Spider < WebWorker
         accepts Website
         outputs URL
 
+        param :limit, Integer, desc: 'The maximum number of pages to visit.'
+
+        param :max_depth, Integer, desc: 'The maximum link depth to follow.'
+
+        param :delay, Integer, desc: 'The number of seconds to pause between each request.'
+
+        param :open_timeout, Integer, desc: 'Optional open connection timeout.'
+
+        param :read_timeout, Integer, desc: 'Optional read timeout.'
+
+        param :ssl_timeout, Integer, desc: 'Optional SSL connection timeout.'
+
+        param :continue_timeout, Integer, desc: 'Optional continue timeout.'
+
+        param :keep_alive_timeout, Integer, desc: 'Optional `Keep-Alive` timeout.'
+
+        param :proxy, URI, desc: 'The proxy information to use.'
+
+        param :referer, String, desc: 'The `Referer` URL to send with each request.'
+
+        param :user_agent, String, desc: 'The `User-Agent` string to send with each requests.'
+
+        param :robots, Boolean, desc: 'Specifies whether `robots.txt` should be honored.'
+
         #
         # Spiders a website and yields every spidered URL.
         #
@@ -60,7 +84,7 @@ class Spider < WebWorker
         def process(website)
           base_uri = website.to_uri
 
-          Ronin::Web::Spider.site(base_uri) do |agent|
+          Ronin::Web::Spider.site(base_uri, **agent_kwargs) do |agent|
             agent.every_page do |page|
               if VALID_STATUS_CODES.include?(page.code)
                 yield URL.new(page.url, status:  page.code,
@@ -91,6 +115,13 @@ def process(website)
           end
         end
 
+        #
+        # Returns Hash based on the params
+        #
+        def agent_kwargs
+          params.slice(:limit, :max_depth, :delay, :open_timeout, :read_timeout, :ssl_timeout, :continue_timeout, :keep_alive_timeout, :proxy, :referer, :user_agent, :robots)
+        end
+
       end
     end
   end

spec/builtin/web/spider_spec.rb

@@ -463,4 +463,41 @@ class AppWithURLsInJavaScript < Sinatra::Base
       end
     end
   end
+
+  describe "#agent_kwargs" do
+    context "when there are params" do
+      let(:params_hash) do
+        {
+          limit: 10,
+          max_depth: 2,
+          delay: 5,
+          open_timeout: 10,
+          read_timeout: 10,
+          ssl_timeout: 10,
+          continue_timeout: 10,
+          keep_alive_timeout: 10,
+          proxy: URI('http://proxy.example.com:8080'),
+          referer: 'http://example.com/',
+          user_agent: 'Test User Agent',
+          robots: true
+        }
+      end
+
+      subject { described_class.new(params: params_hash) }
+
+      it "must return a Hash with the params" do
+        expect(subject.agent_kwargs).to eq(params_hash)
+      end
+    end
+
+    context "when there are no params" do
+      let(:params_hash) { {} }
+
+      subject { described_class.new(params: params_hash) }
+
+      it "must return an empty Hash" do
+        expect(subject.agent_kwargs).to eq(params_hash)
+      end
+    end
+  end
 end