diff --git a/lib/arachnid2.rb b/lib/arachnid2.rb index 1342767..a3e746e 100644 --- a/lib/arachnid2.rb +++ b/lib/arachnid2.rb @@ -105,11 +105,11 @@ def initialize(url) # # @return nil # - def crawl(opts = {}, with_watir = false) + def crawl(opts = {}, with_watir = false, &block) if with_watir crawl_watir(opts, &Proc.new) else - Arachnid2::Typhoeus.new(@url).crawl(opts, &Proc.new) + Arachnid2::Typhoeus.new(@url).crawl(opts, &block) end end diff --git a/lib/arachnid2/typhoeus.rb b/lib/arachnid2/typhoeus.rb index c04564e..1ebfcd4 100644 --- a/lib/arachnid2/typhoeus.rb +++ b/lib/arachnid2/typhoeus.rb @@ -9,7 +9,7 @@ def initialize(url) @cached_data = [] end - def crawl(opts = {}) + def crawl(opts = {}, &block) preflight(opts) typhoeus_preflight @@ -20,11 +20,11 @@ def crawl(opts = {}) break if time_to_stop? @global_visited.insert(q) - found_in_cache = use_cache(q, opts, &Proc.new) + found_in_cache = use_cache(q, opts, &block) return if found_in_cache request = ::Typhoeus::Request.new(q, request_options) - requestable = after_request(request, &Proc.new) + requestable = after_request(request, &block) @hydra.queue(request) if requestable end # max_concurrency.times do @@ -35,10 +35,10 @@ def crawl(opts = {}) end # def crawl(opts = {}) private - def after_request(request) + def after_request(request, &block) request.on_complete do |response| - cacheable = use_response(response, &Proc.new) - return unless cacheable + cacheable = use_response(response, &block) + next unless cacheable put_cached_data(response.effective_url, @options, response) end