Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions lib/arachnid2.rb
Original file line number Diff line number Diff line change
Expand Up @@ -105,11 +105,11 @@ def initialize(url)
#
# @return nil
#
def crawl(opts = {}, with_watir = false)
def crawl(opts = {}, with_watir = false, &block)
if with_watir
crawl_watir(opts, &Proc.new)
else
Arachnid2::Typhoeus.new(@url).crawl(opts, &Proc.new)
Arachnid2::Typhoeus.new(@url).crawl(opts, &block)
end
end

Expand Down
12 changes: 6 additions & 6 deletions lib/arachnid2/typhoeus.rb
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ def initialize(url)
@cached_data = []
end

def crawl(opts = {})
def crawl(opts = {}, &block)
preflight(opts)
typhoeus_preflight

Expand All @@ -20,11 +20,11 @@ def crawl(opts = {})
break if time_to_stop?
@global_visited.insert(q)

found_in_cache = use_cache(q, opts, &Proc.new)
found_in_cache = use_cache(q, opts, &block)
return if found_in_cache

request = ::Typhoeus::Request.new(q, request_options)
requestable = after_request(request, &Proc.new)
requestable = after_request(request, &block)
@hydra.queue(request) if requestable
end # max_concurrency.times do

Expand All @@ -35,10 +35,10 @@ def crawl(opts = {})
end # def crawl(opts = {})

private
def after_request(request)
def after_request(request, &block)
request.on_complete do |response|
cacheable = use_response(response, &Proc.new)
return unless cacheable
cacheable = use_response(response, &block)
next unless cacheable

put_cached_data(response.effective_url, @options, response)
end
Expand Down