From 91b3a8cddc038aaf9cbb4ce57dbe9c2cf59c4fb0 Mon Sep 17 00:00:00 2001 From: Julian Knocke Date: Thu, 6 May 2021 10:28:23 +0200 Subject: [PATCH 1/3] Fix block and proc definition to work with Ruby 3.0 --- lib/arachnid2/typhoeus.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/arachnid2/typhoeus.rb b/lib/arachnid2/typhoeus.rb index c04564e..824b63b 100644 --- a/lib/arachnid2/typhoeus.rb +++ b/lib/arachnid2/typhoeus.rb @@ -9,7 +9,7 @@ def initialize(url) @cached_data = [] end - def crawl(opts = {}) + def crawl(opts = {}, &block) preflight(opts) typhoeus_preflight @@ -20,11 +20,11 @@ def crawl(opts = {}) break if time_to_stop? @global_visited.insert(q) - found_in_cache = use_cache(q, opts, &Proc.new) + found_in_cache = use_cache(q, opts, &block) return if found_in_cache request = ::Typhoeus::Request.new(q, request_options) - requestable = after_request(request, &Proc.new) + requestable = after_request(request, &block) @hydra.queue(request) if requestable end # max_concurrency.times do @@ -35,9 +35,9 @@ def crawl(opts = {}) end # def crawl(opts = {}) private - def after_request(request) + def after_request(request, &block) request.on_complete do |response| - cacheable = use_response(response, &Proc.new) + cacheable = use_response(response, &block) return unless cacheable put_cached_data(response.effective_url, @options, response) From 2e77e559d3d473156268ac8e0e7f6dc3532cc07a Mon Sep 17 00:00:00 2001 From: Julian Knocke Date: Mon, 17 May 2021 09:26:16 +0200 Subject: [PATCH 2/3] Fix block and proc definition to work with Ruby 3.0 --- lib/arachnid2.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/arachnid2.rb b/lib/arachnid2.rb index 1342767..a3e746e 100644 --- a/lib/arachnid2.rb +++ b/lib/arachnid2.rb @@ -105,11 +105,11 @@ def initialize(url) # # @return nil # - def crawl(opts = {}, with_watir = false) + def crawl(opts = {}, with_watir = false, &block) if with_watir crawl_watir(opts, &Proc.new) else - Arachnid2::Typhoeus.new(@url).crawl(opts, &Proc.new) + Arachnid2::Typhoeus.new(@url).crawl(opts, &block) end end From f77f07eb9dc10983254e6e3ada7d1b28f6da0c5e Mon Sep 17 00:00:00 2001 From: Julian Knocke Date: Wed, 19 May 2021 11:08:00 +0200 Subject: [PATCH 3/3] Fix 'unexpected return' in after_request --- lib/arachnid2/typhoeus.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/arachnid2/typhoeus.rb b/lib/arachnid2/typhoeus.rb index 824b63b..1ebfcd4 100644 --- a/lib/arachnid2/typhoeus.rb +++ b/lib/arachnid2/typhoeus.rb @@ -38,7 +38,7 @@ def crawl(opts = {}, &block) def after_request(request, &block) request.on_complete do |response| cacheable = use_response(response, &block) - return unless cacheable + next unless cacheable put_cached_data(response.effective_url, @options, response) end