Merge pull request #56 from dkinzer/allow-harvest-with-until-param

Allow for passing until param.

Author: barmintor

.gitignore

@@ -1,8 +1,9 @@
 /pkg/
 /doc/
 /coverage*
+/tmp/
 /.yardoc/
-
+.DS_Store
 # Exclude Gemfile.lock (best practice for gems)
 Gemfile.lock
 

Rakefile

@@ -34,6 +34,13 @@ namespace :test do
     t.warning = false
   end
 
+  Rake::TestTask.new('harvester') do |t|
+    t.libs << ['lib', 'test/harvester']
+    t.pattern = 'test/harvester/tc_*.rb'
+    #t.verbose = true
+    t.warning = false
+  end
+
   Rake::TestTask.new('provider') do |t|
     t.libs << ['lib', 'test/provider']
     t.pattern = 'test/provider/tc_*.rb'

bin/oai

@@ -5,8 +5,6 @@
 
 require 'optparse'
 
-DIRECTORY_LAYOUT = "%Y/%m".freeze
-
 require 'oai/harvester'
 
 include OAI::Harvester

lib/oai/harvester.rb

@@ -7,7 +7,6 @@
 require 'fileutils'
 require 'ostruct'
 require 'readline'
-require 'chronic'
 require 'socket'
 
 require 'oai/client'

lib/oai/harvester/harvest.rb

@@ -3,14 +3,16 @@
 
 module OAI
   module Harvester
-  
     class Harvest
+      DIRECTORY_LAYOUT = "%Y/%m".freeze
 
-      def initialize(config = nil, directory = nil, date = nil)
+      def initialize(config = nil, directory = nil, date = nil, to = nil)
         @config = config || Config.load
         @directory = directory || @config.storage
         @from = date
         @from.freeze
+        @until = to
+        @until.freeze
         @parser = defined?(XML::Document) ? 'libxml' : 'rexml'
       end
 
@@ -30,7 +32,11 @@ def start(sites = nil, interactive = false)
 
       def harvest(site)
         opts = build_options_hash(@config.sites[site])
-        harvest_time = Time.now.utc
+        if @until
+          harvest_time = @until.to_time.utc
+        else
+          harvest_time = Time.now.utc
+        end
 
         if "YYYY-MM-DD" == granularity(opts[:url])
           opts[:until] = harvest_time.strftime("%Y-%m-%d")
@@ -43,22 +49,27 @@ def harvest(site)
         # Allow a from date to be passed in
         opts[:from] = earliest(opts[:url]) unless opts[:from]
         opts.delete(:set) if 'all' == opts[:set]
-        
         begin
           # Connect, and download
           file, records = call(opts.delete(:url), opts)
 
-          # Move document to storage directory
-          dir = File.join(@directory, date_based_directory(harvest_time))
-          FileUtils.mkdir_p dir
-          FileUtils.mv(file.path, 
-            File.join(dir, "#{site}-#{filename(Time.parse(opts[:from]), 
-            harvest_time)}.xml.gz"))
+          # Move document to storage directory if configured
+          if @directory
+            directory_layout = @config.layouts[site] if @config.layouts
+            dir = File.join(@directory, date_based_directory(harvest_time, directory_layout))
+            FileUtils.mkdir_p dir
+            FileUtils.mv(file.path,
+              File.join(dir, "#{site}-#{filename(Time.parse(opts[:from]),
+              harvest_time)}.xml.gz"))
+          else
+            puts "no configured destination for temp file" if @interactive
+          end
           @config.sites[site]['last'] = harvest_time
-        rescue
-          raise $! unless $!.respond_to?(:code)
-          raise $! if not @interactive || "noRecordsMatch" != $!.code
-          puts "No new records available"
+        rescue OAI::NoMatchException
+          puts "No new records available" if @interactive
+        rescue OAI::Exception => ex
+          raise ex if not @interactive
+          puts ex.message
         end
       end
 
@@ -69,15 +80,15 @@ def call(url, opts)
         records = 0;
         client = OAI::Client.new(url, :parser => @parser)
         provider_config = client.identify
-        
+
         file = Tempfile.new('oai_data')
         gz = Zlib::GzipWriter.new(file)
         gz << "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n"
         gz << "<records>"
         begin
           response = client.list_records(options)
-          get_records(response.doc).each do |rec|
-            gz << rec
+          response.each do |rec|
+            gz << rec._source
             records += 1
           end
           puts "#{records} records retrieved" if @interactive
@@ -89,8 +100,8 @@ def call(url, opts)
             puts "\nresumption token recieved, continuing" if @interactive
             response = client.list_records(:resumption_token => 
               response.resumption_token)
-              get_records(response.doc).each do |rec|
-                gz << rec
+              response.each do |rec|
+                gz << rec._source
                 records += 1
               end
             puts "#{records} records retrieved" if @interactive
@@ -118,8 +129,9 @@ def build_options_hash(site)
         options
       end
 
-      def date_based_directory(time)
-        "#{time.strftime(DIRECTORY_LAYOUT)}"
+      def date_based_directory(time, directory_layout = nil)
+        directory_layout ||= Harvest::DIRECTORY_LAYOUT
+        "#{time.strftime(directory_layout)}"
       end
 
       def filename(from_time, until_time)
@@ -147,4 +159,4 @@ def earliest(url)
     end
 
   end
-end
+end

lib/oai/harvester/logging.rb

@@ -11,10 +11,9 @@ class Harvest
       def initialize(*args)
         orig_init(*args)
         @summary = []
-        @logger = Logger.new(File.join(@config.logfile, "harvester.log"), 
-          shift_age = 'weekly') if @config.logfile
+        @logger = @config.logfile ? Logger.new(File.join(@config.logfile, "harvester.log"), 'weekly') : Logger.new(STDOUT)
         @logger.datetime_format = "%Y-%m-%d %H:%M"
-      
+
         # Turn off logging if no logging directory is specified.
         @logger.level = Logger::FATAL unless @config.logfile
       end
@@ -25,7 +24,7 @@ def start(sites = nil, interactive = false)
           orig_start(sites)
           begin
             OAI::Harvester::
-              Mailer.send(@config.mail_server, @config.email, @summary)
+              Mailer.send(@config.mail_server, @config.email, @summary) if @config.email
           rescue
             @logger.error { "Error sending out summary email: #{$!}"}
           end

test/harvester/tc_harvest.rb

@@ -0,0 +1,42 @@
+require 'test_helper_harvester'
+
+class HarvestTest < Test::Unit::TestCase
+  ONE_HOUR = 3600
+  EARLIEST_FIXTURE = "1998-05-02T04:00:00Z"
+  LATEST_FIXTURE = "2005-12-25T05:00:00Z"
+  def test_harvest
+    until_value = Time.now.utc - ONE_HOUR
+    config = OpenStruct.new(sites: { 'test' => { 'url' => 'http://localhost:3333/oai' }})
+    OAI::Harvester::Harvest.new(config).start
+    last = config.sites.dig('test', 'last')
+    assert_kind_of Time, last
+    assert last >= (until_value + ONE_HOUR), "#{last} < #{(until_value + ONE_HOUR)}"
+  end
+
+  def test_harvest_from_last
+    from_value = Time.parse(LATEST_FIXTURE).utc
+    now = Time.now.utc
+    config = OpenStruct.new(sites: { 'test' => { 'url' => 'http://localhost:3333/oai' }})
+    OAI::Harvester::Harvest.new(config, nil, from_value).start
+    last = config.sites.dig('test', 'last')
+    assert last >= now, "#{last} < #{now}"
+  end
+
+  def test_harvest_after_last
+    from_value = Time.parse(LATEST_FIXTURE).utc + 1
+    config = OpenStruct.new(sites: { 'test' => { 'url' => 'http://localhost:3333/oai' }})
+    OAI::Harvester::Harvest.new(config, nil, from_value).start
+    last = config.sites.dig('test', 'last')
+    assert_kind_of NilClass, last
+  end
+
+  def test_harvest_with_until
+    until_value = Time.parse(EARLIEST_FIXTURE).utc + ONE_HOUR
+    config = OpenStruct.new(sites: { 'test' => { 'url' => 'http://localhost:3333/oai' }})
+    OAI::Harvester::Harvest.new(config, nil, nil, until_value).start
+    last = config.sites.dig('test', 'last')
+    assert_kind_of Time, last
+    assert_equal last, until_value
+  end
+end
+

test/harvester/test_helper_harvester.rb

@@ -0,0 +1,6 @@
+require 'oai'
+require 'oai/harvester'
+require 'test/unit'
+
+require File.dirname(__FILE__) + '/../client/helpers/provider'
+require File.dirname(__FILE__) + '/../client/helpers/test_wrapper'