fetch-external-resources

#!/usr/bin/env ruby
#
# This script is responsible for reaching out to the internet and downloading
# external content which needs to be built into the site.


require 'faraday'
require 'faraday/follow_redirects'
require 'fileutils'
require 'json'
require 'uri'
require 'yaml'
require 'zip'


RESOURCES = [
  [
    'https://updates.jenkins.io/latestCore.txt',
    'content/_tmp/latestCore.txt',
    nil,
    nil
  ],
  [
    'https://updates.jenkins.io/stable/latestCore.txt',
    'content/_tmp/latestLTSCore.txt',
    nil,
    nil
  ],
  [
    'https://github.com/jenkinsci/pipeline-examples/archive/master.zip',
    'content/_tmp/pipeline-examples-master.zip',
    nil,
    'content/_tmp/examples'
  ],
  [
    'https://reports.jenkins.io/allAscii.zip',
    'content/_tmp/allAscii.zip',
    nil,
    'content/doc/pipeline/steps'
  ],
  [
    'https://repo.jenkins-ci.org/api/search/versions?g=org.jenkins-ci.main&a=jenkins-core&repos=releases&v=?.*.1',
    'content/_data/_generated/lts_baselines.yml',
    nil,
    nil
  ],
  [
    'https://repo.jenkins-ci.org/api/search/versions?g=org.jenkins-ci.main&a=jenkins-core&repos=releases&v=?.*.?',
    'content/_data/_generated/lts_releases.yml',
    nil,
    nil
  ],
  [
    'https://updates.jenkins.io/update-center.actual.json',
    'content/_data/_generated/update_center.yml',
    nil,
    nil
  ],
  [
    'https://updates.jenkins.io/tiers.json',
    'content/_data/_generated/core_tiers.yml',
    nil,
    nil
  ],
  [
    'https://reports.jenkins.io/extension-indexer.zip',
    'content/_tmp/extension-indexer.zip',
    nil,
    'content/doc/developer/extensions'
  ],
]


class Fetcher
  # Fetch a single resource and return the +Faraday::Response+ for it
  #
  # This abstraction exists so more behaviors (caching, etc) can be injected
  # into the fetching of specific resources themeslves
  def fetch_resource(origin)
    conn = Faraday.new(:ssl => {:verify => false}) do |f|
      f.use Faraday::FollowRedirects::Middleware, limit: 5
      f.adapter Faraday.default_adapter
    end
    return conn.get(origin)
  end

  # process a list of resources and generate files from them
  def process(resources)
    failures = []
    puts "Fetching external resources:"

    resources.each do |origin, destination, frontmatter, zip_dest|
      response = fetch_resource(origin)

      unless response.success?
        failures << [response, origin]
        next
      end

      FileUtils.mkdir_p File.dirname(destination)

      File.open(destination, 'w+') do |f|
        if frontmatter
          f.write(frontmatter.to_yaml)
          f.write("---\n")
        end

        puts "Writing #{destination}"
        if destination.include? ".yml"
          puts "Converting JSON to YAML"
          f.write(YAML.dump(JSON.load(response.body)))
        else
          f.write(response.body)
        end
      end

      #if downloading a zip file, extract it
      if zip_dest
        puts "Unzipping to #{zip_dest}"
        Zip.on_exists_proc = true

        FileUtils.mkdir_p zip_dest

        Zip::File.open(destination) do |zip_file|
          zip_file.each do |f|
            fpath = File.join(zip_dest, f.name)
            FileUtils.mkdir_p(File.dirname(fpath))
            zip_file.extract(f, fpath)
          end
        end
      end
    end

    # If we collected any failures, dump those out while we're here
    unless failures.empty?
      puts "Failed processing resources!"
      puts
      failures.each do |response, origin|
        puts "#{origin} #{response.inspect}"
      end
      exit 1
    end
  end
end


Fetcher.new.process(RESOURCES)