Provide an alternative implementation of Net::SSH::KnownHost in ssh_options

As discussed in capistrano#326 (comment)
Net::SSH re-parse the known_hosts files every time it needs to lookup for a known key.
This alternative implementation parse it once and for all, and cache the result.

Author: byroot

README.md

@@ -508,6 +508,19 @@ pooling behaviour entirely by setting the idle_timeout to zero:
 SSHKit::Backend::Netssh.pool.idle_timeout = 0 # disabled
 ```
 
+## Known hosts caching
+
+If you connect to many hosts with the `Netssh` backend, looking up `~/.ssh/known_hosts` can significantly impact performances.
+You can mitigate this by using SSHKit's lookup caching like this:
+
+```ruby
+SSHKit::Backend::Netssh.configure do |ssh|
+  ssh.ssh_options = {
+    known_hosts: SSHKit::Backend::Netssh::KnownHosts.new,
+  }
+end
+```
+
 ## Tunneling and other related SSH themes
 
 In order to do special gymnasitcs with SSH, tunneling, aliasing, complex options, etc with SSHKit it is possible to use [the underlying Net::SSH API](https://github.com/capistrano/sshkit/blob/master/EXAMPLES.md#setting-global-ssh-options) however in many cases it is preferred to use the system SSH configuration file at [`~/.ssh/config`](http://man.cx/ssh_config). This allows you to have personal configuration tied to your machine that does not have to be committed with the repository. If this is not suitable (everyone on the team needs a proxy command, or some special aliasing) a file in the same format can be placed in the project directory at `~/yourproject/.ssh/config`, this will be merged with the system settings in `~/.ssh/config`, and with any configuration specified in [`SSHKit::Backend::Netssh.config.ssh_options`](https://github.com/capistrano/sshkit/blob/master/lib/sshkit/backends/netssh.rb#L133).

lib/sshkit/all.rb

@@ -35,5 +35,6 @@
 require_relative 'backends/connection_pool'
 require_relative 'backends/printer'
 require_relative 'backends/netssh'
+require_relative 'backends/netssh/known_hosts'
 require_relative 'backends/local'
 require_relative 'backends/skipper'

lib/sshkit/backends/netssh.rb

@@ -1,4 +1,6 @@
 require 'English'
+require 'strscan'
+require 'mutex_m'
 require 'net/ssh'
 require 'net/scp'
 
@@ -21,13 +23,12 @@ module SSHKit
   module Backend
 
     class Netssh < Abstract
-
       class Configuration
         attr_accessor :connection_timeout, :pty
         attr_writer :ssh_options
 
         def ssh_options
-          @ssh_options || {}
+          @ssh_options ||= {}
         end
       end
 

lib/sshkit/backends/netssh/known_hosts.rb

@@ -0,0 +1,141 @@
+module SSHKit
+
+  module Backend
+
+    class Netssh < Abstract
+
+      class KnownHostsKeys
+        include Mutex_m
+
+        def initialize(path)
+          super()
+          @path = File.expand_path(path)
+          @hosts_keys = nil
+        end
+
+        def keys_for(hostlist)
+          keys, hashes = hosts_keys, hosts_hashes
+          parse_file unless keys && hashes
+          keys, hashes = hosts_keys, hosts_hashes
+
+          hostlist.split(',').each do |host|
+            key_list = keys[host]
+            return key_list if key_list
+
+            hashes.each do |(hmac, salt), hash_keys|
+              if OpenSSL::HMAC.digest(sha1, salt, host) == hmac
+                return hash_keys
+              end
+            end
+          end
+
+          []
+        end
+
+        private
+
+        attr_reader :path
+        attr_accessor :hosts_keys, :hosts_hashes
+
+        def sha1
+          @sha1 ||= OpenSSL::Digest.new('sha1')
+        end
+
+        def parse_file
+          synchronize do
+            return if hosts_keys && hosts_hashes
+
+            unless File.readable?(path)
+              self.hosts_keys = {}
+              self.hosts_hashes = []
+              return
+            end
+
+            new_keys = {}
+            new_hashes = []
+            File.open(path) do |file|
+              scanner = StringScanner.new("")
+              file.each_line do |line|
+                scanner.string = line
+                parse_line(scanner, new_keys, new_hashes)
+              end
+            end
+            self.hosts_keys = new_keys
+            self.hosts_hashes = new_hashes
+          end
+        end
+
+        def parse_line(scanner, hosts_keys, hosts_hashes)
+          return if empty_line?(scanner)
+
+          hostlist = parse_hostlist(scanner)
+          return unless supported_type?(scanner)
+          key = parse_key(scanner)
+
+          if hostlist.size == 1 && hostlist.first =~ /\A\|1(\|.+){2}\z/
+            hosts_hashes << [parse_host_hash(hostlist.first), key]
+          else
+            hostlist.each do |host|
+              (hosts_keys[host] ||= []) << key
+            end
+          end
+        end
+
+        def parse_host_hash(line)
+          _, _, salt, hmac = line.split('|')
+          [Base64.decode64(hmac), Base64.decode64(salt)]
+        end
+
+        def empty_line?(scanner)
+          scanner.skip(/\s*/)
+          scanner.match?(/$|#/)
+        end
+
+        def parse_hostlist(scanner)
+          scanner.skip(/\s*/)
+          scanner.scan(/\S+/).split(',')
+        end
+
+        def supported_type?(scanner)
+          scanner.skip(/\s*/)
+          Net::SSH::KnownHosts::SUPPORTED_TYPE.include?(scanner.scan(/\S+/))
+        end
+
+        def parse_key(scanner)
+          scanner.skip(/\s*/)
+          Net::SSH::Buffer.new(scanner.rest.unpack("m*").first).read_key
+        end
+      end
+
+      class KnownHosts
+        include Mutex_m
+
+        def initialize
+          super()
+          @files = {}
+        end
+
+        def search_for(host, options = {})
+          keys = ::Net::SSH::KnownHosts.hostfiles(options).map do |path|
+            known_hosts_file(path).keys_for(host)
+          end.flatten
+          ::Net::SSH::HostKeys.new(keys, host, self, options)
+        end
+
+        def add(*args)
+          ::Net::SSH::KnownHosts.add(*args)
+          synchronize { @files = {} }
+        end
+
+        private
+
+        def known_hosts_file(path)
+          @files[path] || synchronize { @files[path] ||= KnownHostsKeys.new(path) }
+        end
+      end
+
+    end
+
+  end
+
+end

test/known_hosts/github

@@ -0,0 +1 @@
+github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==

test/known_hosts/github_hash

@@ -0,0 +1 @@
+|1|eKp+6E0rZ3lONgsIziurXEnaIik=|rcQB/rlJMUquUyFta64KugPjX4o= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==

test/unit/backends/test_netssh.rb

@@ -1,4 +1,5 @@
 require 'helper'
+require 'tempfile'
 
 module SSHKit
   module Backend
@@ -53,6 +54,29 @@ def test_transfer_summarizer
         end
       end
 
+      if Net::SSH::Version::CURRENT >= Net::SSH::Version[3, 1, 0]
+        def test_known_hosts_for_when_all_hosts_are_recognized
+          perform_known_hosts_test("github")
+        end
+
+        def test_known_hosts_for_when_an_host_hash_is_recognized
+          perform_known_hosts_test("github_hash")
+        end
+      end
+
+      private
+
+      def perform_known_hosts_test(hostfile)
+        source = File.join(File.dirname(__FILE__), '../../known_hosts', hostfile)
+        kh = Netssh::KnownHosts.new
+        keys = kh.search_for('github.com', user_known_hosts_file: source, global_known_hosts_file: Tempfile.new('sshkit-test').path)
+
+        assert_instance_of ::Net::SSH::HostKeys, keys
+        assert_equal(1, keys.count)
+        keys.each do |key|
+          assert_equal("ssh-rsa", key.ssh_type)
+        end
+      end
     end
   end
 end