From d5d296095dde763beccae6d94ed492dfe599a007 Mon Sep 17 00:00:00 2001 From: Abhishek Rana <48236838+AbhishekRana95@users.noreply.github.com> Date: Tue, 10 Nov 2020 08:56:14 +0530 Subject: [PATCH] CPD-183: Added pre-check to plugin (#270) Change: minor Purpose: feature --- docs/plugins/rotate_asg_instances.md | 7 +- lib/moonshot/ssh_command_builder.rb | 1 + lib/moonshot/ssh_config.rb | 2 + lib/moonshot/ssh_fork_executor.rb | 10 +- lib/plugins/rotate_asg_instances.rb | 31 +- lib/plugins/rotate_asg_instances/asg.rb | 388 +++++++++--------- lib/plugins/rotate_asg_instances/ssh.rb | 34 ++ .../asg_spec.rb} | 116 +++--- .../plugins/rotate_asg_instances/ssh_spec.rb | 52 +++ .../plugins/rotate_asg_instances_spec.rb | 49 +++ spec/moonshot/ssh_spec.rb | 5 +- 11 files changed, 422 insertions(+), 273 deletions(-) create mode 100644 lib/plugins/rotate_asg_instances/ssh.rb rename spec/moonshot/plugins/{asg.rb => rotate_asg_instances/asg_spec.rb} (75%) create mode 100644 spec/moonshot/plugins/rotate_asg_instances/ssh_spec.rb create mode 100644 spec/moonshot/plugins/rotate_asg_instances_spec.rb diff --git a/docs/plugins/rotate_asg_instances.md b/docs/plugins/rotate_asg_instances.md index dc0b5832..dc9dabe8 100644 --- a/docs/plugins/rotate_asg_instances.md +++ b/docs/plugins/rotate_asg_instances.md @@ -9,12 +9,9 @@ After all outdated instances are shutdown successfully, it terminates them and r It allows gracefully shutting down each instance instead of terminating them and killing all the running processes. ## Configuration -The plugin uses the ssh username specified in the `MOONSHOT_SSH_USER` or the `LOGNAME` environment variable for logging into the ASG instances to shutdown. The value should be the username with which you have the access to the instances. For example: -```ruby -export MOONSHOT_SSH_USER=abhishek.rana -``` +The plugin uses config.ssh_config.ssh_user value for logging into the ASG instances to shutdown. The value should be the username with which you have the access to the instances. -The plugin needs no additional configuration parameters: +The plugin accepts additional configuration from config.ssh_config.ssh_options for ssh. ## Example ```ruby diff --git a/lib/moonshot/ssh_command_builder.rb b/lib/moonshot/ssh_command_builder.rb index 5d8e6b1c..dccca04e 100644 --- a/lib/moonshot/ssh_command_builder.rb +++ b/lib/moonshot/ssh_command_builder.rb @@ -12,6 +12,7 @@ def initialize(ssh_config, instance_id) def build(command = nil) cmd = ['ssh', '-t'] + cmd << @config.ssh_options if @config.ssh_options cmd << "-i #{@config.ssh_identity_file}" if @config.ssh_identity_file cmd << "-l #{@config.ssh_user}" if @config.ssh_user cmd << instance_ip diff --git a/lib/moonshot/ssh_config.rb b/lib/moonshot/ssh_config.rb index 3d77be76..8f590f45 100644 --- a/lib/moonshot/ssh_config.rb +++ b/lib/moonshot/ssh_config.rb @@ -2,10 +2,12 @@ module Moonshot class SSHConfig attr_accessor :ssh_identity_file attr_accessor :ssh_user + attr_accessor :ssh_options def initialize @ssh_identity_file = ENV['MOONSHOT_SSH_KEY_FILE'] @ssh_user = ENV['MOONSHOT_SSH_USER'] + @ssh_options = ENV['MOONSHOT_SSH_OPTIONS'] end end end diff --git a/lib/moonshot/ssh_fork_executor.rb b/lib/moonshot/ssh_fork_executor.rb index 9862d5e1..8fcf4796 100644 --- a/lib/moonshot/ssh_fork_executor.rb +++ b/lib/moonshot/ssh_fork_executor.rb @@ -3,18 +3,18 @@ module Moonshot # Run an SSH command via fork/exec. class SSHForkExecutor - Result = Struct.new(:output, :exitstatus) + Result = Struct.new(:output, :error, :exitstatus) def run(cmd) - output = StringIO.new - + output = error = StringIO.new exit_status = nil - Open3.popen3(cmd) do |_, stdout, _, wt| + Open3.popen3(cmd) do |_, stdout, stderr, wt| + error << stderr.read until stderr.eof? output << stdout.read until stdout.eof? exit_status = wt.value.exitstatus end - Result.new(output.string.chomp, exit_status) + Result.new(output.string.chomp, error.string.chomp, exit_status) end end end diff --git a/lib/plugins/rotate_asg_instances.rb b/lib/plugins/rotate_asg_instances.rb index e923b75a..d8a341c4 100644 --- a/lib/plugins/rotate_asg_instances.rb +++ b/lib/plugins/rotate_asg_instances.rb @@ -1,16 +1,39 @@ # frozen_string_literal: true require 'aws-sdk' -require_relative 'rotate_asg_instances/asg' module Moonshot module Plugins # Rotate ASG instances after update. class RotateAsgInstances + include DoctorHelper + + def doctor(resources) + @resources = resources + run_all_checks + end + + def pre_update(resources) + @resources = resources + asg.verify_ssh + end + def post_update(resources) - asg = ASG.new(resources) - asg.rotate_asg_instances - asg.teardown_outdated_instances + @resources = resources + asg.perform_rotation + end + + private + + def asg + Moonshot::RotateAsgInstances::ASG.new(@resources) + end + + def doctor_check_ssh + asg.verify_ssh + success('Successfully opened SSH connection to an instance.') + rescue Moonshot::RotateAsgInstances::SSHValidationError + critical('SSH connection test failed, check your SSH settings') end end end diff --git a/lib/plugins/rotate_asg_instances/asg.rb b/lib/plugins/rotate_asg_instances/asg.rb index 0263aaaa..aa3efb08 100644 --- a/lib/plugins/rotate_asg_instances/asg.rb +++ b/lib/plugins/rotate_asg_instances/asg.rb @@ -1,238 +1,246 @@ -require 'moonshot/ssh_fork_executor' - module Moonshot - class ASG # rubocop:disable Metrics/ClassLength - include Moonshot::CredsHelper - - def initialize(resources) - @resources = resources - @ilog = @resources.ilog - @ssh_user = ENV['MOONSHOT_SSH_USER'] || ENV['LOGNAME'] - end + module RotateAsgInstances + class ASG # rubocop:disable Metrics/ClassLength + include Moonshot::CredsHelper + + def initialize(resources) + @resources = resources + @ssh = Moonshot::RotateAsgInstances::SSH.new + @ilog = @resources.ilog + end - def asg - @asg ||= - Aws::AutoScaling::AutoScalingGroup.new(name: physical_resource_id) - end + def perform_rotation + rotate_asg_instances + teardown_outdated_instances + end - def rotate_asg_instances - @ilog.start_threaded('Rotating ASG instances...') do |step| - @step = step - @volumes_to_delete = outdated_volumes(outdated_instances) - @shutdown_instances = cycle_instances(outdated_instances) - @step.success('ASG instances rotated successfully!') + def verify_ssh + @ssh.test_ssh_connection(first_instance_id) end - end - def teardown_outdated_instances - @ilog.start_threaded('Tearing down outdated instances...') do |step| - @step = step - terminate_instances(@shutdown_instances) - reap_volumes(@volumes_to_delete) - @step.success('Outdated instances removed successfully!') + private + + def asg + @asg ||= + Aws::AutoScaling::AutoScalingGroup.new(name: physical_resource_id) end - end - def physical_resource_id - @resources.controller.stack - .resources_of_type('AWS::AutoScaling::AutoScalingGroup') - .first.physical_resource_id - end + def first_instance_id + SSHTargetSelector.new( + @resources.controller.stack, + asg_name: Moonshot.config.ssh_auto_scaling_group_name + ).choose! + end - def outdated_instances - @outdated_instances ||= - asg.instances.reject do |i| - i.launch_configuration_name == asg.launch_configuration_name + def rotate_asg_instances + @ilog.start_threaded('Rotating ASG instances...') do |step| + @step = step + outdated = identify_outdated_instances + @volumes_to_delete = outdated_volumes(outdated) + @shutdown_instances = cycle_instances(outdated) + @step.success('ASG instances rotated successfully!') end - end + end - private + def teardown_outdated_instances + @ilog.start_threaded('Tearing down outdated instances...') do |step| + @step = step + terminate_instances(@shutdown_instances) + reap_volumes(@volumes_to_delete) + @step.success('Outdated instances removed successfully!') + end + end - def outdated_volumes(outdated_instances) - volumes = [] - outdated_instances.each do |i| - begin - inst = Aws::EC2::Instance.new(id: i.id) - volumes << inst.block_device_mappings.first.ebs.volume_id - rescue StandardError => e - # We're catching all errors here, because failing to reap a volume - # is not a critical error, will not cause issues with the update. - @step.failure('Failed to get volumes for instance '\ - "#{i.instance_id}: #{e.message}") + def identify_outdated_instances + asg.instances.reject do |i| + i.launch_configuration_name == asg.launch_configuration_name end end - volumes - end - # Cycle the instances in the ASG. - # - # Each instance will be detached one at a time, waiting for the new instance - # to be ready before stopping the worker and terminating the instance. - # - # @param instances [Array] (outdated instances) - # List of instances to cycle. Defaults to all instances with outdated - # launch configurations. - # @return [Array] (array of Aws::AutoScaling::Instance) - # List of shutdown instances. - def cycle_instances(outdated_instances) - shutdown_instances = [] + def physical_resource_id + @resources.controller.stack + .resources_of_type('AWS::AutoScaling::AutoScalingGroup') + .first.physical_resource_id + end - if outdated_instances.empty? - @step.success('No instances cycled!') - return [] + def outdated_volumes(outdated_instances) + volumes = [] + outdated_instances.each do |i| + begin + inst = Aws::EC2::Instance.new(id: i.id) + volumes << inst.block_device_mappings.first.ebs.volume_id + rescue StandardError => e + # We're catching all errors here, because failing to reap a volume + # is not a critical error, will not cause issues with the update. + @step.failure('Failed to get volumes for instance '\ + "#{i.instance_id}: #{e.message}") + end + end + volumes end - @step.success("Cycling #{outdated_instances.size} " \ - "of #{asg.instances.size} instances in " \ - "#{physical_resource_id}...") + # Cycle the instances in the ASG. + # + # Each instance will be detached one at a time, waiting for the new instance + # to be ready before stopping the worker and terminating the instance. + # + # @param instances [Array] (outdated instances) + # List of instances to cycle. Defaults to all instances with outdated + # launch configurations. + # @return [Array] (array of Aws::AutoScaling::Instance) + # List of shutdown instances. + def cycle_instances(outdated_instances) + shutdown_instances = [] + + if outdated_instances.empty? + @step.success('No instances cycled!') + return [] + end - # Iterate over the instances in the stack, detaching and terminating each - # one. - outdated_instances.each do |i| - next if %w(Terminating Terminated).include?(i.lifecycle_state) + @step.success("Cycling #{outdated_instances.size} " \ + "of #{asg.instances.size} instances in " \ + "#{physical_resource_id}...") - wait_for_instance(i) - detach_instance(i) + # Iterate over the instances in the stack, detaching and terminating each + # one. + outdated_instances.each do |i| + next if %w(Terminating Terminated).include?(i.lifecycle_state) - @step.success("Shutting down #{i.instance_id}") - shutdown_instance(i.instance_id) - shutdown_instances << i - end + wait_for_instance(i) + detach_instance(i) - @step.success('All instances cycled.') + @step.success("Shutting down #{i.instance_id}") + shutdown_instance(i.instance_id) + shutdown_instances << i + end - shutdown_instances - end + @step.success('All instances cycled.') - # Waits for an instance to reach a ready state. - # - # @param instance [Aws::AutoScaling::Instance] Auto scaling instance to wait - # for. - def wait_for_instance(instance, state = 'InService') - instance.wait_until(max_attempts: 60, delay: 10) do |i| - i.lifecycle_state == state + shutdown_instances end - end - # Detach an instance from its ASG. Re-attach if failed. - # - # @param instance [Aws::AutoScaling::Instance] Instance to detach. - def detach_instance(instance) - @step.success("Detaching instance: #{instance.instance_id}") - - # If the ASG can't be brought up to capacity, re-attach the instance. - begin - instance.detach(should_decrement_desired_capacity: false) - @step.success('- Waiting for the AutoScaling '\ - 'Group to be up to capacity') - wait_for_capacity - rescue StandardError => e - @step.failure("Error bringing the ASG up to capacity: #{e.message}") - @step.failure("Attaching instance: #{instance.instance_id}") - reattach_instance(instance) - raise e + # Waits for an instance to reach a ready state. + # + # @param instance [Aws::AutoScaling::Instance] Auto scaling instance to wait + # for. + def wait_for_instance(instance, state = 'InService') + instance.wait_until(max_attempts: 60, delay: 10) do |i| + i.lifecycle_state == state + end end - end - # Re-attach an instance to its ASG. - # - # @param instance [Aws::AutoScaling::Instance] Instance to re-attach. - def reattach_instance(instance) - instance.load - return unless instance.data.nil? \ - || %w(Detached Detaching).include?(instance.lifecycle_state) + # Detach an instance from its ASG. Re-attach if failed. + # + # @param instance [Aws::AutoScaling::Instance] Instance to detach. + def detach_instance(instance) + @step.success("Detaching instance: #{instance.instance_id}") - until instance.data.nil? || instance.lifecycle_state == 'Detached' - sleep 10 - instance.load + # If the ASG can't be brought up to capacity, re-attach the instance. + begin + instance.detach(should_decrement_desired_capacity: false) + @step.success('- Waiting for the AutoScaling '\ + 'Group to be up to capacity') + wait_for_capacity + rescue StandardError => e + @step.failure("Error bringing the ASG up to capacity: #{e.message}") + @step.failure("Attaching instance: #{instance.instance_id}") + reattach_instance(instance) + raise e + end end - instance.attach - end - # Terminate instances. - # - # @param instances [Array] (instances for termination) - # List of instances to terminate. Defaults to all instances with outdated - # launch configurations. - def terminate_instances(outdated_instances) - if outdated_instances.any? - @step.continue( - "Terminating #{outdated_instances.size} outdated instances..." - ) - end - outdated_instances.each do |asg_instance| - instance = Aws::EC2::Instance.new(asg_instance.instance_id) - begin + # Re-attach an instance to its ASG. + # + # @param instance [Aws::AutoScaling::Instance] Instance to re-attach. + def reattach_instance(instance) + instance.load + return unless instance.data.nil? \ + || %w(Detached Detaching).include?(instance.lifecycle_state) + + until instance.data.nil? || instance.lifecycle_state == 'Detached' + sleep 10 instance.load - rescue Aws::EC2::Errors::InvalidInstanceIDNotFound - next end + instance.attach + end - next unless %w(stopping stopped).include?(instance.state.name) + # Terminate instances. + # + # @param instances [Array] (instances for termination) + # List of instances to terminate. Defaults to all instances with outdated + # launch configurations. + def terminate_instances(shutdown_instances) + if shutdown_instances.any? + @step.continue( + "Terminating #{shutdown_instances.size} outdated instances..." + ) + end + shutdown_instances.each do |asg_instance| + instance = Aws::EC2::Instance.new(asg_instance.instance_id) + begin + instance.load + rescue Aws::EC2::Errors::InvalidInstanceIDNotFound + next + end - instance.wait_until_stopped + next unless %w(stopping stopped).include?(instance.state.name) - @step.continue("Terminating #{instance.instance_id}") - instance.terminate - end - end + instance.wait_until_stopped - def reap_volumes(volumes) - volumes.each do |volume_id| - begin - @step.continue("Deleting volume: #{volume_id}") - ec2_client(region: ENV['AWS_REGION']) - .delete_volume(volume_id: volume_id) - rescue StandardError => e - # We're catching all errors here, because failing to reap a volume - # is not a critical error, will not cause issues with the release. - @step.failure("Failed to delete volume #{volume_id}: #{e.message}") + @step.continue("Terminating #{instance.instance_id}") + instance.terminate end end - end - # Waits for the ASG to reach the desired capacity. - def wait_for_capacity - @step.continue( - 'Replacing outdated instances with new instances for the AutoScaling Group...' - ) - # While we wait for the asg to reach capacity, report instance statuses - # to the user. - before_wait = proc do - instances = [] - asg.reload.instances.each do |i| - instances << " #{i.instance_id} (#{i.lifecycle_state})" + def reap_volumes(volumes) + volumes.each do |volume_id| + begin + @step.continue("Deleting volume: #{volume_id}") + ec2_client(region: ENV['AWS_REGION']) + .delete_volume(volume_id: volume_id) + rescue StandardError => e + # We're catching all errors here, because failing to reap a volume + # is not a critical error, will not cause issues with the release. + @step.failure("Failed to delete volume #{volume_id}: #{e.message}") + end end - - @step.continue("Instances: #{instances.join(', ')}") end - asg.reload.wait_until(before_wait: before_wait, max_attempts: 60, - delay: 30) do |a| - instances_up = a.instances.select do |i| - i.lifecycle_state == 'InService' + # Waits for the ASG to reach the desired capacity. + def wait_for_capacity + @step.continue( + 'Replacing outdated instances with new instances for the AutoScaling Group...' + ) + # While we wait for the asg to reach capacity, report instance statuses + # to the user. + before_wait = proc do + instances = [] + asg.reload.instances.each do |i| + instances << " #{i.instance_id} (#{i.lifecycle_state})" + end + + @step.continue("Instances: #{instances.join(', ')}") + end + + asg.reload.wait_until(before_wait: before_wait, max_attempts: 60, + delay: 30) do |a| + instances_up = a.instances.select do |i| + i.lifecycle_state == 'InService' + end + instances_up.length == a.desired_capacity end - instances_up.length == a.desired_capacity + @step.success('AutoScaling Group up to capacity!') end - @step.success('AutoScaling Group up to capacity!') - end - # Shuts down an instance, waiting for the instance to stop processing requests - # first. We do this so that services will be stopped properly. - # - # @param id [String] ID of the instance to terminate. - def shutdown_instance(id) - instance = Aws::EC2::Instance.new(id: id) - options = [ - 'UserKnownHostsFile=/dev/null', - 'StrictHostKeyChecking=no' - ] - remote = "#{@ssh_user}@#{instance.public_dns_name}" - cmd = "'sudo shutdown -h now'" - remote_cmd = "ssh -o #{options.join(' -o ')} #{remote} #{cmd}" - SSHForkExecutor.new.run(remote_cmd) - - instance.wait_until_stopped + # Shuts down an instance, waiting for the instance to stop processing requests + # first. We do this so that services will be stopped properly. + # + # @param id [String] ID of the instance to terminate. + def shutdown_instance(id) + instance = Aws::EC2::Instance.new(id: id) + @ssh.exec('sudo shutdown -h now', id) + instance.wait_until_stopped + end end end end diff --git a/lib/plugins/rotate_asg_instances/ssh.rb b/lib/plugins/rotate_asg_instances/ssh.rb new file mode 100644 index 00000000..e23d079c --- /dev/null +++ b/lib/plugins/rotate_asg_instances/ssh.rb @@ -0,0 +1,34 @@ +module Moonshot + module RotateAsgInstances + class SSHValidationError < StandardError + def initialize(response) + super("SSH failed. exit status: #{response.exitstatus} " \ + "output: #{response.output} error: #{response.error}") + end + end + + class SSH + # As per the standard it is raising correctly but still giving an error. + def test_ssh_connection(instance_id) + Retriable.retriable(base_interval: 5, tries: 3) do + response = exec('/bin/true', instance_id) + # rubocop:disable Style/RaiseArgs + raise SSHValidationError.new(response) unless + response.exitstatus.zero? + end + end + + def exec(command, instance_id) + fe = SSHForkExecutor.new + fe.run(build_command(command, instance_id)) + end + + private + + def build_command(command, instance_id) + cb = SSHCommandBuilder.new(Moonshot.config.ssh_config, instance_id) + cb.build(command).cmd + end + end + end +end diff --git a/spec/moonshot/plugins/asg.rb b/spec/moonshot/plugins/rotate_asg_instances/asg_spec.rb similarity index 75% rename from spec/moonshot/plugins/asg.rb rename to spec/moonshot/plugins/rotate_asg_instances/asg_spec.rb index 0ab532ee..73429981 100644 --- a/spec/moonshot/plugins/asg.rb +++ b/spec/moonshot/plugins/rotate_asg_instances/asg_spec.rb @@ -1,21 +1,31 @@ -describe Moonshot::ASG do +describe Moonshot::RotateAsgInstances::ASG do let(:name) { 'cdb-worker-dev-jarmes-WorkerASG-Q7DYM7901RBY' } let(:instance_id) { 'i-585e91dc' } let(:instance_id_2) { 'i-685e91dc' } - let(:system) { instance_double(System) } + let(:ssh_executor) { Moonshot::SSHForkExecutor } + let(:moonshot_config) { Moonshot.config } + let(:controller) do + instance_double('Moonshot::Controller', + config: moonshot_config, + stack: instance_double( + Moonshot::Stack, + name: 'test_name', + parameters: {} + ) + ) + end let(:resources) do instance_double( Moonshot::Resources, - stack: instance_double( - Moonshot::Stack, - name: 'test_name', - parameters: {} - ), ilog: instance_double(Moonshot::InteractiveLoggerProxy), - controller: instance_double( - Moonshot::Controller, - config: instance_double(Moonshot::ControllerConfig, app_name: 'test') - ) + controller: controller + ) + end + + let(:asg) do + instance_double( + Aws::AutoScaling::AutoScalingGroup, + name: 'asg' ) end @@ -38,11 +48,7 @@ ) end - before(:each) do - allow(System).to receive(:new) - .and_return(system) - stub_cf_client - end + before(:each) { stub_cf_client } def stub_cf_client @cf_client = instance_double(Aws::CloudFormation::Client) @@ -53,9 +59,7 @@ def stub_cf_client allow(@cf_client).to receive(:validate_template).and_return(true) end - subject do - described_class.new(resources) - end + subject { described_class.new(resources) } describe '#cycle_instances' do before(:each) do @@ -108,66 +112,42 @@ def stub_cf_client end end - describe '#initialize' do - before(:each) do - allow(System).to receive(:new) - .and_return(system) - stub_cf_client - end - context 'when MOONSHOT_SSH_USER is not defined' do - it 'uses LOGNAME for ssh_user' do - ENV['LOGNAME'] = 'SemiCoolDude' - ENV.delete('MOONSHOT_SSH_USER') - expect(subject.instance_variable_get(:@ssh_user)) - .to eq('SemiCoolDude') - end - end - - context 'when MOONSHOT_SSH_USER is defined' do - it 'uses MOONSHOT_SSH_USER for ssh_user' do - ENV['MOONSHOT_SSH_USER'] = 'CoolDude' - expect(subject.instance_variable_get(:@ssh_user)) - .to eq('CoolDude') - end - end - end - describe '#shutdown_instance' do - let(:hostname) { 'ec2-54-236-102-14.compute-1.amazonaws.com' } + let(:public_ip_address) { '10.234.32.21' } let(:instance) { instance_double(Aws::EC2::Instance) } + let(:command_builder) { Moonshot::SSHCommandBuilder } subject { super().send(:shutdown_instance, instance_id) } before(:each) do + moonshot_config.ssh_config.ssh_user = 'ci_user' + moonshot_config.ssh_config.ssh_options = ssh_options allow(Aws::EC2::Instance).to receive(:new).and_return(instance) - allow(instance).to receive(:public_dns_name) \ - .and_return(hostname) - allow(System).to receive(:exec) + allow_any_instance_of(command_builder).to receive(:instance_ip).and_return(public_ip_address) + allow(instance).to receive(:wait_until_stopped) end - it 'looks up the DNS name of the host' do - expect(instance).to receive(:public_dns_name) \ - .and_return(hostname) - subject - end - it 'issues a shutdown to the instance' do - ENV['MOONSHOT_SSH_USER'] = 'ci_user' - expect(System).to receive(:exec).with( - /ssh (.*) ci_user@#{hostname} 'sudo shutdown -h now'/, - raise_on_failure: false - ) - subject + context 'when ssh_options are not defined' do + let(:ssh_options) { nil } + + it 'issues a shutdown without options to the instance' do + expect_any_instance_of(ssh_executor).to receive(:run).with( + "ssh -t -l #{moonshot_config.ssh_config.ssh_user} #{public_ip_address} sudo\\ shutdown\\ -h\\ now" + ) + subject + end end - it 'runs SSH with proper option to ignore host keys' do - ENV['MOONSHOT_SSH_USER'] = 'ci_user' - opts_string = '-o UserKnownHostsFile=/dev/null ' \ - '-o StrictHostKeyChecking=no' - expect(System).to receive(:exec).with( - /#{opts_string}/, - any_args - ) - subject + context 'when ssh_options are defined' do + let(:ssh_options) { '-v -o UserKnownHostsFile=/dev/null' } + + it 'issues a shutdown with options to the instance' do + expect_any_instance_of(ssh_executor).to receive(:run).with( + 'ssh -t -v -o UserKnownHostsFile=/dev/null ' \ + "-l ci_user #{public_ip_address} sudo\\ shutdown\\ -h\\ now" + ) + subject + end end end diff --git a/spec/moonshot/plugins/rotate_asg_instances/ssh_spec.rb b/spec/moonshot/plugins/rotate_asg_instances/ssh_spec.rb new file mode 100644 index 00000000..f6f897ee --- /dev/null +++ b/spec/moonshot/plugins/rotate_asg_instances/ssh_spec.rb @@ -0,0 +1,52 @@ +describe Moonshot::RotateAsgInstances::SSH do + let(:instance_id) { 'i-585e91dc' } + let(:command) { '/bin/true' } + let(:result) { Struct.new(:output, :error, :exitstatus) } + let(:successful_response) { result.new('Output', 'No Failure', 0) } + let(:validation_error) do + Moonshot::RotateAsgInstances::SSHValidationError.new( + result.new('Output', 'Failure', 255) + ) + end + let(:moonshot_config) { Moonshot::ControllerConfig.new } + let(:controller) do + instance_double('Moonshot::Controller', + config: moonshot_config, + stack: instance_double( + Moonshot::Stack, + name: 'test_name', + parameters: {} + ) + ) + end + let(:resources) do + instance_double( + Moonshot::Resources, + ilog: instance_double(Moonshot::InteractiveLoggerProxy), + controller: controller + ) + end + + let(:config) { resources.controller.config } + + subject { described_class.new } + + describe '#test_ssh_connection' do + it 'raise error if #test_ssh_connection fails' do + allow(subject).to receive(:test_ssh_connection).with(instance_id).and_raise(validation_error) + expect { subject.test_ssh_connection(instance_id) }.to raise_error(validation_error) + end + + it 'does not raise error if ssh is successful' do + allow(subject).to receive(:test_ssh_connection).with(instance_id).and_return(successful_response) + expect { subject.test_ssh_connection(instance_id) }.not_to raise_error + end + end + + describe '#exec' do + it 'executes the command given' do + allow(subject).to receive(:exec).with(command, instance_id).and_return(successful_response) + expect(subject.exec(command, instance_id)).to eql(successful_response) + end + end +end diff --git a/spec/moonshot/plugins/rotate_asg_instances_spec.rb b/spec/moonshot/plugins/rotate_asg_instances_spec.rb new file mode 100644 index 00000000..91733a0f --- /dev/null +++ b/spec/moonshot/plugins/rotate_asg_instances_spec.rb @@ -0,0 +1,49 @@ +describe Moonshot::Plugins::RotateAsgInstances do + let(:instance_id) { 'i-585e91dc' } + let(:result) { Struct.new(:output, :error, :exitstatus) } + let(:validation_error) do + Moonshot::RotateAsgInstances::SSHValidationError.new( + result.new('Output', 'Failure', 255) + ) + end + let(:successful_response) { result.new('Output', 'No Failure', 0) } + let(:moonshot_config) { Moonshot::ControllerConfig.new } + let(:controller) do + instance_double('Moonshot::Controller', + config: moonshot_config, + stack: instance_double( + Moonshot::Stack, + name: 'test_name', + parameters: {} + ) + ) + end + let(:resources) do + instance_double( + Moonshot::Resources, + ilog: instance_double(Moonshot::InteractiveLoggerProxy), + controller: controller + ) + end + + subject { described_class.new } + + let(:ssh) { Moonshot::RotateAsgInstances::SSH } + + before(:each) do + subject.instance_variable_set(:@resources, resources) + expect_any_instance_of(Moonshot::SSHTargetSelector).to receive(:choose!).and_return(instance_id) + end + + describe '#doctor' do + it 'raises error if check is not passed' do + allow_any_instance_of(ssh).to receive(:test_ssh_connection).with(instance_id).and_raise(validation_error) + expect{ subject.send(:doctor_check_ssh) }.to raise_error(Moonshot::DoctorCritical) + end + + it 'does not raise error when check is passed' do + allow_any_instance_of(ssh).to receive(:test_ssh_connection).with(instance_id).and_return(successful_response) + expect{ subject.send(:doctor_check_ssh) }.not_to raise_error + end + end +end diff --git a/spec/moonshot/ssh_spec.rb b/spec/moonshot/ssh_spec.rb index 7be8f990..56423c3d 100644 --- a/spec/moonshot/ssh_spec.rb +++ b/spec/moonshot/ssh_spec.rb @@ -6,11 +6,14 @@ c.ssh_config.ssh_user = 'joeuser' c.ssh_config.ssh_identity_file = '/Users/joeuser/.ssh/thegoods.key' c.ssh_command = 'cat /etc/passwd' - Moonshot::Controller.new(c) end describe 'Moonshot::Controller#ssh' do + before(:each) do + ENV.delete('MOONSHOT_SSH_OPTIONS') + end + context 'normally' do it 'should execute an ssh command with proper parameters' do ts = instance_double(Moonshot::SSHTargetSelector)