From 2413a55a250ee63514c92bf107679f4e38bcc8d4 Mon Sep 17 00:00:00 2001 From: Kyle Smith Date: Wed, 9 Mar 2016 16:17:58 -0500 Subject: [PATCH] Initial Commit. --- .gitignore | 4 + .rspec | 3 + .rubocop.yml | 13 + .ruby-version | 1 + .travis.yml | 21 ++ Gemfile | 10 + LICENSE.txt | 201 +++++++++++ README.md | 14 + Rakefile | 13 + docs/contribute.md | 15 + docs/index.md | 193 ++++++++++ docs/parent_stacks.md | 24 ++ docs/plugins.md | 47 +++ lib/moonshot.rb | 41 +++ lib/moonshot/artifact_repository/s3_bucket.rb | 60 ++++ .../s3_bucket_via_github_releases.rb | 89 +++++ .../build_mechanism/github_release.rb | 148 ++++++++ lib/moonshot/build_mechanism/script.rb | 84 +++++ lib/moonshot/build_mechanism/travis_deploy.rb | 70 ++++ lib/moonshot/build_mechanism/version_proxy.rb | 55 +++ lib/moonshot/cli.rb | 145 ++++++++ lib/moonshot/controller.rb | 143 ++++++++ lib/moonshot/controller_config.rb | 25 ++ lib/moonshot/creds_helper.rb | 28 ++ .../deployment_mechanism/code_deploy.rb | 303 ++++++++++++++++ lib/moonshot/doctor_helper.rb | 52 +++ lib/moonshot/environment_parser.rb | 32 ++ lib/moonshot/interactive_logger_proxy.rb | 49 +++ lib/moonshot/resources.rb | 13 + lib/moonshot/resources_helper.rb | 24 ++ lib/moonshot/shell.rb | 52 +++ lib/moonshot/stack.rb | 339 ++++++++++++++++++ lib/moonshot/stack_asg_printer.rb | 151 ++++++++ lib/moonshot/stack_config.rb | 12 + lib/moonshot/stack_events_poller.rb | 56 +++ lib/moonshot/stack_lister.rb | 20 ++ lib/moonshot/stack_output_printer.rb | 16 + lib/moonshot/stack_parameter_printer.rb | 73 ++++ lib/moonshot/stack_template.rb | 35 ++ lib/moonshot/unicode_table.rb | 63 ++++ mkdocs.yml | 4 + moonshot.gemspec | 27 ++ spec/.rspec | 1 + .../cloud_formation/rspec-app.json | 12 + .../build_mechanism/github_release_spec.rb | 76 ++++ .../build_mechanism/travis_deploy_spec.rb | 49 +++ spec/moonshot/cli_spec.rb | 73 ++++ .../moonshot/interactive_logger_proxy_spec.rb | 30 ++ spec/moonshot/plugins_spec.rb | 71 ++++ spec/moonshot/shell_spec.rb | 68 ++++ spec/moonshot/stack_spec.rb | 142 ++++++++ spec/spec_helper.rb | 27 ++ 52 files changed, 3317 insertions(+) create mode 100644 .gitignore create mode 100644 .rspec create mode 100644 .rubocop.yml create mode 100644 .ruby-version create mode 100644 .travis.yml create mode 100644 Gemfile create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100644 Rakefile create mode 100644 docs/contribute.md create mode 100644 docs/index.md create mode 100644 docs/parent_stacks.md create mode 100644 docs/plugins.md create mode 100644 lib/moonshot.rb create mode 100644 lib/moonshot/artifact_repository/s3_bucket.rb create mode 100644 lib/moonshot/artifact_repository/s3_bucket_via_github_releases.rb create mode 100644 lib/moonshot/build_mechanism/github_release.rb create mode 100644 lib/moonshot/build_mechanism/script.rb create mode 100644 lib/moonshot/build_mechanism/travis_deploy.rb create mode 100644 lib/moonshot/build_mechanism/version_proxy.rb create mode 100644 lib/moonshot/cli.rb create mode 100644 lib/moonshot/controller.rb create mode 100644 lib/moonshot/controller_config.rb create mode 100644 lib/moonshot/creds_helper.rb create mode 100644 lib/moonshot/deployment_mechanism/code_deploy.rb create mode 100644 lib/moonshot/doctor_helper.rb create mode 100644 lib/moonshot/environment_parser.rb create mode 100644 lib/moonshot/interactive_logger_proxy.rb create mode 100644 lib/moonshot/resources.rb create mode 100644 lib/moonshot/resources_helper.rb create mode 100644 lib/moonshot/shell.rb create mode 100644 lib/moonshot/stack.rb create mode 100644 lib/moonshot/stack_asg_printer.rb create mode 100644 lib/moonshot/stack_config.rb create mode 100644 lib/moonshot/stack_events_poller.rb create mode 100644 lib/moonshot/stack_lister.rb create mode 100644 lib/moonshot/stack_output_printer.rb create mode 100644 lib/moonshot/stack_parameter_printer.rb create mode 100644 lib/moonshot/stack_template.rb create mode 100644 lib/moonshot/unicode_table.rb create mode 100644 mkdocs.yml create mode 100644 moonshot.gemspec create mode 100644 spec/.rspec create mode 100644 spec/fs_fixtures/cloud_formation/rspec-app.json create mode 100644 spec/moonshot/build_mechanism/github_release_spec.rb create mode 100644 spec/moonshot/build_mechanism/travis_deploy_spec.rb create mode 100644 spec/moonshot/cli_spec.rb create mode 100644 spec/moonshot/interactive_logger_proxy_spec.rb create mode 100644 spec/moonshot/plugins_spec.rb create mode 100644 spec/moonshot/shell_spec.rb create mode 100644 spec/moonshot/stack_spec.rb create mode 100644 spec/spec_helper.rb diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..dd048e74 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/Gemfile.lock +/vendor +/coverage +site/ \ No newline at end of file diff --git a/.rspec b/.rspec new file mode 100644 index 00000000..c3434ad5 --- /dev/null +++ b/.rspec @@ -0,0 +1,3 @@ +--require spec_helper.rb +--color +--format documentation diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 00000000..4c949962 --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,13 @@ +--- +AllCops: + TargetRubyVersion: 2.2 + Exclude: + - '*.gemspec' +Metrics/AbcSize: + Max: 20 +Metrics/MethodLength: + Max: 30 +Metrics/LineLength: + Max: 100 +Style/ClassAndModuleChildren: + Enabled: false diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 00000000..6b4d1577 --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +2.2.3 \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..62041edf --- /dev/null +++ b/.travis.yml @@ -0,0 +1,21 @@ +--- +language: ruby + +cache: bundler +# This will enable the container-based infrastructure for +# travis tests. It should result in faster test vm load times. +# @see http://docs.travis-ci.com/user/workers/container-based-infrastructure/ +sudo: false + +rvm: + - 2.1 + - 2.2 + +install: + - bundle install + +script: + - rake + +env: + - AWS_REGION=us-east-1 diff --git a/Gemfile b/Gemfile new file mode 100644 index 00000000..a5a3a029 --- /dev/null +++ b/Gemfile @@ -0,0 +1,10 @@ +source 'https://rubygems.org' + +gemspec + +gem 'rake', require: false + +group :test do + gem 'rubocop', '~> 0.38.0' + gem 'pry' +end diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 00000000..628b7d1d --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016 Acquia, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 00000000..1689bd29 --- /dev/null +++ b/README.md @@ -0,0 +1,14 @@ +# Moonshot +_Because releasing services shouldn't be a moonshot._ + +Moonshot is a tool for provisioning infrastructure and applications in AWS with CloudFormation and CodeDeploy using a CLI. Its main goal is to make it possible to control the deployment in a programmable and extensible way so that there is less room for human errors in the AWS console when creating and updating cloudformation templates but also deploying new software using CodeDeploy. + +The software is relying on a single CloudFormation stack and supported by pluggable systems: + +- A DeploymentMechanism controls releasing code. +- A BuildMechanism creates a release artifact. +- A ArtifactRepository stores the release artifacts. + +Read the documentation at __insert_link_here_to_read_the_docs__ or see the docs folder. + +Discussions and support: Please see the issues in the current repository \ No newline at end of file diff --git a/Rakefile b/Rakefile new file mode 100644 index 00000000..79c9da1e --- /dev/null +++ b/Rakefile @@ -0,0 +1,13 @@ +require 'bundler/setup' +require 'rspec/core/rake_task' +require 'rubocop/rake_task' + +desc 'Run RuboCop against the source code.' +RuboCop::RakeTask.new(:rubocop) do |task| + task.options << '--display-cop-names' + task.options << '--display-style-guide' +end + +RSpec::Core::RakeTask.new(:spec) + +task default: [:spec, :rubocop] diff --git a/docs/contribute.md b/docs/contribute.md new file mode 100644 index 00000000..19d318ba --- /dev/null +++ b/docs/contribute.md @@ -0,0 +1,15 @@ +# How to Contribute + +So, you want to help? Awesome! In order to guide this the best way we can and to make sure we can help you in either getting a bug fixed or improve our documentation, we are giving you an issue template that should be used. With this issue template, we can make sure we have all the information needed to move on. + +## Expectations +Even though this is an Open project, it does not mean that we have 24/7 support for it. The best way to make sure that we accept your bugfix or add in a feature you want is still to make a pull request and link it in the issue. Include some reasons how to reproduce or even better, make sure the bug is tested and passes highy increases your chances for the change to get committed. + +So, you've done all that? It very likely that it could get committed really quickly but since some of the contributers to this project could be on a holiday or we are in no way responsible in actually merging it in soon. But have no fear! Since you filed a pull request from your fork, you are able to use a version of moonshot yourselves. And we promise you, we will do our utter best! + +## Issue template +Here comes how to contribute including an issue template that can be used. + +## Issue Example + +And again, thanks! We're looking forward working with you. \ No newline at end of file diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..40849e10 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,193 @@ +# Moonshot +_Because releasing services shouldn't be a moonshot._ + +Moonshot is a Ruby gem for provisioning environments in AWS using a CLI. +The environments are centered around a single CloudFormation stack and supported +by pluggable systems: +- A DeploymentMechanism controls releasing code. +- A BuildMechanism creates a release artifact. +- A ArtifactRepository stores the release artifacts. + +Supported DeploymentMechanisms: +- CodeDeploy + +Supported ArtifactRepositories: +- S3 + +# Design Goals + +These are core ideas to the creation of this project. Not all are met to the +level we'd like (e.g. CloudFormation isn't much of a Choice currently), but we +should aspire to meet them with each iteration. + +- Simplicity: It shouldn't take more than a few hours to understand what your + release tooling does. +- Choice: As much as possible, each component should be pluggable and omittable, + so teams are free to use what works best for them. +- Verbosity: The output of core Moonshot code should explain in detail what + changes are being made, so knowledge is shared and not abstracted. + +# Configuring an AWS account + +## CodeDeploy Role + +Create a role called CodeDeployRole with the AWSCodeDeployRole policy + +``` +aws iam create-role --role-name CodeDeployRole --assume-role-policy-document '{"Version":"2012-10-17","Statement":[{"Sid":"","Effect":"Allow","Principal":{"Service":["codedeploy.amazonaws.com"]},"Action":"sts:AssumeRole"}]}' +aws iam attach-role-policy --role-name CodeDeployRole --policy-arn arn:aws:iam::aws:policy/service-role/AWSCodeDeployRole +``` + +# Basic Usage + +The base class is a subclass of Thor, so you can extend it using all the normal +Thor stuff. Here's a really basic example using all the defaults: + +```ruby +#!/usr/bin/env ruby + +require 'moonshot' + +# Set up Moonshot tooling for our environment. +class MyService < Moonshot::CLI + self.application_name = 'my-service' + self.artifact_repository = S3Bucket.new('my-service-builds') + self.build_mechanism = Script.new('build/script.sh') + self.deployment_mechanism = CodeDeploy.new(asg: 'AutoScalingGroup') + + desc 'my-custom-function' + def my_custom_function + puts "<:3)~~ eek! a mouse!" + end +end + +begin + MyService.start +rescue => e + warn "Uncaught exception: #{e.class}: #{e.message}" + warn "at: #{e.backtrace.first}" + exit(1) +end +``` + +This example assumes: +- You have a CloudFormation JSON template in "cloud_formation/my-service.json". +- You have an S3 bucket called "my-service-builds". +- You have a script in "script/build.sh" that will build a tarball output.tar.gz. +- You have a working CodeDeploy setup, including the CodeDeployRole. +- You have some need to display an ASCII mouse to the terminal with your release + tooling. + +If all that is true, you can now deploy your software to a new stack with: +``` +$ ./bin/environment create +``` + +By default, you'll get a development environment named `my-service-dev-giraffe`, +where `giraffe` is your username. If you want to provision test or production +named environment, use: +``` +$ ./bin/environment create -n my-service-staging +$ ./bin/environment create -n my-service-production +``` + +By default, create launches the stack and deploys code. If you want to only +create the stack and not deploy code, use: +``` +$ ./bin/environment create --no-deploy +``` + +If you make changes to your application and want to release a development build +to your stack, run: +``` +$ ./bin/environment deploy-code +``` + +To build a "named build" for releasing through test and production environments, +use: +``` +$ ./bin/environment build-version v0.1.0 +$ ./bin/environment deploy-version v0.1.0 -n +``` + +We recommend using a CI system like Jenkins to perform those activities, for +consistency. + +# Including Moonshot-based tooling in your project + +You'll want to build a tool using the template in Basic Usage as a starting point, +then grab a **release build**, either by using git directly from Bundler, or +including the gem directly in your `vendor/cache` directory. + +# Stack Parameter overrides + +One of the challenges we faced using CloudFormation was being consistent about +setting stack parameters as you make requests. We settled on a strategy that +keeps all per-environment tunings in the source repository acting as both a +safety net and documentation of existing environments. Here's how it works: + +When a stack update is performed, a *parameter overrides file* is checked in +`cloud_formation/parameters/environment-name.yml`. This file is YAML formatted +and takes a hash of stack parameter names and values, for example: +```yaml +--- +AsgDesiredCap: 12 +AsgMaxCap: 15 +ELBCertificate: iam::something:star_example_com +``` + +If a file exists, it's used every time a CloudFormation change request is sent, +so no configuration can revert back to defaults through this tool. It's highly +recommended that you add these files back to source control as soon as possible +and be in the habit of pulling latest changes before applying any infrastructure +updates. + +# Built-In Mechanisms + +## BuildMechanism + +### Script + +The Script BuildMechanism will execute a local shell script, with certain +expectations. The script will run with some environment variables: + +- `VERSION`: The named version string passed to `build-version`. +- `OUTPUT_FILE`: The file that the script is expected to produce. + +If the file is not created by the build script, deployment will fail. Otherwise, +the output file will be uploaded using the ArtifactRepository. + +## DeploymentMechanism + +### CodeDeploy + +The CodeDeploy DeploymentMechanism will create a CodeDeploy Application and +Deployment Group matching the application name. The created Deployment Group +will point at the logical resource id provided to the constructor (e.g. +`CodeDeploy.new(asg: 'MyAutoScalingGroup')`). During the `deploy-code` action, +the ArtifactRepository is checked for compatibility with CodeDeploy. Currently +only the S3Bucket is supported, though CodeDeploy itself supports deploying from +a git source. + +Assumptions made by the CodeDeploy mechanism: +- You are using an S3Bucket ArtifactRepository. +- You want to deploy using the OneAtATime method. +- Your build artifact contains an appspec.yml file. + +For more information about CodeDeploy, see the [AWS Documentation][1]. + +[1]: http://docs.aws.amazon.com/codedeploy/latest/userguide/welcome.html + +## ArtifactRepository + +### S3Bucket + +To create a new S3Bucket ArtifactRepository: +```ruby +class MyApplication < Moonshot::CLI + self.artifact_repository = S3Bucket.new('my-bucket-name') +end +``` + +The store action will simply upload the file using the S3 PutObject API call. +The local environment must be configured with appropriate credentials. diff --git a/docs/parent_stacks.md b/docs/parent_stacks.md new file mode 100644 index 00000000..eacfbe6d --- /dev/null +++ b/docs/parent_stacks.md @@ -0,0 +1,24 @@ +## Parent Stacks + +Since 0.5.3, Moonshot supports referencing another CloudFormation stack as a +"Parent" during creation time. This relationship is used only for creation, +where any outputs of that stack that match names of the parameters for the local +stack will be used as parameters, and saved into a local .yml file for future +use. + +The order of precedence for parameters is: +- Existing parameter overrides in the .yml file. +- The value from the parent stack's output. +- Any default value in the CloudFormation template. + +### A word of caution + +It's not advisable to use default values in the CloudFormation template. +Consider the following example: + +- Developer A launches a stack, referencing a specific parent stack. +- Values are copied into a local .yml file, and used during stack creation. +- Developer B assists Developer A with a stack update issue and runs 'update' + without the local overrides .yml file. In doing so, the default values in the + template are used. +- Developer A is sad. diff --git a/docs/plugins.md b/docs/plugins.md new file mode 100644 index 00000000..a3f02db7 --- /dev/null +++ b/docs/plugins.md @@ -0,0 +1,47 @@ +# Plugin Support + +**Warning, the plugin support in Moonshot is a work-in-progress. The interface +to plugins may change dramatically in future versions.** + +Moonshot supports adding plugins (implemented as a Ruby class) to the controller +that can perform actions before and after the `create`, `update`, `delete`, +`status` and `doctor` actions. + +## Writing a Moonshot Plugin + +A Moonshot Plugin is a Ruby class that responds to one or more of the following +methods: + +- pre_create +- post_create +- pre_update +- post_update +- pre_delete +- post_delete +- pre_status +- post_status +- pre_doctor +- post_doctor + +The method will be handed a single argument, which is an instance of the +`Moonshot::Resources` class. This instance gives the plugin access to three +important resources: + +- `Moonshot::Resources#ilog` is an instance of `InteractiveLogger`, used to +display status to the user of the CLI interface. +- `Moonshot::Resources#stack` is an instance of `Moonshot::Stack` which can +retreive the name of the stack, stack parameters and stack outputs. This support +should be expanded in the future to provide Plugins with more control over the +CloudFormation stack. + +## Adding a plugin to a CLI tool. + +Once you have defined or included your plugin class, you can add a plugin like so: + +```ruby +class MyApp < Moonshot::CLI + self.application_name = 'my-app' + # ... + plugin MyPlugin.new +end +``` diff --git a/lib/moonshot.rb b/lib/moonshot.rb new file mode 100644 index 00000000..2661a5e0 --- /dev/null +++ b/lib/moonshot.rb @@ -0,0 +1,41 @@ +require 'English' +require 'aws-sdk' +require 'logger' +require 'thor' + +module Moonshot + module ArtifactRepository # rubocop:disable Documentation + end + module BuildMechanism # rubocop:disable Documentation + end + module DeploymentMechanism # rubocop:disable Documentation + end +end + +[ + # Helpers + 'creds_helper', + 'doctor_helper', + 'resources', + 'resources_helper', + 'environment_parser', + + # Core + 'interactive_logger_proxy', + 'controller', + 'controller_config', + 'cli', + 'stack', + 'stack_config', + 'stack_lister', + 'stack_events_poller', + + # Built-in mechanisms + 'artifact_repository/s3_bucket', + 'artifact_repository/s3_bucket_via_github_releases', + 'build_mechanism/script', + 'build_mechanism/github_release', + 'build_mechanism/travis_deploy', + 'build_mechanism/version_proxy', + 'deployment_mechanism/code_deploy' +].each { |f| require_relative "moonshot/#{f}" } diff --git a/lib/moonshot/artifact_repository/s3_bucket.rb b/lib/moonshot/artifact_repository/s3_bucket.rb new file mode 100644 index 00000000..c232ba95 --- /dev/null +++ b/lib/moonshot/artifact_repository/s3_bucket.rb @@ -0,0 +1,60 @@ +# The S3Bucket stores builds in an S3 Bucket. +# +# For example: +# +# def MyApplication < Moonshot::CLI +# self.artifact_repository = S3Bucket.new('my-application-builds') +# end +class Moonshot::ArtifactRepository::S3Bucket + include Moonshot::ResourcesHelper + include Moonshot::CredsHelper + include Moonshot::DoctorHelper + + attr_reader :bucket_name + + def initialize(bucket_name) + @bucket_name = bucket_name + end + + def store_hook(build_mechanism, version_name) + unless build_mechanism.respond_to?(:output_file) + raise "S3Bucket does not know how to store artifacts from #{build_mechanism.class}, no method '#output_file'." # rubocop:disable LineLength + end + + file = build_mechanism.output_file + bucket_name = @bucket_name + key = filename_for_version(version_name) + + ilog.start_threaded "Uploading #{file} to s3://#{bucket_name}/#{key}" do |s| + s3_client.put_object(key: key, body: File.open(file), bucket: bucket_name) + s.success "Uploaded s3://#{bucket_name}/#{key} successfully." + end + end + + def filename_for_version(version_name) + "#{version_name}.tar.gz" + end + + private + + def doctor_check_bucket_exists + s3_client.get_bucket_location(bucket: @bucket_name) + success "Bucket '#{@bucket_name}' exists." + rescue => e + # This is warning because the role you use for deployment may not actually + # be able to read builds, however the instance role assigned to the nodes + # might. + str = "Could not get information about bucket '#{@bucket_name}'." + warning(str, e.message) + end + + def doctor_check_bucket_writable + s3_client.put_object(key: 'test-object', body: '', bucket: @bucket_name) + s3_client.delete_object(key: 'test-object', bucket: @bucket_name) + success 'Bucket is writable, new builds can be uploaded.' + rescue => e + # This is a warning because you may deploy to an environment where you have + # read access to builds, but could not publish a new build. + warning('Could not write to bucket, you may still be able to deploy existing builds.', e.message) # rubocop:disable LineLength + end +end diff --git a/lib/moonshot/artifact_repository/s3_bucket_via_github_releases.rb b/lib/moonshot/artifact_repository/s3_bucket_via_github_releases.rb new file mode 100644 index 00000000..269d34d6 --- /dev/null +++ b/lib/moonshot/artifact_repository/s3_bucket_via_github_releases.rb @@ -0,0 +1,89 @@ +require 'moonshot/artifact_repository/s3_bucket' +require 'moonshot/shell' +require 'securerandom' +require 'semantic' +require 'tmpdir' + +module Moonshot::ArtifactRepository + # S3 Bucket repository backed by GitHub releases. + # If a SemVer package isn't found in S3, it is copied from GitHub releases. + class S3BucketViaGithubReleases < S3Bucket + include Moonshot::BuildMechanism + include Moonshot::Shell + + # @override + # If release version, transfer from GitHub to S3. + def store_hook(build_mechanism, version) + if release?(version) + if (@output_file = build_mechanism.output_file) + attach_release_asset(version, @output_file) + # Upload to s3. + super + else + # If there is no output file, assume it's on GitHub already. + transfer_release_asset_to_s3(version) + end + else + super + end + end + + # @override + # If release version, transfer from GitHub to S3. + # @todo This is a super hacky place to handle the transfer, give + # artifact repositories a hook before deploy. + def filename_for_version(version) + s3_name = super + if !@output_file && release?(version) && !in_s3?(s3_name) + github_to_s3(version, s3_name) + end + s3_name + end + + private + + def release?(version) + ::Semantic::Version.new(version) + rescue ArgumentError + false + end + + def in_s3?(key) + s3_client.head_object(key: key, bucket: bucket_name) + rescue ::Aws::S3::Errors::NotFound + false + end + + def attach_release_asset(version, file) + # -m '' leaves message unchanged. + cmd = "hub release edit #{version} -m '' --attach=#{file}" + sh_step(cmd) + end + + def transfer_release_asset_to_s3(version) + ilog.start_threaded "Transferring #{version} to S3" do |s| + key = filename_for_version(version) + s.success "Uploaded s3://#{bucket_name}/#{key} successfully." + end + end + + def github_to_s3(version, s3_name) + Dir.mktmpdir('github_to_s3', Dir.getwd) do |tmpdir| + Dir.chdir(tmpdir) do + sh_out("hub release download #{version}") + file = File.open(Dir.glob("*#{version}*.tar.gz").fetch(0)) + s3_client.put_object(key: s3_name, body: file, bucket: bucket_name) + end + end + end + + def doctor_check_hub_release_download + sh_out('hub release download --help') + rescue + critical '`hub release download` command missing, upgrade hub.' \ + ' See https://github.com/github/hub/pull/1103' + else + success '`hub release download` command available.' + end + end +end diff --git a/lib/moonshot/build_mechanism/github_release.rb b/lib/moonshot/build_mechanism/github_release.rb new file mode 100644 index 00000000..57a42dac --- /dev/null +++ b/lib/moonshot/build_mechanism/github_release.rb @@ -0,0 +1,148 @@ +require 'forwardable' +require 'moonshot/shell' +require 'open3' +require 'semantic' +require 'shellwords' +require 'tempfile' +require 'vandamme' + +module Moonshot::BuildMechanism + # A build mechanism that creates a tag and GitHub release. + class GithubRelease # rubocop:disable Metrics/ClassLength + extend Forwardable + include Moonshot::ResourcesHelper + include Moonshot::DoctorHelper + include Moonshot::Shell + + def_delegator :@build_mechanism, :output_file + + # @param build_mechanism Delegates building after GitHub release is created. + def initialize(build_mechanism) + @build_mechanism = build_mechanism + end + + def doctor_hook + super + @build_mechanism.doctor_hook + end + + def resources=(r) + super + @build_mechanism.resources = r + end + + def pre_build_hook(version) + @semver = ::Semantic::Version.new(version) + @target_version = [@semver.major, @semver.minor, @semver.patch].join('.') + sh_step('git fetch --tags upstream') + @sha = `git rev-parse HEAD`.chomp + validate_commit + @changes = validate_changelog(@target_version) + confirm_or_fail(@semver) + @build_mechanism.pre_build_hook(version) + end + + def build_hook(version) + assert_state(version) + git_tag(version, @sha, @changes) + git_push_tag('upstream', version) + hub_create_release(@semver, @sha, @changes) + ilog.msg("#{releases_url}/tag/#{version}") + @build_mechanism.build_hook(version) + end + + def post_build_hook(version) + assert_state(version) + @build_mechanism.post_build_hook(version) + end + + private + + # We carry state between hooks, make sure that's still valid. + def assert_state(version) + raise "#{version} != #{@semver}" unless version == @semver.to_s + end + + def confirm_or_fail(version) + say("\nCommit Summary", :yellow) + say("#{@commit_detail}\n") + say('Commit CI Status', :yellow) + say("#{@ci_statuses}\n") + say("Changelog for #{version}", :yellow) + say("#{@changes}\n\n") + + q = "Do you wan't to tag and release this commit as #{version}? [y/n]" + raise Thor::Error, 'Release declined.' unless yes?(q) + end + + def git_tag(tag, sha, annotation) + cmd = "git tag -a #{tag} #{sha} --file=-" + sh_step(cmd, stdin: annotation) + end + + def git_push_tag(remote, tag) + cmd = "git push #{remote} refs/tags/#{tag}:refs/tags/#{tag}" + sh_step(cmd) do + sleep 2 # GitHub needs a moment to register the tag. + end + end + + def hub_create_release(semver, commitish, changelog_entry) + message = "#{semver}\n\n#{changelog_entry}" + cmd = "hub release create #{semver} --commitish=#{commitish}" + cmd << ' --prerelease' if semver.pre || semver.build + cmd << " --message=#{Shellwords.escape(message)}" + sh_step(cmd) + end + + def validate_commit + cmd = "git show --stat #{@sha}" + sh_step(cmd, msg: "Validate commit #{@sha}.") do |_, out| + @commit_detail = out + end + cmd = "hub ci-status --verbose #{@sha}" + sh_step(cmd, msg: "Check CI status for #{@sha}.") do |_, out| + @ci_statuses = out + end + end + + def validate_changelog(version) + changes = nil + ilog.start_threaded('Validate `CHANGELOG.md`.') do |step| + changes = fetch_changes(version) + step.success + end + changes + end + + def fetch_changes(version) + parser = Vandamme::Parser.new( + changelog: File.read('CHANGELOG.md'), + format: 'markdown' + ) + parser.parse.fetch(version) do + raise "#{version} not found in CHANGELOG.md" + end + end + + def releases_url + `hub browse -u -- releases`.chomp + end + + def doctor_check_upstream + sh_out('git remote | grep ^upstream$') + rescue => e + critical "git remote `upstream` not found.\n#{e.message}" + else + success 'git remote `upstream` exists.' + end + + def doctor_check_hub_auth + sh_out('hub ci-status 0.0.0') + rescue => e + critical "`hub` failed, install hub and authorize it.\n#{e.message}" + else + success '`hub` installed and authorized.' + end + end +end diff --git a/lib/moonshot/build_mechanism/script.rb b/lib/moonshot/build_mechanism/script.rb new file mode 100644 index 00000000..db3df3c1 --- /dev/null +++ b/lib/moonshot/build_mechanism/script.rb @@ -0,0 +1,84 @@ +require 'open3' +include Open3 + +# Compile a release artifact using a shell script. +# +# The output file will be deleted before the script is run, and is expected to +# exist after the script exits. Any non-zero exit status will be consider a +# build failure, and any output will be displayed to the user. +# +# Creating a new Script BuildMechanism looks like this: +# +# class MyReleaseTool < Moonshot::CLI +# include Moonshot::BuildMechanism +# self.build_mechanism = Script.new('script/build.sh') +# end +# +class Moonshot::BuildMechanism::Script + include Moonshot::ResourcesHelper + include Moonshot::DoctorHelper + + attr_reader :output_file + + def initialize(script, output_file: 'output.tar.gz') + @script = script + @output_file = output_file + end + + def pre_build_hook(_version) + File.delete(@output_file) if File.exist?(@output_file) + end + + def build_hook(version) + env = { + 'VERSION' => version, + 'OUTPUT_FILE' => @output_file + } + ilog.start_threaded "Running Script: #{@script}" do |s| + run_script(s, env: env) + end + end + + def post_build_hook(_version) + unless File.exist?(@output_file) # rubocop:disable GuardClause + raise Thor::Error, 'Build command did not produce output file!' + end + end + + private + + def run_script(step, env: {}) # rubocop:disable AbcSize + popen2e(env, @script) do |_, out, wait| + output = [] + + loop do + str = out.gets + unless str.nil? + output << str.chomp + ilog.debug(str.chomp) + end + break if out.eof? + end + + result = wait.value + if result.exitstatus == 0 + step.success "Build script #{@script} exited successfully!" + end + unless result.exitstatus == 0 + ilog.error "Build script failed with exit status #{result.exitstatus}!" + ilog.error 'Last 10 lines of output follows:' + output.pop(10).each { |l| ilog.error l } + + step.failure "Build script #{@script} failed with exit status #{result.exitstatus}!" + end + end + end + + def doctor_check_script_exists + if File.exist?(@script) + success "Script '#{@script}' exists." + else + critical "Could not find build script '#{@script}'!" + end + end +end diff --git a/lib/moonshot/build_mechanism/travis_deploy.rb b/lib/moonshot/build_mechanism/travis_deploy.rb new file mode 100644 index 00000000..19ea02ad --- /dev/null +++ b/lib/moonshot/build_mechanism/travis_deploy.rb @@ -0,0 +1,70 @@ +require 'moonshot/shell' + +module Moonshot::BuildMechanism + # This simply waits for Travis-CI to finish building a job matching the + # version and 'BUILD=1'. + class TravisDeploy + include Moonshot::ResourcesHelper + include Moonshot::DoctorHelper + include Moonshot::Shell + + attr_reader :output_file + + def initialize(slug, pro: false) + @slug = slug + @endpoint = pro ? '--pro' : '--org' + @cli_args = "-r #{@slug} #{@endpoint}" + end + + def pre_build_hook(_) + end + + def build_hook(version) + job_number = find_build_and_job(version) + wait_for_job(job_number) + check_build(version) + end + + def post_build_hook(_) + end + + private + + def find_build_and_job(version) + job_number = nil + ilog.start_threaded('Find Travis CI build') do |step| + sleep 2 + build_out = sh_out("bundle exec travis show #{@cli_args} #{version}") + unless (job_number = build_out.match(/^#(\d+\.\d+) .+BUILD=1.+/)[1]) + raise "Build for #{version} not found.\n#{build_out}" + end + step.success("Travis CI ##{job_number.gsub(/\..*/, '')} running.") + end + job_number + end + + def wait_for_job(job_number) + cmd = "bundle exec travis logs #{@cli_args} #{job_number}" + # This log tailing fails at the end of the file. travis bug. + sh_step(cmd, fail: false) + end + + def check_build(version) + cmd = "bundle exec travis show #{@cli_args} #{version}" + sh_step(cmd) do |step, out| + raise "Build didn't pass.\n#{build_out}" \ + if out =~ /^#(\d+\.\d+) (?!passed).+BUILD=1.+/ + + step.success("Travis CI build for #{version} passed.") + end + end + + def doctor_check_travis_auth + sh_out("bundle exec travis raw #{@endpoint} repos/#{@slug}") + rescue => e + critical "`travis` not available or not authorized.\n#{e.message}" + else + success '`travis` installed and authorized.' + end + end +end diff --git a/lib/moonshot/build_mechanism/version_proxy.rb b/lib/moonshot/build_mechanism/version_proxy.rb new file mode 100644 index 00000000..fe5fdbbb --- /dev/null +++ b/lib/moonshot/build_mechanism/version_proxy.rb @@ -0,0 +1,55 @@ +require 'forwardable' +require 'semantic' + +# This proxies build request do different mechanisms. One for semver compliant +# releases and another for everything else. +class Moonshot::BuildMechanism::VersionProxy + extend Forwardable + include Moonshot::ResourcesHelper + + def_delegator :@active, :output_file + + def initialize(release:, dev:) + @release = release + @dev = dev + end + + def doctor_hook + @release.doctor_hook + @dev.doctor_hook + end + + def resources=(r) + super + @release.resources = r + @dev.resources = r + end + + def pre_build_hook(version) + active(version).pre_build_hook(version) + end + + def build_hook(version) + active(version).build_hook(version) + end + + def post_build_hook(version) + active(version).post_build_hook(version) + end + + private + + def active(version) + @active = if release?(version) + @release + else + @dev + end + end + + def release?(version) + ::Semantic::Version.new(version) + rescue ArgumentError + false + end +end diff --git a/lib/moonshot/cli.rb b/lib/moonshot/cli.rb new file mode 100644 index 00000000..3374bc3f --- /dev/null +++ b/lib/moonshot/cli.rb @@ -0,0 +1,145 @@ +require 'interactive-logger' + +# Base class for Moonshot-powered project tooling. +module Moonshot + # The main entry point for Moonshot, this class should be extended by + # project tooling. + class CLI < Thor # rubocop:disable ClassLength + class_option(:name, aliases: 'n', default: nil, type: :string) + class_option(:interactive_logger, type: :boolean, default: true) + class_option(:verbose, aliases: 'v', type: :boolean) + + class << self + attr_accessor :application_name + attr_accessor :artifact_repository + attr_accessor :auto_prefix_stack + attr_accessor :build_mechanism + attr_accessor :deployment_mechanism + attr_accessor :default_parent_stack + attr_reader :plugins + + def plugin(plugin) + @plugins ||= [] + @plugins << plugin + end + + def parent(value) + @default_parent_stack = value + end + + def check_class_configuration + raise Thor::Error, 'No application_name is set!' unless application_name + end + + def exit_on_failure? + true + end + + def inherited(base) + base.include(Moonshot::ArtifactRepository) + base.include(Moonshot::BuildMechanism) + base.include(Moonshot::DeploymentMechanism) + end + end + + def initialize(*args) + super + @log = Logger.new(STDOUT) + @log.formatter = proc do |s, d, _, msg| + "[#{self.class.name} #{s} #{d.strftime('%T')}] #{msg}\n" + end + @log.level = options[:verbose] ? Logger::DEBUG : Logger::INFO + + EnvironmentParser.parse(@log) + self.class.check_class_configuration + end + + no_tasks do + # Build a Moonshot::Controller from the CLI options. + def controller # rubocop:disable AbcSize, CyclomaticComplexity, PerceivedComplexity + Moonshot::Controller.new do |config| + config.app_name = self.class.application_name + config.artifact_repository = self.class.artifact_repository + config.auto_prefix_stack = self.class.auto_prefix_stack + config.build_mechanism = self.class.build_mechanism + config.deployment_mechanism = self.class.deployment_mechanism + config.environment_name = options[:name] + config.logger = @log + + # Degrade to a more compatible logger if the terminal seems outdated, + # or at the users request. + if !$stdout.isatty || !options[:interactive_logger] + config.interactive_logger = InteractiveLoggerProxy.new(@log) + end + + config.show_all_stack_events = true if options[:show_all_events] + config.plugins = self.class.plugins if self.class.plugins + + if options[:parent] + config.parent_stacks << options[:parent] + elsif self.class.default_parent_stack + config.parent_stacks << self.class.default_parent_stack + end + end + rescue => e + raise Thor::Error, e.message + end + end + + desc :list, 'List stacks for this application.' + def list + controller.list + end + + desc :create, 'Create a new environment.' + option( + :parent, + type: :string, + aliases: '-p', + desc: "Parent stack to import parameters from. (Default: #{default_parent_stack || 'None'})") + option :deploy, default: true, type: :boolean, aliases: '-d', + desc: 'Choose if code should be deployed after stack is created' + option :show_all_events, desc: 'Show all stack events during update. (Default: errors only)' + def create + controller.create + controller.deploy_code if options[:deploy] + end + + desc :update, 'Update the CloudFormation stack within an environment.' + option :show_all_events, desc: 'Show all stack events during update. (Default: errors only)' + def update + controller.update + end + + desc :status, 'Get the status of an existing environment.' + def status + controller.status + end + + desc 'deploy-code', 'Create a build from the working directory, and deploy it.' # rubocop:disable LineLength + def deploy_code + controller.deploy_code + end + + desc 'build-version VERSION', 'Build a tarball of the software, ready for deployment.' # rubocop:disable LineLength + def build_version(version_name) + controller.build_version(version_name) + end + + desc 'deploy-version VERSION_NAME', 'Deploy a versioned release to both EB environments in an environment.' # rubocop:disable LineLength + def deploy_version(version_name) + controller.deploy_version(version_name) + end + + desc :delete, 'Delete an existing environment.' + option :show_all_events, desc: 'Show all stack events during update. (Default: errors only)' + def delete + controller.delete + end + + desc :doctor, 'Run configuration checks against current environment.' + def doctor + controller.doctor + end + end +end diff --git a/lib/moonshot/controller.rb b/lib/moonshot/controller.rb new file mode 100644 index 00000000..6688b794 --- /dev/null +++ b/lib/moonshot/controller.rb @@ -0,0 +1,143 @@ +module Moonshot + # The Controller coordinates and performs all Moonshot actions. + class Controller # rubocop:disable ClassLength + def initialize + @config = ControllerConfig.new + yield @config if block_given? + end + + def list + Moonshot::StackLister.new( + @config.app_name, log: @config.logger).list + end + + def create + run_plugins(:pre_create) + run_hook(:deploy, :pre_create) + stack_ok = stack.create + if stack_ok # rubocop:disable GuardClause + run_hook(:deploy, :post_create) + run_plugins(:post_create) + end + end + + def update + run_plugins(:pre_update) + run_hook(:deploy, :pre_update) + stack.update + run_hook(:deploy, :post_update) + run_plugins(:post_update) + end + + def status + run_plugins(:status) + run_hook(:deploy, :status) + stack.status + end + + def deploy_code + version = "#{stack_name}-#{Time.now.to_i}" + build_version(version) + deploy_version(version) + end + + def build_version(version_name) + run_plugins(:pre_build) + run_hook(:build, :pre_build, version_name) + run_hook(:build, :build, version_name) + run_hook(:build, :post_build, version_name) + run_plugins(:post_build) + run_hook(:repo, :store, @config.build_mechanism, version_name) + end + + def deploy_version(version_name) + run_plugins(:pre_deploy) + run_hook(:deploy, :deploy, @config.artifact_repository, version_name) + run_plugins(:post_deploy) + end + + def delete + run_plugins(:pre_delete) + run_hook(:deploy, :pre_delete) + stack.delete + run_hook(:deploy, :post_delete) + run_plugins(:post_delete) + end + + def doctor + # @todo use #run_hook when Stack becomes an InfrastructureProvider + stack.doctor_hook + run_hook(:build, :doctor) + run_hook(:repo, :doctor) + run_hook(:deploy, :doctor) + run_plugins(:doctor) + end + + def stack + @stack ||= Stack.new(stack_name, + app_name: @config.app_name, + log: @config.logger, + ilog: @config.interactive_logger) do |config| + config.parent_stacks = @config.parent_stacks + config.show_all_events = @config.show_all_stack_events + end + end + + private + + def default_stack_name + user = ENV.fetch('USER').gsub(/\W/, '') + "#{@config.app_name}-dev-#{user}" + end + + def ensure_prefix(name) + if name.start_with?(@config.app_name + '-') + name + else + @config.app_name + "-#{name}" + end + end + + def stack_name + name = @config.environment_name || default_stack_name + if @config.auto_prefix_stack == false + name + else + ensure_prefix(name) + end + end + + def resources + @resources ||= + Resources.new(stack: stack, log: @config.logger, + ilog: @config.interactive_logger) + end + + def run_hook(type, name, *args) + mech = get_mechanism(type) + name = name.to_s << '_hook' + + @config.logger.debug("Calling hook=#{name} on mech=#{mech.class}") + return unless mech && mech.respond_to?(name) + + mech.resources = resources + mech.send(name, *args) + end + + def run_plugins(type) + @config.plugins.each do |plugin| + plugin.send(type, resources) if plugin.respond_to?(type) + end + end + + def get_mechanism(type) + case type + when :build then @config.build_mechanism + when :repo then @config.artifact_repository + when :deploy then @config.deployment_mechanism + else + raise "Unknown hook type: #{type}" + end + end + end +end diff --git a/lib/moonshot/controller_config.rb b/lib/moonshot/controller_config.rb new file mode 100644 index 00000000..ef8e33f5 --- /dev/null +++ b/lib/moonshot/controller_config.rb @@ -0,0 +1,25 @@ +module Moonshot + # Holds configuration for Moonshot::Controller + class ControllerConfig + attr_accessor :app_name + attr_accessor :artifact_repository + attr_accessor :auto_prefix_stack + attr_accessor :build_mechanism + attr_accessor :deployment_mechanism + attr_accessor :environment_name + attr_accessor :interactive_logger + attr_accessor :logger + attr_accessor :parent_stacks + attr_accessor :plugins + attr_accessor :show_all_stack_events + + def initialize + @auto_prefix_stack = true + @interactive_logger = InteractiveLogger.new + @logger = Logger.new(STDOUT) + @parent_stacks = [] + @plugins = [] + @show_all_stack_events = false + end + end +end diff --git a/lib/moonshot/creds_helper.rb b/lib/moonshot/creds_helper.rb new file mode 100644 index 00000000..ba5fe839 --- /dev/null +++ b/lib/moonshot/creds_helper.rb @@ -0,0 +1,28 @@ +module Moonshot + # Create convenience methods for various AWS client creation. + module CredsHelper + def cf_client + Aws::CloudFormation::Client.new + end + + def cd_client + Aws::CodeDeploy::Client.new + end + + def ec2_client + Aws::EC2::Client.new + end + + def iam_client + Aws::IAM::Client.new + end + + def as_client + Aws::AutoScaling::Client.new + end + + def s3_client + Aws::S3::Client.new + end + end +end diff --git a/lib/moonshot/deployment_mechanism/code_deploy.rb b/lib/moonshot/deployment_mechanism/code_deploy.rb new file mode 100644 index 00000000..468ddb2d --- /dev/null +++ b/lib/moonshot/deployment_mechanism/code_deploy.rb @@ -0,0 +1,303 @@ +require 'colorize' + +# This mechanism is used to deploy software to an auto-scaling group within +# a stack. It currently only works with the S3Bucket ArtifactRepository. +# +# Usage: +# class MyApp < Moonshot::CLI +# self.artifact_repository = S3Bucket.new('foobucket') +# self.deployment_mechanism = CodeDeploy.new(asg: 'AutoScalingGroup') +# end +class Moonshot::DeploymentMechanism::CodeDeploy # rubocop:disable ClassLength + include Moonshot::ResourcesHelper + include Moonshot::CredsHelper + include Moonshot::DoctorHelper + + # @param asg [String] + # The logical name of the AutoScalingGroup to create and manage a Deployment + # Group for in CodeDeploy. + # @param app_name [String, nil] (nil) + # The name of the CodeDeploy Application and Deployment Group. By default, + # this is the same as the stack name, and probably what you want. If you + # have multiple deployments in a single Stack, they must have unique names. + def initialize(asg:, app_name: nil) + @asg_logical_id = asg + @app_name = app_name + end + + def post_create_hook + create_application_if_needed + create_deployment_group_if_needed + + wait_for_asg_capacity + end + + def post_update_hook + post_create_hook + + unless deployment_group_ok? # rubocop:disable GuardClause + delete_deployment_group + create_deployment_group_if_needed + end + end + + def status_hook + t = Moonshot::UnicodeTable.new('') + application = t.add_leaf("CodeDeploy Application: #{app_name}") + application.add_line(code_deploy_status_msg) + t.draw_children + end + + def deploy_hook(artifact_repo, version_name) + ilog.start_threaded 'Creating Deployment' do |s| + res = cd_client.create_deployment( + application_name: app_name, + deployment_group_name: app_name, + revision: revision_for_artifact_repo(artifact_repo, version_name), + deployment_config_name: 'CodeDeployDefault.OneAtATime', + description: "Deploying version #{version_name}" + ) + deployment_id = res.deployment_id + s.continue "Created Deployment #{deployment_id.blue}." + wait_for_deployment(deployment_id, s) + end + end + + def post_delete_hook + ilog.start 'Cleaning up CodeDeploy Application' do |s| + if application_exists? + cd_client.delete_application(application_name: app_name) + s.success "Deleted CodeDeploy Application '#{app_name}'." + else + s.success "CodeDeploy Application '#{app_name}' does not exist." + end + end + end + + private + + # By default, use the stack name as the application and deployment group + # names, unless one has been provided. + def app_name + @app_name || stack.name + end + + def pretty_app_name + "CodeDeploy Application #{app_name.blue}" + end + + def pretty_deploy_group + "CodeDeploy Deployment Group #{app_name.blue}" + end + + def create_application_if_needed + ilog.start "Creating #{pretty_app_name}." do |s| + if application_exists? + s.success "#{pretty_app_name} already exists." + else + cd_client.create_application(application_name: app_name) + s.success "Created #{pretty_app_name}." + end + end + end + + def create_deployment_group_if_needed + ilog.start "Creating #{pretty_deploy_group}." do |s| + if deployment_group_exists? + s.success "CodeDeploy #{pretty_deploy_group} already exists." + else + create_deployment_group + s.success "Created #{pretty_deploy_group}." + end + end + end + + def code_deploy_status_msg + case [application_exists?, deployment_group_exists?, deployment_group_ok?] + when [true, true, true] + 'Application and Deployment Group are configured correctly.'.green + when [true, true, false] + 'Deployment Group exists, but not associated with the correct '\ + "Auto-Scaling Group, try running #{'update'.yellow}." + when [true, false, false] + "Deployment Group does not exist, try running #{'create'.yellow}." + when [false, false, false] + 'Application and Deployment Group do not exist, try running'\ + " #{'create'.yellow}." + end + end + + def auto_scaling_group + @auto_scaling_group ||= load_auto_scaling_group + end + + def load_auto_scaling_group + asg_name = stack.physical_id_for(@asg_logical_id) + unless asg_name + raise Thor::Error, "Could not find #{@asg_logical_id} resource in Stack." + end + + groups = as_client.describe_auto_scaling_groups( + auto_scaling_group_names: [asg_name]) + if groups.auto_scaling_groups.empty? + raise Thor::Error, "Could not find ASG #{asg_name}." + end + + groups.auto_scaling_groups.first + end + + def asg_name + auto_scaling_group.auto_scaling_group_name + end + + def application_exists? + cd_client.get_application(application_name: app_name) + true + rescue Aws::CodeDeploy::Errors::ApplicationDoesNotExistException + false + end + + def deployment_group + cd_client.get_deployment_group( + application_name: app_name, deployment_group_name: app_name) + .deployment_group_info + end + + def deployment_group_exists? + cd_client.get_deployment_group( + application_name: app_name, deployment_group_name: app_name) + true + rescue Aws::CodeDeploy::Errors::ApplicationDoesNotExistException, + Aws::CodeDeploy::Errors::DeploymentGroupDoesNotExistException + false + end + + def deployment_group_ok? + return false unless deployment_group_exists? + asg = deployment_group.auto_scaling_groups.first + return false unless asg + asg.name == auto_scaling_group.auto_scaling_group_name + end + + def role + iam_client.get_role(role_name: 'CodeDeployRole').role + rescue Aws::IAM::Errors::NoSuchEntity + raise Thor::Error, 'Did not find an IAM Role: CodeDeployRole' + end + + def delete_deployment_group + ilog.start "Deleting #{pretty_deploy_group}." do |s| + cd_client.delete_deployment_group( + application_name: app_name, + deployment_group_name: app_name) + s.success + end + end + + def create_deployment_group + cd_client.create_deployment_group( + application_name: app_name, + deployment_group_name: app_name, + service_role_arn: role.arn, + auto_scaling_groups: [asg_name]) + end + + def wait_for_asg_capacity + ilog.start 'Waiting for AutoScaling Group to reach capacity...' do |s| + loop do + asg = load_auto_scaling_group + count = asg.instances.count { |i| i.lifecycle_state == 'InService' } + break if asg.desired_capacity == count + s.continue "DesiredCapacity is #{asg.desired_capacity}, currently #{count} instance(s) are InService." # rubocop:disable LineLength + sleep 5 + end + + s.success 'AutoScaling Group up to capacity!' + end + end + + def wait_for_deployment(id, step) + loop do + sleep 5 + info = cd_client.get_deployment(deployment_id: id).deployment_info + status = info.status + + case status + when 'Created', 'Queued', 'InProgress' + step.continue "Waiting for Deployment #{id.blue} to complete, current status is '#{status}'." # rubocop:disable LineLength + when 'Succeeded' + step.success "Deployment #{id.blue} completed successfully!" + break + when 'Failed', 'Stopped' + step.failure "Deployment #{id.blue} failed with status '#{status}'" + handle_deployment_failure(id) + end + end + end + + def handle_deployment_failure(deployment_id) # rubocop:disable AbcSize + instances = cd_client.list_deployment_instances(deployment_id: deployment_id) + .instances_list.map do |instance_id| + cd_client.get_deployment_instance(deployment_id: deployment_id, + instance_id: instance_id) + end + + instances.map(&:instance_summary).each do |inst_summary| + next unless inst_summary.status == 'Failed' + + inst_summary.lifecycle_events.each do |event| + next unless event.status == 'Failed' + + ilog.error(event.diagnostics.message) + event.diagnostics.log_tail.each_line do |line| + ilog.error(line) + end + end + end + + raise Thor::Error, 'Deployment was unsuccessful!' + end + + def revision_for_artifact_repo(artifact_repo, version_name) + case artifact_repo + when Moonshot::ArtifactRepository::S3Bucket + s3_revision_for(artifact_repo, version_name) + when NilClass + raise 'Must specify an ArtifactRepository with CodeDeploy. Take a look at the S3Bucket example.' # rubocop:disable LineLength + else + raise "Cannot use #{artifact_repo.class} to deploy with CodeDeploy." + end + end + + def s3_revision_for(artifact_repo, version_name) + { + revision_type: 'S3', + s3_location: { + bucket: artifact_repo.bucket_name, + key: artifact_repo.filename_for_version(version_name), + bundle_type: 'tgz' + } + } + end + + def doctor_check_code_deploy_role + iam_client.get_role(role_name: 'CodeDeployRole').role + success('CodeDeployRole exists.') + rescue => e + help = <<-EOF +Error: #{e.message} + +For information on provisioning an account for use with CodeDeploy, see: +http://docs.aws.amazon.com/codedeploy/latest/userguide/how-to-create-service-role.html + EOF + critical('Could not find CodeDeployRole, ', help) + end + + def doctor_check_auto_scaling_resource_defined + if stack.template.resource_names.include?(@asg_logical_id) + success("Resource '#{@asg_logical_id}' exists in the CloudFormation template.") # rubocop:disable LineLength + else + critical("Resource '#{@asg_logical_id}' does not exist in the CloudFormation template!") # rubocop:disable LineLength + end + end +end diff --git a/lib/moonshot/doctor_helper.rb b/lib/moonshot/doctor_helper.rb new file mode 100644 index 00000000..3c8632b1 --- /dev/null +++ b/lib/moonshot/doctor_helper.rb @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +require 'colorize' + +module Moonshot + DoctorCritical = Class.new(RuntimeError) + + # + # A series of methods for adding "doctor" checks to a mechanism. + # + module DoctorHelper + def doctor_hook + run_all_checks + end + + private + + def run_all_checks + puts + puts self.class.name.split('::').last + private_methods.each do |meth| + begin + send(meth) if meth =~ /^doctor_check_/ + rescue DoctorCritical + # Stop running checks in this Mechanism. + break + rescue => e + print ' ✗ '.red + puts "Exception while running check: #{e.class}: #{e.message.lines.first}" + break + end + end + end + + def success(str) + print ' ✓ '.green + puts str + end + + def warning(str, additional_info = nil) + print ' ? '.yellow + puts str + additional_info.lines.each { |l| puts " #{l}" } if additional_info + end + + def critical(str, additional_info = nil) + print ' ✗ '.red + puts str + additional_info.lines.each { |l| puts " #{l}" } if additional_info + raise DoctorCritical + end + end +end diff --git a/lib/moonshot/environment_parser.rb b/lib/moonshot/environment_parser.rb new file mode 100644 index 00000000..12a7121c --- /dev/null +++ b/lib/moonshot/environment_parser.rb @@ -0,0 +1,32 @@ +require 'ostruct' + +module Moonshot + # This module supports massaging of the incoming environment. + module EnvironmentParser + def self.parse(log) + log.debug('Starting to parse environment.') + + # Ops Bastion servers export AWS_CREDENTIAL_FILE, instead of key and + # secret keys, so we support both here. We then set them as environment + # variables which will be respected by aws-sdk. + parse_credentials_file if ENV.key?('AWS_CREDENTIAL_FILE') + + # Ensure the aws-sdk is able to find a set of credentials. + creds = Aws::CredentialProviderChain.new(OpenStruct.new).resolve + + raise 'Unable to find AWS credentials!' unless creds + + log.debug('Environment parsing complete.') + end + + def self.parse_credentials_file + File.open(ENV.fetch('AWS_CREDENTIAL_FILE')).each_line do |line| + key, val = line.chomp.split('=') + case key + when 'AWSAccessKeyId' then ENV['AWS_ACCESS_KEY_ID'] = val + when 'AWSSecretKey' then ENV['AWS_SECRET_ACCESS_KEY'] = val + end + end + end + end +end diff --git a/lib/moonshot/interactive_logger_proxy.rb b/lib/moonshot/interactive_logger_proxy.rb new file mode 100644 index 00000000..4fb8b5b8 --- /dev/null +++ b/lib/moonshot/interactive_logger_proxy.rb @@ -0,0 +1,49 @@ +require 'forwardable' + +module Moonshot + # This class pretends to be an InteractiveLogger for systems that are + # non-interactive. + class InteractiveLoggerProxy + # Non-interactive version of InteractiveLogger::Step. + class Step + def initialize(logger) + @logger = logger + end + + def blank + end + + def continue(str = nil) + @logger.info(str) if str + end + + def failure(str = 'Failure') + @logger.error(str) + end + + def repaint + end + + def success(str = 'Success') + @logger.info(str) + end + end + + extend Forwardable + + def_delegator :@debug, :itself, :debug? + def_delegators :@logger, :debug, :error, :info + alias msg info + + def initialize(logger, debug: false) + @debug = debug + @logger = logger + end + + def start(str) + @logger.info(str) + yield Step.new(@logger) + end + alias start_threaded start + end +end diff --git a/lib/moonshot/resources.rb b/lib/moonshot/resources.rb new file mode 100644 index 00000000..c3b36f29 --- /dev/null +++ b/lib/moonshot/resources.rb @@ -0,0 +1,13 @@ +module Moonshot + # Resources is a dependency container that holds references to instances + # provided to a Mechanism (build, deploy, etc.). + class Resources + attr_reader :log, :stack, :ilog + + def initialize(log:, stack:, ilog:) + @log = log + @stack = stack + @ilog = ilog + end + end +end diff --git a/lib/moonshot/resources_helper.rb b/lib/moonshot/resources_helper.rb new file mode 100644 index 00000000..4438cc06 --- /dev/null +++ b/lib/moonshot/resources_helper.rb @@ -0,0 +1,24 @@ +module Moonshot + # Provides shorthand methods for accessing resources provided by the Resources + # container. + module ResourcesHelper + attr_writer :resources + + private + + def log + raise 'Resources not provided to Mechanism!' unless @resources + @resources.log + end + + def stack + raise 'Resources not provided to Mechanism!' unless @resources + @resources.stack + end + + def ilog + raise 'Resources not provided to Mechanism!' unless @resources + @resources.ilog + end + end +end diff --git a/lib/moonshot/shell.rb b/lib/moonshot/shell.rb new file mode 100644 index 00000000..0f8657ec --- /dev/null +++ b/lib/moonshot/shell.rb @@ -0,0 +1,52 @@ +# Mixin providing the Thor::Shell methods and other shell execution helpers. +module Moonshot::Shell + # Run a command, returning stdout. Stderr is suppressed unless the command + # returns non-zero. + def sh_out(cmd, fail: true, stdin: '') # rubocop:disable AbcSize + r_in, w_in = IO.pipe + r_out, w_out = IO.pipe + r_err, w_err = IO.pipe + w_in.write(stdin) + w_in.close + pid = Process.spawn(cmd, in: r_in, out: w_out, err: w_err) + Process.wait(pid) + + r_in.close + w_out.close + w_err.close + stdout = r_out.read + r_out.close + stderr = r_err.read + r_err.close + + if fail && $CHILD_STATUS.exitstatus != 0 + raise "`#{cmd}` exited #{$CHILD_STATUS.exitstatus}\n" \ + "stdout:\n" \ + "#{stdout}\n" \ + "stderr:\n" \ + "#{stderr}\n" + end + stdout + end + module_function :sh_out + + def shell + @thor_shell ||= Thor::Base.shell.new + end + + Thor::Shell::Basic.public_instance_methods(false).each do |meth| + define_method(meth) { |*args| shell.public_send(meth, *args) } + end + + def sh_step(cmd, args = {}) + msg = args.delete(:msg) || cmd + if msg.length > (terminal_width - 18) + msg = "#{msg[0..(terminal_width - 22)]}..." + end + ilog.start_threaded(msg) do |step| + out = sh_out(cmd, args) + yield step, out if block_given? + step.success + end + end +end diff --git a/lib/moonshot/stack.rb b/lib/moonshot/stack.rb new file mode 100644 index 00000000..37918f6d --- /dev/null +++ b/lib/moonshot/stack.rb @@ -0,0 +1,339 @@ +require_relative 'creds_helper' +require_relative 'doctor_helper' + +require_relative 'stack_template' +require_relative 'stack_parameter_printer' +require_relative 'stack_output_printer' +require_relative 'stack_asg_printer' +require_relative 'unicode_table' +require 'yaml' + +module Moonshot + # The Stack wraps all CloudFormation actions performed by Moonshot. It + # stores the state of the active stack running on AWS, but contains a + # reference to the StackTemplate that would be applied with an update + # action. + class Stack # rubocop:disable ClassLength + include CredsHelper + include DoctorHelper + + attr_reader :app_name + attr_reader :name + + # TODO: Refactor more of these parameters into the config object. + def initialize(name, app_name:, log:, ilog:, config: StackConfig.new) + @name = name + @app_name = app_name + @log = log + @ilog = ilog + @config = config + yield @config if block_given? + end + + def create + import_parent_parameters + + should_wait = true + @ilog.start "Creating #{stack_name}." do |s| + if stack_exists? + s.success "#{stack_name} already exists." + should_wait = false + else + create_stack + s.success "Created #{stack_name}." + end + end + + should_wait ? wait_for_stack_state(:stack_create_complete, 'created') : true + end + + def update + raise Thor::Error, "No stack found #{@name.blue}!" unless stack_exists? + + should_wait = true + @ilog.start "Updating #{stack_name}." do |s| + if update_stack + s.success "Initiated update for #{stack_name}." + else + s.success 'No Stack update required.' + should_wait = false + end + end + + should_wait ? wait_for_stack_state(:stack_update_complete, 'updated') : true + end + + def delete + should_wait = true + @ilog.start "Deleting #{stack_name}." do |s| + if stack_exists? + cf_client.delete_stack(stack_name: @name) + s.success "Initiated deletion of #{stack_name}." + else + s.success "#{stack_name} does not exist." + should_wait = false + end + end + + should_wait ? wait_for_stack_state(:stack_delete_complete, 'deleted') : true + end + + def status + if exists? + puts "#{stack_name} exists." + t = UnicodeTable.new('') + StackParameterPrinter.new(self, t).print + StackOutputPrinter.new(self, t).print + StackASGPrinter.new(self, t).print + t.draw_children + else + puts "#{stack_name} does NOT exist." + end + end + + def parameters + get_stack(@name).parameters + end + + def outputs + get_stack(@name) + .outputs + .map { |o| [o.output_key, o.output_value] } + .to_h + end + + def exists? + cf_client.describe_stacks(stack_name: @name) + true + rescue Aws::CloudFormation::Errors::ValidationError + false + end + alias stack_exists? exists? + + def resource_summaries + cf_client.list_stack_resources(stack_name: @name).stack_resource_summaries + end + + # @return [String, nil] + def physical_id_for(logical_id) + resource_summary = resource_summaries.find do |r| + r.logical_resource_id == logical_id + end + resource_summary.physical_resource_id if resource_summary + end + + # @return [Array] + def resources_of_type(type) + resource_summaries.select do |r| + r.resource_type == type + end + end + + # Build a hash of overrides that would be applied to this stack by an + # update. + def overrides + if File.exist?(parameters_file) + YAML.load_file(parameters_file) + else + {} + end + end + + # Return a Hash of the default values defined in the stack template. + def default_values + h = {} + JSON.parse(template.body).fetch('Parameters', {}).map do |k, v| + h[k] = v['Default'] + end + h + end + + def template + @template ||= StackTemplate.new(template_file, log: @log) + end + + # @return [String] the path to the template file. + def template_file + File.join(Dir.pwd, 'cloud_formation', "#{@app_name}.json") + end + + # @return [String] the path to the parameters file. + def parameters_file + File.join(Dir.pwd, 'cloud_formation', 'parameters', "#{@name}.yml") + end + + def add_parameter_overrides(hash) + new_overrides = hash.merge(overrides) + File.open(parameters_file, 'w') do |f| + YAML.dump(new_overrides, f) + end + end + + private + + def stack_name + "CloudFormation Stack #{@name.blue}" + end + + def load_parameters_file + @ilog.msg "Loading stack parameters file '#{parameters_file}'." + result = stack_parameter_overrides + + if result.empty? + @ilog.msg "No parameters file for #{@name.blue}, using defaults." + return result + end + + @ilog.msg 'Setting stack parameter overrides:' + result.each do |e| + @ilog.msg " #{e[:parameter_key]}: #{e[:parameter_value]}" + end + end + + def stack_parameter_overrides + overrides.map do |k, v| + { parameter_key: k, parameter_value: v.to_s } + end + end + + def stack_parameters + @stack_parameters ||= JSON.parse(template.body).fetch('Parameters', {}).keys + end + + def import_parent_parameters + add_parameter_overrides(parent_stack_outputs) + end + + # Return a Hash of parent stack outputs that match parameter names for this + # stack. + def parent_stack_outputs + result = {} + + @config.parent_stacks.each do |stack_name| + resp = cf_client.describe_stacks(stack_name: stack_name) + raise "Parent Stack #{stack_name} not found!" unless resp.stacks.size == 1 + + # If there is an input parameters matching a stack output, pass it. + resp.stacks[0].outputs.each do |output| + if stack_parameters.include?(output.output_key) + result[output.output_key] = output.output_value + end + end + end + + result + end + + # @return [Aws::CloudFormation::Types::Stack] + def get_stack(name) + stacks = cf_client.describe_stacks(stack_name: name).stacks + raise Thor::Error, "Could not describe stack: #{name}" if stacks.empty? + + stacks.first + rescue Aws::CloudFormation::Errors::ValidationError + raise Thor::Error, "Could not describe stack: #{name}" + end + + def create_stack + cf_client.create_stack( + stack_name: @name, + template_body: template.body, + capabilities: ['CAPABILITY_IAM'], + parameters: load_parameters_file, + tags: [ + { key: 'ah_stage', value: @name } + ] + ) + rescue Aws::CloudFormation::Errors::AccessDenied + raise Thor::Error, 'You are not authorized to perform create_stack calls.' + end + + # @return [Boolean] + # true if a stack update was required and initiated, false otherwise. + def update_stack + cf_client.update_stack( + stack_name: @name, + template_body: template.body, + capabilities: ['CAPABILITY_IAM'], + parameters: stack_parameter_overrides + ) + true + rescue Aws::CloudFormation::Errors::ValidationError => e + raise Thor::Error, e.message unless + e.message == 'No updates are to be performed.' + false + end + + # TODO: Refactor this into it's own class. + def wait_for_stack_state(wait_target, past_tense_verb) # rubocop:disable AbcSize + result = true + + stack_id = get_stack(@name).stack_id + + events = StackEventsPoller.new(cf_client, stack_id) + events.show_only_errors unless @config.show_all_events + + @ilog.start_threaded "Waiting for #{stack_name} to be successfully #{past_tense_verb}." do |s| + cf_client.wait_until(wait_target, stack_name: stack_id) do |w| + w.delay = 10 + w.max_attempts = 180 # 30 minutes. + w.before_wait do |attempt, resp| + begin + events.latest_events.each do |event| + @ilog.error(format_event(event)) + end + # rubocop:disable Lint/HandleExceptions + rescue Aws::CloudFormation::Errors::ValidationError + # Do nothing. The above event logging block may result in + # a ValidationError while waiting for a stack to delete. + end + # rubocop:enable Lint/HandleExceptions + + if attempt == w.max_attempts - 1 + s.failure "#{stack_name} was not #{past_tense_verb} after 30 minutes." + result = false + + # We don't want the interactive logger to catch an exception. + throw :success + end + s.continue "Waiting for CloudFormation Stack to be successfully #{past_tense_verb}, current status '#{resp.stacks.first.stack_status}'." # rubocop:disable LineLength + end + end + + s.success "#{stack_name} successfully #{past_tense_verb}." if result + end + + result + end + + def format_event(event) + str = case event.resource_status + when /FAILED/ + event.resource_status.red + when /IN_PROGRESS/ + event.resource_status.yellow + else + event.resource_status.green + end + str << " #{event.logical_resource_id}" + str << " #{event.resource_status_reason.light_black}" if event.resource_status_reason + + str + end + + def doctor_check_template_exists + if File.exist?(template_file) + success "CloudFormation template found at '#{template_file}'." + else + critical "CloudFormation template not found at '#{template_file}'!" + end + end + + def doctor_check_template_against_aws + cf_client.validate_template(template_body: template.body) + success('CloudFormation template is valid.') + rescue => e + critical('Invalid CloudFormation template!', e.message) + end + end +end diff --git a/lib/moonshot/stack_asg_printer.rb b/lib/moonshot/stack_asg_printer.rb new file mode 100644 index 00000000..e0d8adf5 --- /dev/null +++ b/lib/moonshot/stack_asg_printer.rb @@ -0,0 +1,151 @@ +# coding: utf-8 +require 'colorize' +require 'ruby-duration' + +module Moonshot + # Display information about the AutoScaling Groups, associated ELBs, and + # managed instances to the user. + class StackASGPrinter # rubocop:disable ClassLength + include CredsHelper + + def initialize(stack, table) + @stack = stack + @table = table + end + + def print + asgs.each do |asg| + asg_info = as_client.describe_auto_scaling_groups( + auto_scaling_group_names: [asg.physical_resource_id]) + .auto_scaling_groups.first + t_asg_info = @table.add_leaf("ASG: #{asg.logical_resource_id}") + + add_asg_info(t_asg_info, asg_info) + instances_leaf = t_asg_info.add_leaf('Instances') + + if asg_info.instances.empty? + instances_leaf.add_line('There are no instances in this Auto-Scaling Group.') + else + instances_leaf.add_table(create_instance_table(asg_info)) + end + + add_recent_activity_leaf(t_asg_info, asg.physical_resource_id) + end + end + + private + + def asgs + @stack.resources_of_type('AWS::AutoScaling::AutoScalingGroup') + end + + def status_with_color(status) + case status + when 'Successful' + status.green + when 'Failed' + status.red + else + status.yellow + end + end + + def lifecycle_color(lifecycle) + case lifecycle + when 'InService' + lifecycle.green + else + lifecycle.red + end + end + + def health_color(health) + case health + when 'Healthy' + health.green + else + health.red + end + end + + # Get additional information about instances not returned by the ASG API. + def get_addl_info(instance_ids) + resp = ec2_client.describe_instances(instance_ids: instance_ids) + + data = {} + resp.reservations.map(&:instances).flatten.each do |instance| + data[instance.instance_id] = instance + end + data + end + + def add_asg_info(table, asg_info) # rubocop:disable AbcSize + name = asg_info.auto_scaling_group_name.blue + table.add_line "Name: #{name}" + + hc = asg_info.health_check_type.blue + gp = (asg_info.health_check_grace_period.to_s << 's').blue + table.add_line "Using #{hc} health checks, with a #{gp} health check grace period." # rubocop:disable LineLength + + dc = asg_info.desired_capacity.to_s.blue + min = asg_info.min_size.to_s.blue + max = asg_info.max_size.to_s.blue + table.add_line "Desired Capacity is #{dc} (Min: #{min}, Max: #{max})." + + lbs = asg_info.load_balancer_names + table.add_line "Has #{lbs.count.to_s.blue} Load Balancer(s): #{lbs.map(&:blue).join(', ')}" # rubocop:disable LineLength + end + + def create_instance_table(asg_info) + current_lc = asg_info.launch_configuration_name + ec2_info = get_addl_info(asg_info.instances.map(&:instance_id)) + asg_info.instances.map do |asg_instance| + row = instance_row(asg_instance, + ec2_info[asg_instance.instance_id]) + row << if current_lc == asg_instance.launch_configuration_name + '(launch config up to date)'.green + else + '(launch config out of date)'.red + end + end + end + + def instance_row(asg_instance, ec2_instance) + [ + asg_instance.instance_id, + # @todo What about ASGs with only private IPs? + ec2_instance.public_ip_address, + lifecycle_color(asg_instance.lifecycle_state), + health_color(asg_instance.health_status), + uptime_format(ec2_instance.launch_time) + ] + end + + def uptime_format(launch_time) + # %td is "total days", instead of counting up again to weeks. + Duration.new(Time.now.to_i - launch_time.to_i) + .format('%tdd %hh %mm %ss') + end + + def add_recent_activity_leaf(table, asg_name) + recent = table.add_leaf('Recent Activity') + resp = as_client.describe_scaling_activities( + auto_scaling_group_name: asg_name).activities + + rows = resp.sort_by(&:start_time).reverse.first(10).map do |activity| + row_for_activity(activity) + end + + recent.add_table(rows) + end + + def row_for_activity(activity) + [ + activity.start_time.to_s.light_black, + activity.description, + status_with_color(activity.status_code), + activity.progress.to_s << '%' + ] + end + end +end diff --git a/lib/moonshot/stack_config.rb b/lib/moonshot/stack_config.rb new file mode 100644 index 00000000..7093ce1a --- /dev/null +++ b/lib/moonshot/stack_config.rb @@ -0,0 +1,12 @@ +module Moonshot + # Configuration for the Moonshot::Stack class. + class StackConfig + attr_accessor :parent_stacks + attr_accessor :show_all_events + + def initialize + @parent_stacks = [] + @show_all_events = false + end + end +end diff --git a/lib/moonshot/stack_events_poller.rb b/lib/moonshot/stack_events_poller.rb new file mode 100644 index 00000000..46a90092 --- /dev/null +++ b/lib/moonshot/stack_events_poller.rb @@ -0,0 +1,56 @@ +module Moonshot + # The StackEventsPoller queries DescribeStackEvents every time #latest_events + # is invoked, filtering out events that have already been returned. It can + # also, optionally, filter all non-error events (@see #show_errors_only). + class StackEventsPoller + def initialize(cf_client, stack_name) + @cf_client = cf_client + @stack_name = stack_name + + # Start showing events from now. + @last_time = Time.now + end + + def show_only_errors + @errors_only = true + end + + # Build a list of events that have occurred since the last call to this + # method. + # + # @return [Array] + def latest_events + events = get_stack_events.select do |event| + event.timestamp > @last_time + end + + @last_time = Time.now + + filter_events(events.sort_by(&:timestamp)) + end + + def filter_events(events) + if @errors_only + events.select do |event| + %w(CREATE_FAILED UPDATE_FAILED DELETE_FAILED).include?(event.resource_status) + end + else + events + end + end + + def get_stack_events(token = nil) + opts = { + stack_name: @stack_name + } + + opts[:next_token] = token if token + + result = @cf_client.describe_stack_events(**opts) + events = result.stack_events + events += get_stack_events(result.next_token) if result.next_token + + events + end + end +end diff --git a/lib/moonshot/stack_lister.rb b/lib/moonshot/stack_lister.rb new file mode 100644 index 00000000..a8b99e5c --- /dev/null +++ b/lib/moonshot/stack_lister.rb @@ -0,0 +1,20 @@ +module Moonshot + # The StackLister is world renoun for it's ability to list stacks. + class StackLister + include CredsHelper + + def initialize(app_name, log:) + @app_name = app_name + @log = log + end + + def list + all_stacks = cf_client.describe_stacks.stacks + app_stacks = all_stacks.reject { |s| s.stack_name !~ /^#{@app_name}/ } + + app_stacks.each do |stack| + puts stack.stack_name + end + end + end +end diff --git a/lib/moonshot/stack_output_printer.rb b/lib/moonshot/stack_output_printer.rb new file mode 100644 index 00000000..b58c863c --- /dev/null +++ b/lib/moonshot/stack_output_printer.rb @@ -0,0 +1,16 @@ +module Moonshot + # Display the stack outputs to the user. + class StackOutputPrinter + def initialize(stack, table) + @stack = stack + @table = table + end + + def print + o_table = @table.add_leaf('Stack Outputs') + @stack.outputs.each do |k, v| + o_table.add_line("#{k}: #{v}") + end + end + end +end diff --git a/lib/moonshot/stack_parameter_printer.rb b/lib/moonshot/stack_parameter_printer.rb new file mode 100644 index 00000000..5a557b0c --- /dev/null +++ b/lib/moonshot/stack_parameter_printer.rb @@ -0,0 +1,73 @@ +module Moonshot + # Displays information about existing stack parameters to the user, with + # information on what a stack update would do. + class StackParameterPrinter + def initialize(stack, table) + @stack = stack + @table = table + end + + def print # rubocop:disable AbcSize + p_table = @table.add_leaf('Stack Parameters') + overrides = @stack.overrides + rows = @stack.parameters.sort_by(&:parameter_key).map do |parm| + t_param = @stack.template.parameters.find do |p| + p.name == parm.parameter_key + end + + properties = determine_change(t_param ? t_param.default : nil, + overrides[parm.parameter_key], + parm.parameter_value) + + [ + parm.parameter_key << ':', + format_value(parm.parameter_value), + format_properties(properties) + ] + end + + p_table.add_table(rows) + end + + def determine_change(default, override, current) + properties = [] + + # If there is a stack override, determine if it would change the current + # stack. + if override + properties << 'overridden' + if current == '****' + properties << 'may be updated, NoEcho set' + elsif override != current + properties << "would be updated to #{override}" + end + + else + # Otherwise, compare the template default with the current value to + # determine outcome. + properties << 'default' + properties << "would be updated to #{default}" if default != current + end + + properties + end + + def format_properties(properties) + string = " (#{properties.join(', ')})" + + if properties.any? { |p| p =~ /be updated/ } + string.yellow + else + string.green + end + end + + def format_value(value) + if value.size > 40 + value[0..40] + '...' + else + value + end + end + end +end diff --git a/lib/moonshot/stack_template.rb b/lib/moonshot/stack_template.rb new file mode 100644 index 00000000..f18587fe --- /dev/null +++ b/lib/moonshot/stack_template.rb @@ -0,0 +1,35 @@ +module Moonshot + # A StackTemplate loads the JSON template from disk and stores information + # about it. + class StackTemplate + Parameter = Struct.new(:name, :default) do + def required? + default.nil? + end + end + + attr_reader :body + + def initialize(filename, log:) + @log = log + + unless File.exist?(filename) + @log.error("Could not find CloudFormation template at #{filename}") + raise + end + + @body = File.read(filename) + end + + def parameters + JSON.parse(@body).fetch('Parameters', {}).map do |k, v| + Parameter.new(k, v['Default']) + end + end + + # Return a list of defined resource names in the template. + def resource_names + JSON.parse(@body).fetch('Resources', {}).keys + end + end +end diff --git a/lib/moonshot/unicode_table.rb b/lib/moonshot/unicode_table.rb new file mode 100644 index 00000000..917d2ac5 --- /dev/null +++ b/lib/moonshot/unicode_table.rb @@ -0,0 +1,63 @@ +# coding: utf-8 + +require 'colorize' + +module Moonshot + # A class for drawing hierarchical information using unicode lines. + class UnicodeTable + def initialize(name) + @name = name + @lines = [] + @children = [] + end + + def add_leaf(name) + new_leaf = UnicodeTable.new(name) + @children << new_leaf + new_leaf + end + + def add_line(line) + @lines << line + self + end + + def add_table(table) + # Calculate widths + widths = [] + table.each do |line| + line.each_with_index do |col, i| + col = '?' unless col.respond_to?(:length) + widths[i] = [widths[i] || 0, col.length].max + end + end + + format = widths.collect { |n| "%-#{n}s" }.join(' ') << "\n" + table.each { |line| add_line(format(format, *line)) } + end + + def draw(depth = 1, first = true) + print first ? '┌' : '├' + print '─' * depth + puts ' ' << @name.light_black + @lines = [''] + @lines + [''] + @lines.each do |line| + puts '│' << (' ' * depth) << line + end + @children.each do |child| + child.draw(depth + 1, false) + end + end + + # Draw all children at the same level, for having multiple top-level + # peer leaves. + def draw_children + first = true + @children.each do |child| + child.draw(1, first) + first = false + end + puts '└──' + end + end +end diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..898cd09d --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,4 @@ +site_name: Moonshot +pages: +- Home: index.md +- Contribute: contribute.md \ No newline at end of file diff --git a/moonshot.gemspec b/moonshot.gemspec new file mode 100644 index 00000000..f8ba698c --- /dev/null +++ b/moonshot.gemspec @@ -0,0 +1,27 @@ +Gem::Specification.new do |s| + s.name = 'moonshot' + s.version = '0.6.0' + s.licenses = ['MIT'] + s.summary = 'A library for launching services into AWS' + s.description = 'A library for launching services into AWS.' + s.authors = [ + 'Cloud Engineering ' + ] + s.email = 'engineering@acquia.com' + s.files = Dir['lib/**/*.rb'] + s.homepage = 'https://github.com/acquia/moonshot' + + s.add_dependency('aws-sdk', '~> 2.2.0') + s.add_dependency('colorize') + s.add_dependency('highline', '~> 1.7.2') + s.add_dependency('interactive-logger', '~> 0.1.1') + s.add_dependency('rotp', '~> 2.1.1') + s.add_dependency('ruby-duration') + s.add_dependency('thor', '~> 0.19.1') + s.add_dependency('semantic') + s.add_dependency('vandamme') + + s.add_development_dependency('rspec') + s.add_development_dependency('simplecov') + s.add_development_dependency('fakefs') +end diff --git a/spec/.rspec b/spec/.rspec new file mode 100644 index 00000000..cbb31ea9 --- /dev/null +++ b/spec/.rspec @@ -0,0 +1 @@ +--require spec_helper.rb --color \ No newline at end of file diff --git a/spec/fs_fixtures/cloud_formation/rspec-app.json b/spec/fs_fixtures/cloud_formation/rspec-app.json new file mode 100644 index 00000000..f42da782 --- /dev/null +++ b/spec/fs_fixtures/cloud_formation/rspec-app.json @@ -0,0 +1,12 @@ +{ + "Resources": { + + }, + + "Parameters": { + "Parent1": { + "Type": "String", + "Description": "This is imported from the parent stack on create." + } + } +} diff --git a/spec/moonshot/build_mechanism/github_release_spec.rb b/spec/moonshot/build_mechanism/github_release_spec.rb new file mode 100644 index 00000000..41878f88 --- /dev/null +++ b/spec/moonshot/build_mechanism/github_release_spec.rb @@ -0,0 +1,76 @@ +module Moonshot + describe BuildMechanism::GithubRelease do + let(:build_mechanism) do + instance_double(BuildMechanism::Script).as_null_object + end + + let(:resources) do + Resources.new( + ilog: instance_double(InteractiveLogger).as_null_object, + log: instance_double(Logger).as_null_object, + stack: instance_double(Stack).as_null_object + ) + end + + let(:slug) { 'myorg/myrepo' } + + subject do + s = described_class.new(build_mechanism) + s.resources = resources + s + end + + describe '#doctor_hook' do + it 'should call our hooks' do + allow(subject).to receive(:puts) + allow(subject).to receive(:print) + expect(subject).to receive(:doctor_check_hub_auth) + expect(subject).to receive(:doctor_check_upstream) + subject.doctor_hook + end + + describe '#doctor_check_upstream' do + around do |example| + Dir.mktmpdir do |path| + Dir.chdir(path) do + `git init` + example.run + end + end + end + + it 'should fail without upstream.' do + expect(subject).to receive(:critical) + .with(/git remote `upstream` not found/) + subject.send(:doctor_check_upstream) + end + + it 'should succeed with upstream remote.' do + `git remote add upstream https://example.com/my/repo.git` + expect(subject).to receive(:success) + .with('git remote `upstream` exists.') + subject.send(:doctor_check_upstream) + end + end + + describe '#doctor_check_hub_auth' do + it 'should succeed with 0 exit status.' do + expect(subject).to receive(:sh_out) + .with('hub ci-status 0.0.0') + expect(subject).to receive(:success) + .with('`hub` installed and authorized.') + subject.send(:doctor_check_hub_auth) + end + + it 'should critical with non-zero exit status.' do + expect(subject).to receive(:sh_out) + .with('hub ci-status 0.0.0') + .and_raise(RuntimeError, 'oops') + expect(subject).to receive(:critical) + .with("`hub` failed, install hub and authorize it.\noops") + subject.send(:doctor_check_hub_auth) + end + end + end + end +end diff --git a/spec/moonshot/build_mechanism/travis_deploy_spec.rb b/spec/moonshot/build_mechanism/travis_deploy_spec.rb new file mode 100644 index 00000000..ec414258 --- /dev/null +++ b/spec/moonshot/build_mechanism/travis_deploy_spec.rb @@ -0,0 +1,49 @@ +module Moonshot + describe BuildMechanism::TravisDeploy do + let(:resources) do + Resources.new( + ilog: double(InteractiveLogger).as_null_object, + log: double(Logger).as_null_object, + stack: double(Stack).as_null_object + ) + end + let(:slug) { 'myorg/myrepo' } + + subject do + s = described_class.new(slug) + s.resources = resources + s + end + + describe '#doctor_hook' do + it 'should call our hooks' do + allow(subject).to receive(:puts) + expect(subject).to receive(:puts).with('we did it') + expect(subject).to receive(:print).with(' ✓ '.green) + expect(subject).to receive(:doctor_check_travis_auth) do + subject.send(:success, 'we did it') + end + subject.doctor_hook + end + + describe '#doctor_check_travis_auth' do + it 'should pass if travis exits 0' do + expect(subject).to receive(:sh_out) + .with('bundle exec travis raw --org repos/myorg/myrepo') + expect(subject).to receive(:success) + .with('`travis` installed and authorized.') + subject.send(:doctor_check_travis_auth) + end + + it 'should pass fail travis exits 1' do + expect(subject).to receive(:sh_out) + .with('bundle exec travis raw --org repos/myorg/myrepo') + .and_raise(RuntimeError, 'stuffs broke man') + expect(subject).to receive(:critical) + .with("`travis` not available or not authorized.\nstuffs broke man") + subject.send(:doctor_check_travis_auth) + end + end + end + end +end diff --git a/spec/moonshot/cli_spec.rb b/spec/moonshot/cli_spec.rb new file mode 100644 index 00000000..7ff54ed4 --- /dev/null +++ b/spec/moonshot/cli_spec.rb @@ -0,0 +1,73 @@ +require 'moonshot' + +class AppWithAutoPrefix < Moonshot::CLI + self.application_name = 'my-app' +end + +class AppWithoutAutoPrefix < Moonshot::CLI + self.auto_prefix_stack = false + self.application_name = 'my-other-app' +end + +class AppToTest < Moonshot::CLI + self.application_name = 'my-app' +end + +describe AppWithAutoPrefix do + let(:stack) { instance_double(Moonshot::Stack) } + before(:each) do + expect(Moonshot::EnvironmentParser).to receive(:parse) + expect(stack).to receive(:status) + stub_const('ENV', 'USER' => 'rspec') + end + + it 'should prepend the app name when not specified' do + expect(Moonshot::Stack).to receive(:new) + .with('my-app-stack1', an_instance_of(Hash)).and_return(stack) + described_class.start('status -n stack1'.split) + end + + it 'should not prepend the app name with already specified' do + expect(Moonshot::Stack).to receive(:new) + .with('my-app-stack1', an_instance_of(Hash)).and_return(stack) + described_class.start('status -n my-app-stack1'.split) + end +end + +describe AppWithoutAutoPrefix do + let(:stack) { instance_double(Moonshot::Stack) } + before(:each) do + expect(Moonshot::EnvironmentParser).to receive(:parse) + expect(stack).to receive(:status) + stub_const('ENV', 'USER' => 'rspec') + end + + it 'should not prepend the app name when not specified' do + expect(Moonshot::Stack).to receive(:new) + .with('stack1', an_instance_of(Hash)).and_return(stack) + described_class.start('status -n stack1'.split) + end + it 'should not prepend the app name when already specified' do + expect(Moonshot::Stack).to receive(:new) + .with('my-other-app-stack1', an_instance_of(Hash)).and_return(stack) + described_class.start('status -n my-other-app-stack1'.split) + end +end + +describe AppToTest do + before(:each) do + expect(Moonshot::EnvironmentParser).to receive(:parse) + stub_const('ENV', 'USER' => 'rspec') + end + + it 'created a new controller object' do + controller = instance_double(Moonshot::Controller) + expect(Moonshot::Controller).to receive(:new).and_return(controller) + described_class.new.controller + end + + it 'can access stack by inheriting Moonshot::CLI' do + expect(Moonshot::Stack).to receive(:new) + described_class.new.controller.stack + end +end diff --git a/spec/moonshot/interactive_logger_proxy_spec.rb b/spec/moonshot/interactive_logger_proxy_spec.rb new file mode 100644 index 00000000..e589ce01 --- /dev/null +++ b/spec/moonshot/interactive_logger_proxy_spec.rb @@ -0,0 +1,30 @@ +shared_examples 'compatible class' do |expected_class| + expected_class.public_instance_methods(false).each do |name| + describe "##{name}" do + it 'should be defined.' do + described_class.public_instance_method(name) + end + + it "should have method arguments compatible with #{expected_class.name}##{name}" do + expected_method = expected_class.instance_method(name) + actual_method = described_class.instance_method(name) + expect(actual_method.arity).to eq(-1).or be >= expected_method.arity + actual_method.parameters.each_with_index do |(type, arg), index| + # If it's a required argument, expect it to be identical. + if type == :req + expect(name => expected_method.parameters[index]) + .to eq(name => [type, arg]) + end + end + end + end + end +end + +describe Moonshot::InteractiveLoggerProxy do + include_examples 'compatible class', InteractiveLogger +end + +describe Moonshot::InteractiveLoggerProxy::Step do + include_examples 'compatible class', InteractiveLogger::Step +end diff --git a/spec/moonshot/plugins_spec.rb b/spec/moonshot/plugins_spec.rb new file mode 100644 index 00000000..e6cdf45a --- /dev/null +++ b/spec/moonshot/plugins_spec.rb @@ -0,0 +1,71 @@ +class MockPlugin + def pre_create + end + + def post_create + end +end + +describe 'the Moonshot::CLI interface to plugins' do + let(:controller) { instance_double('Moonshot::Controller') } + + subject do + Class.new(Moonshot::CLI) do + self.application_name = 'my-app' + plugin MockPlugin.new + plugin MockPlugin.new + end + end + + before(:each) do + expect(Moonshot::EnvironmentParser).to receive(:parse) + end + + it 'sets the plugins provided to Moonshot::Controller' do + config = Moonshot::ControllerConfig.new + expect(Moonshot::Controller).to receive(:new).and_yield(config).and_return(controller) + expect(controller).to receive(:status) + + subject.start(['status']) + expect(config.plugins.size).to eq(2) + end +end + +describe 'Plugins support' do + let(:plugin1) { MockPlugin.new } + let(:plugin2) { MockPlugin.new } + + let(:stack) { instance_double('Moonshot::Stack') } + + subject do + Moonshot::Controller.new do |config| + config.app_name = 'my-app' + config.plugins = [plugin1, plugin2] + config.logger = Logger.new(nil) + end + end + + before(:each) do + expect(Moonshot::Stack).to receive(:new).and_return(stack) + end + + it 'calls defined methods on plugins in order, providing them with a Moonshot::Resources' do + expect(plugin1).to receive(:pre_create).with(an_instance_of(Moonshot::Resources)).ordered + expect(plugin2).to receive(:pre_create).with(an_instance_of(Moonshot::Resources)).ordered + expect(stack).to receive(:create).ordered.and_return(true) + expect(plugin1).to receive(:post_create).with(an_instance_of(Moonshot::Resources)).ordered + expect(plugin2).to receive(:post_create).with(an_instance_of(Moonshot::Resources)).ordered + + subject.create + end + + it "doesn't call an undefined method" do + expect(stack).to receive(:delete) + + # The assertion here is that calling MockPlugin#pre_delete would cause an + # exception. Using an expect().not_to receive() changes the behavior of + # #respond_to?, so we can't write that expectation. + expect { subject.delete } + .not_to raise_error + end +end diff --git a/spec/moonshot/shell_spec.rb b/spec/moonshot/shell_spec.rb new file mode 100644 index 00000000..c59bc946 --- /dev/null +++ b/spec/moonshot/shell_spec.rb @@ -0,0 +1,68 @@ +require 'securerandom' + +module Moonshot + describe Shell do + include ResourcesHelper + include described_class + + let(:resources) do + log = instance_double(Logger).as_null_object + Resources.new( + ilog: InteractiveLoggerProxy.new(log), + log: log, + stack: double(Stack).as_null_object + ) + end + + before { self.resources = resources } + + describe '#shell' do + it 'should return a shell compatible with Thor::Shell::Basic.' do + expect(shell).to be_a(Thor::Shell::Basic) + end + end + + describe '#sh_out' do + it 'should raise on non-zero exit.' do + expect { sh_out('false') }.to raise_error(/`false` exited 1/) + end + + it 'should not raise if fail is disabled.' do + sh_out('false', fail: false) + end + end + + describe '#sh_step' do + before do + expect(InteractiveLoggerProxy::Step).to receive(:new).and_return(step) + end + + let(:step) { instance_double(InteractiveLoggerProxy::Step) } + + it 'should raise an error if the step fails.' do + expect { sh_step('false') }.to raise_error(/`false` exited 1/) + end + + it 'should provide the step and sh output to a block.' do + output = nil + expect(step).to receive(:continue).with('reticulating splines') + expect(step).to receive(:success) + sh_step('echo yo') do |step, out| + step.continue('reticulating splines') + output = out + end + expect(output).to match('yo') + end + + it 'should truncate a long messages.' do + long_s = SecureRandom.urlsafe_base64(terminal_width) + cmd = "echo #{long_s}" + truncated_s = "#{cmd[0..(terminal_width - 22)]}..." + expect(resources.ilog).to receive(:start_threaded).with(truncated_s) + .and_call_original + allow(step).to receive(:success) + sh_step(cmd) + end + end + end +end diff --git a/spec/moonshot/stack_spec.rb b/spec/moonshot/stack_spec.rb new file mode 100644 index 00000000..66db20fa --- /dev/null +++ b/spec/moonshot/stack_spec.rb @@ -0,0 +1,142 @@ +describe Moonshot::Stack do + include_context 'with a working moonshot application' + + let(:ilog) { Moonshot::InteractiveLoggerProxy.new(log) } + let(:log) { instance_double('Logger').as_null_object } + let(:parent_stacks) { [] } + let(:cf_client) { instance_double(Aws::CloudFormation::Client) } + + subject do + described_class.new('test', app_name: 'rspec-app', log: log, ilog: ilog) do |c| + c.parent_stacks = parent_stacks + end + end + + before(:each) do + allow(Aws::CloudFormation::Client).to receive(:new).and_return(cf_client) + end + + describe '#create' do + let(:step) { instance_double('InteractiveLogger::Step') } + let(:stack_exists) { false } + + before(:each) do + expect(ilog).to receive(:start).and_yield(step) + expect(subject).to receive(:stack_exists?).and_return(stack_exists) + expect(step).to receive(:success) + end + + context 'when the stack creation takes too long' do + it 'should display a helpful error message and return false' do + expect(subject).to receive(:create_stack) + expect(subject).to receive(:wait_for_stack_state) + .with(:stack_create_complete, 'created').and_return(false) + expect(subject.create).to eq(false) + end + end + + context 'when the stack creation completes in the expected time frame' do + it 'should log the process and return true' do + expect(subject).to receive(:create_stack) + expect(subject).to receive(:wait_for_stack_state) + .with(:stack_create_complete, 'created').and_return(true) + expect(subject.create).to eq(true) + end + end + + context 'when the stack already exists' do + let(:stack_exists) { true } + + it 'should log a successful step and return true' do + expect(subject).not_to receive(:create_stack) + expect(subject.create).to eq(true) + end + end + + context 'when a parent stack is specified' do + let(:parent_stacks) { ['myappdc-dc1'] } + let(:cf_client) do + stubs = { + describe_stacks: { + stacks: [ + { + stack_name: 'myappdc-dc1', + creation_time: Time.now, + stack_status: 'CREATE_COMPLETE', + outputs: [ + { output_key: 'Parent1', output_value: 'parents value' }, + { output_key: 'Parent2', output_value: 'other value' } + ] + } + ] + } + } + Aws::CloudFormation::Client.new(stub_responses: stubs) + end + let(:expected_create_stack_options) do + { + stack_name: 'test', + template_body: an_instance_of(String), + tags: [ + { key: 'ah_stage', value: 'test' } + ], + parameters: [ + { parameter_key: 'Parent1', parameter_value: 'parents value' } + ], + capabilities: ['CAPABILITY_IAM'] + } + end + + context 'when local yml file contains the override already' do + it 'should import outputs as paramters for this stack' do + expect(cf_client).to receive(:create_stack) + .with(hash_including(expected_create_stack_options)) + subject.create + + expect(File.exist?('/cloud_formation/parameters/test.yml')).to eq(true) + yaml_data = subject.overrides + expected_data = { + 'Parent1' => 'parents value' + } + expect(yaml_data).to match(expected_data) + end + end + + context 'when the local yml file does not contain the override' do + it 'should import outputs as paramters for this stack' do + File.open('/cloud_formation/parameters/test.yml', 'w') do |fp| + data = { + 'Parent1' => 'Existing Value!' + } + YAML.dump(data, fp) + end + expected_create_stack_options[:parameters][0][:parameter_value] = 'Existing Value!' + expect(cf_client).to receive(:create_stack) + .with(hash_including(expected_create_stack_options)) + subject.create + + expect(File.exist?('/cloud_formation/parameters/test.yml')).to eq(true) + yaml_data = subject.overrides + expected_data = { + 'Parent1' => 'Existing Value!' + } + expect(yaml_data).to match(expected_data) + end + end + end + end + + describe '#template_file' do + it 'should return the template file path' do + path = File.join(Dir.pwd, 'cloud_formation', 'rspec-app.json') + expect(subject.template_file).to eq(path) + end + end + + describe '#parameters_file' do + it 'should return the parameters file path' do + path = File.join(Dir.pwd, 'cloud_formation', 'parameters', 'test.yml') + expect(subject.parameters_file).to eq(path) + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 00000000..6576c771 --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,27 @@ +if ENV['COVERAGE'] + require 'simplecov' + + SimpleCov.start do + add_filter '/spec/' + end +end + +# RSpec brings this in ad-hoc, but if it comes in after fakefs we get +# superclass mismatch errors. +require 'pp' +require 'moonshot' +require 'fakefs/spec_helpers' + +shared_examples 'with a working moonshot application' do + include FakeFS::SpecHelpers + + before(:all) do + # Force aws-sdk to load metadata before FakeFS takes over. + Aws::CloudFormation::Client.new + end + + before(:each) do + FileUtils.mkdir_p '/cloud_formation/parameters' + FakeFS::FileSystem.clone(File.join(File.dirname(__FILE__), 'fs_fixtures'), '/') + end +end