forked from davidfic/estate
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
initial work transfered from internal repo
- Loading branch information
Kyle Rockman
committed
Aug 31, 2017
1 parent
149558a
commit 733cbc4
Showing
92 changed files
with
5,245 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
{ | ||
"presets": ["stage-2","react"], | ||
"plugins": ["react-hot-loader/babel"] | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
env |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -99,3 +99,11 @@ ENV/ | |
|
||
# mypy | ||
.mypy_cache/ | ||
|
||
# Mac OSX | ||
.DS_Store | ||
|
||
# Project Specific | ||
webpack-stats.json | ||
reports | ||
local.py |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,62 @@ | ||
FROM amazonlinux:2017.03 | ||
|
||
WORKDIR /usr/local/service | ||
|
||
ENV DJANGO_SETTINGS_MODULE=estate.settings \ | ||
PYTHONPATH=/usr/local/service \ | ||
PATH=/usr/local/service/node_modules/.bin/:$PATH | ||
|
||
RUN yum update -y && \ | ||
yum install -y ca-certificates gcc libffi-devel libyaml-devel postgresql94-devel python27-devel python27-pip unzip docker git && \ | ||
mkdir -p /usr/local/service | ||
|
||
COPY ./TERRAFORM_URL.txt /usr/local/service/TERRAFORM_URL.txt | ||
RUN curl -L --silent $(cat /usr/local/service/TERRAFORM_URL.txt) > /terraform.zip && \ | ||
unzip /terraform.zip -d /bin/ && \ | ||
rm /terraform.zip | ||
|
||
ENV NODE_VERSION 6.10.2 | ||
RUN curl -sLO "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.xz" && \ | ||
tar -xJf "node-v$NODE_VERSION-linux-x64.tar.xz" -C /usr/local --strip-components=1 && \ | ||
rm "node-v$NODE_VERSION-linux-x64.tar.xz" | ||
|
||
RUN pip install coreapi==2.3.0 \ | ||
boto3==1.4.4 \ | ||
dj-database-url==0.4.1 \ | ||
Django==1.10.7 \ | ||
django-braces==1.11.0 \ | ||
django-crispy-forms==1.6.1 \ | ||
django-cors-headers==2.0.2 \ | ||
django-extensions==1.7.8 \ | ||
django-filter==1.0.2 \ | ||
django-permanent==1.1.6 \ | ||
django-rest-swagger==2.1.2 \ | ||
django-simple-history==1.9.0 \ | ||
django-storages==1.5.2 \ | ||
django-webpack-loader==0.4.1 \ | ||
djangorestframework==3.6.3 \ | ||
gevent==1.2.1 \ | ||
gunicorn==19.7.1 \ | ||
hvac==0.2.17 \ | ||
Jinja2==2.9.6 \ | ||
markdown==2.6.8 \ | ||
psycopg2==2.7.1 \ | ||
pyhcl==0.3.5 \ | ||
python-consul==0.7.0 \ | ||
python-memcached==1.58 \ | ||
raven==6.1.0 \ | ||
semantic_version==2.6.0 \ | ||
structlog==17.1.0 \ | ||
whitenoise==3.3.0 && \ | ||
pip install --global-option="--with-libyaml" pyyaml==3.12 | ||
|
||
COPY ./package.json /usr/local/service/package.json | ||
RUN npm install | ||
|
||
COPY ./.babelrc /usr/local/service/.babelrc | ||
COPY ./webpack /usr/local/service/webpack | ||
COPY ./estate /usr/local/service/estate | ||
|
||
RUN webpack --bail --config webpack/webpack.prod.config.js && django-admin collectstatic --noinput | ||
|
||
CMD [ "gunicorn", "--config", "python:estate.gunicorn", "estate.wsgi"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,2 +1,131 @@ | ||
# estate | ||
Terraform UI | ||
Estate (Terraform UX) | ||
===================== | ||
|
||
|
||
**Latin:** Status **Old French:** estat **English:** Esate | ||
|
||
> a piece of landed property or status of an owner, considered with respect to property, especially one of large extent with an elaborate house on it | ||
Estate is essentially at Terraform UI/UX experiance that makes Terraform easier for everyone to use. | ||
|
||
It it designed around a some key principles: | ||
|
||
* Self-service infrastructure as code for everyone | ||
* Reduce the learning curve of Terraform | ||
* Make the right way the easy thing to do | ||
* Standarize usage of Terraform across an organiztion | ||
* Get out of the way of a power user limiting impact on their productivity | ||
* Make management of Terraform easier | ||
|
||
This project was presented at [HashiConf 2017 in Austin](https://www.hashiconf.com/talks/underarmour-terraform.html) | ||
|
||
The slides for the presentation can be found [here](http://slides.com/rocktavious/estate#/) | ||
|
||
Getting Started & Bootstraping | ||
------------------------------ | ||
|
||
(Only for AWS Users) There is a Terraform file in the root of the repository that will provision the necessary AWS resources to run Estate | ||
|
||
For those who arn't using AWS or have their own deployment tooling as long as it can run a docker container then you can stand this puppy up. | ||
|
||
``` | ||
docker pull underarmourconnectedfitness/estate:master | ||
docker run --privileged \ | ||
-p 9200:9200 \ | ||
-e SECRET_KEY=super_secret \ | ||
-e DATABASE_URL=postgres://username:[email protected]:5432/estate \ | ||
-v /var/run/docker.sock:/var/run/docker.sock \ | ||
underarmourconnectedfitness/estate:master | ||
``` | ||
|
||
The only requirements that Estate has are: | ||
* The `DATABASE_URL` which leverages the [Django Database URL plugin](https://github.com/kennethreitz/dj-database-url) style confirguration, so if you'd like to use MySQL you can easily | ||
* The Django [SECRET_KEY](https://docs.djangoproject.com/en/1.11/ref/settings/#std:setting-SECRET_KEY) variable | ||
* The docker socket is needed because Estate runs terraform in the context of another other docker container that it spins up on demand - the docker socket requires the docker container to run in privileged mode | ||
|
||
Configuration | ||
------------- | ||
|
||
Estate is a Django application, this means it can have complex configuration, many plugins added to it which add additional features and configuration. As such we've tried to keep the core configuration needed down to just environment variables. That being said we want to make it's configuration as flexiable and pluggable as possible so we've exposed a way to plugin a normal django configuration file as well. | ||
|
||
The main environment variables that Estate will pickup are as follows: | ||
|
||
* **TERRAFORM_DOCKER_IMAGE**: Specify the docker container to use as a context to run terraform in (Default: `underarmourconnectedfitness/estate:master`) | ||
* **TERRAFORM_EXTRA_ARGS**: Extra commandline arguments that will be applied to every terraform command, except for experiment functionality (Default: `-input=false`) | ||
* **TERRAFORM_INIT_EXTRA_ARGS**: Extra commandline arguments that will be applied only the `terraform init` command (Default ``) | ||
* **TERRAFORM_PLAN_EXTRA_ARGS**: Extra commandline arguments that will be applied only to the `terraform plan` command (Default: `-detailed-exitcode -out=plan`) | ||
* **TERRAFORM_APPLY_EXTRA_ARGS**: Extra commandline arguments that will be applied only to the `terraform apply` command (Default: ``) | ||
|
||
The following can only be applied as environment variables | ||
|
||
* **GUNICORN_BIND_ADDRESS**: The network interface to bind to (Default: `0.0.0.0`) | ||
* **GUNICORN_BIND_PORT**: The network port to bind to (Default: `8000`) | ||
* **GUNICORN_WORKER_COUNT**: The amount of gunicorn workers to run (Default: `<cpu_count> * 10 + 1`) | ||
* **GUNICORN_WORKER_CLASS**: See the gunicorn documentation on worker classes for more information (Default: "gevent") | ||
* **GUNICORN_LOG_LEVEL**: See the gunicorn documenation on log levels for more information (Default: "info") | ||
|
||
|
||
As well you can configure a django settings file, which is just pure python, and mount it into the container | ||
|
||
contents of custom.py | ||
``` | ||
from . import INSTALLED_APPS, MIDDLEWARE | ||
# Add other django apps - IE Sentry | ||
INSTALLED_APPS += [ | ||
'raven.contrib.django.raven_compat', | ||
] | ||
MIDDLEWARE = ( | ||
'raven.contrib.django.raven_compat.middleware.Sentry404CatchMiddleware', | ||
) + MIDDLEWARE | ||
# Configure estate settings as well | ||
TERRAFORM_INIT_EXTRA_ARGS = "-input=false -backend-config 'access_token=6ae45dff-1272-4v75-8gd7-ad52bd756e66' -backend-config 'scheme=https' -backend-config 'address=consul.example.com' -backend-config 'path=estate/remote_state/{NAMESPACE}'" | ||
``` | ||
|
||
Then mount this file into the container at the path `/usr/local/service/estate/settings/custom.py` | ||
``` | ||
docker run -v ./custom.py:/usr/local/service/estate/settings/custom.py underarmourconnectedfitness/estate:master | ||
``` | ||
|
||
Running as a Cluster | ||
-------------------- | ||
|
||
Estate by default is setup to only run as a single standalone service, but as your team grows you'll likely need to scale it horizontally. This is quite easy with estate it just requries 1 thing - a cache database | ||
|
||
Estate uses a cache database to store the output of the different terraform commands run, by default it stores them on disk inside the container, but when you start to cluster Estate this won't work, so you will need to set up something like redis or memcached and configure the Django [cache framework](https://docs.djangoproject.com/en/1.11/topics/cache/) to store the cache data in the database. | ||
|
||
Sentry Integration | ||
------------------ | ||
|
||
Sentry integration is a first class citizen integration with Estate. There is only one variable you'll need to configure to connect to your sentry cluster | ||
|
||
* **SENTRY_DSN**: You can view the sentry documentation about DSN's [here](https://docs.sentry.io/quickstart/#configure-the-dsn) | ||
|
||
Developing Terraform | ||
-------------------- | ||
|
||
If you wish to hack on Estate, you'll first need to understand its architecture. | ||
|
||
Estate is a single docker container that runs a [Django](https://www.djangoproject.com/) application with [Gunicorn](http://gunicorn.org/) workers. The backend leverages [Django Rest Framework](http://www.django-rest-framework.org/) to design it REST API functionality. The frontend is compiled by [Webpack](https://webpack.github.io/) using a standard single page app design that leverages [React](https://facebook.github.io/react/) + [Redux](http://redux.js.org/). | ||
|
||
Local development has been made a breeze, and long build/compile times have been reduced as much as possible. To get started use [Git](https://git-scm.com/) to clone this repository and run `docker-compose build` from the root of the repository. Once the build has completed you only need to run this command again if you change the Dockerfile itself, from here on out any changes you make to the codebase will be detected and use hot-reloading techniques to update the running application. | ||
|
||
To start up the application just use `docker-compose up` this will spin up a series of containers, as well as Estate itself, and then you can begin editing the code. | ||
|
||
Contributing | ||
------------ | ||
|
||
* The master branch is meant to be stable. I usually work on unstable stuff on a personal branch. | ||
* Fork the master branch ( https://github.com/underarmour/estate/fork ) | ||
* Create your branch (git checkout -b my-branch) | ||
* Commit your changes (git commit -am 'added fixes for something') | ||
* Push to the branch (git push origin my-branch) | ||
* Create a new Pull Request (Travis CI will test your changes) | ||
* And you're done! | ||
|
||
Features, Bug fixes, bug reports and new documentation are all appreciated! | ||
See the github issues page for outstanding things that could be worked on. | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
https://releases.hashicorp.com/terraform/0.9.11/terraform_0.9.11_linux_amd64.zip |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
# Terraform Modules | ||
|
||
This folder contains modules for Terraform that can setup Estate for | ||
various systems. The infrastructure provider that is used is designated | ||
by the folder above. See the `variables.tf` file in each for more documentation. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
output "server_address" { | ||
value = "${aws_instance.estate.0.public_dns}" | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,150 @@ | ||
resource "aws_vpc" "estate" { | ||
cidr_block = "10.0.0.0/16" | ||
|
||
tags { | ||
Name = "${var.tagName}-VPC" | ||
} | ||
} | ||
|
||
resource "aws_subnet" "estate" { | ||
vpc_id = "${aws_vpc.estate.id}" | ||
cidr_block = "10.0.0.0/24" | ||
availability_zone = "${var.region}a" | ||
|
||
tags { | ||
Name = "${var.tagName}-SUBNET" | ||
} | ||
} | ||
|
||
resource "aws_security_group" "estate" { | ||
name = "estate_sg" | ||
description = "Estate" | ||
vpc_id = "${aws_vpc.estate.id}" | ||
|
||
tags { | ||
Name = "${var.tagName}-SG" | ||
} | ||
} | ||
|
||
resource "aws_security_group_rule" "estate_self" { | ||
type = "ingress" | ||
from_port = 0 | ||
to_port = 0 | ||
protocol = "-1" | ||
self = true | ||
security_group_id = "${aws_security_group.estate.id}" | ||
} | ||
|
||
resource "aws_security_group_rule" "estate_ssh" { | ||
type = "ingress" | ||
from_port = 22 | ||
to_port = 22 | ||
protocol = "tcp" | ||
cidr_blocks = ["0.0.0.0/0"] | ||
security_group_id = "${aws_security_group.estate.id}" | ||
} | ||
|
||
resource "aws_security_group_rule" "estate_outbound" { | ||
type = "egress" | ||
from_port = 0 | ||
to_port = 0 | ||
protocol = "-1" | ||
cidr_blocks = ["0.0.0.0/0"] | ||
security_group_id = "${aws_security_group.estate.id}" | ||
} | ||
|
||
resource "aws_db_subnet_group" "estate" { | ||
name = "estate_rds_sng" | ||
subnet_ids = ["${aws_subnet.estate.id}"] | ||
|
||
tags { | ||
Name = "${var.tagName}-RDS" | ||
} | ||
} | ||
|
||
resource "aws_db_instance" "estate" { | ||
identifier = "estate_db" | ||
allocated_storage = "10" | ||
storage_type = "gp2" | ||
engine = "postgres" | ||
engine_version = "9.5.4" | ||
instance_class = "db.m3.medium" | ||
name = "estate" | ||
username = "${db_user}" | ||
password = "${db_password}" | ||
vpc_security_group_ids = ["${aws_security_group.estate.name}"] | ||
db_subnet_group_name = ${aws_db_subnet_group.estate.id} | ||
skip_final_snapshot = "true" | ||
backup_retention_period = 0 | ||
copy_tags_to_snapshot = "true" | ||
multi_az = "true" | ||
apply_immediately = "true" | ||
maintenance_window = "wed:04:30-wed:05:30" | ||
tags { | ||
Name = "${var.tagName}-RDS" | ||
} | ||
} | ||
|
||
resource "aws_elasticache_parameter_group" "estate" { | ||
name = "estate_parameter_group" | ||
family = "memcached1.4" | ||
|
||
} | ||
|
||
resource "aws_elasticache_subnet_group" "estate" { | ||
name = "estate_elasticache_sng" | ||
description = "Estate" | ||
subnet_ids = ["${aws_subnet.estate.id}"] | ||
} | ||
|
||
resource "aws_elasticache_cluster" "estate" { | ||
cluster_id = "estate" | ||
engine = "memcached1.4" | ||
node_type = "cache.m3.medium" | ||
num_cache_nodes = 2 | ||
port = 11211 | ||
subnet_group_name = "${aws_elasticache_subnet_group}.estate.name" | ||
security_group_ids = ["${aws_security_group.estate.name}"] | ||
parameter_group_name = "${aws_elasticache_parameter_group.estate.name}" | ||
az_mode = "cross-az" | ||
maintenance_window = "wed:04:30-wed:05:30" | ||
tags { | ||
Name = "${var.tagName}-RDS" | ||
} | ||
} | ||
|
||
data "user_data" "estate" { | ||
template = "${file("${path.module}/../shared/scripts/bootstrap.sh")}" | ||
|
||
vars { | ||
db_url = "postgres://${var.db_user}:${var.db_password}@${aws_db_instance.estate.endpoint}/estate" | ||
} | ||
} | ||
|
||
resource "aws_instance" "estate" { | ||
count = "${var.servers}" | ||
ami = "${lookup(var.ami, "${var.region}")}" | ||
instance_type = "${var.instance_type}" | ||
key_name = "${var.key_name}" | ||
security_groups = ["${aws_security_group.estate.name}"] | ||
subnet_id = ["${aws_subnet.estate.id}"] | ||
|
||
ebs_optimized = true | ||
disable_api_termination = true | ||
root_block_device { | ||
volume_type = gp2 | ||
volume_size = "20" | ||
delete_on_termination = true | ||
} | ||
|
||
connection { | ||
user = "${var.user}" | ||
private_key = "${file("${var.key_path}")}" | ||
} | ||
|
||
user_data = "${data.user_data.estate.rendered}" | ||
|
||
tags { | ||
Name = "${var.tagName}-${count.index}" | ||
} | ||
} |
Oops, something went wrong.