enable automatic statefile management, fix some docker executor bugs

estate/assets/js/api/terraform.js

@@ -182,6 +182,18 @@ export function doApplyForNamespace(id, plan_hash) {
     return req
 }
 
+export function getStateForNamespace(id) {
+    const req = axios.get(`/api/terraform/state/?namespace=${id}`)
+    req.then((res) => {
+        console.log(res)
+        dispatch({
+            type: "UPDATE_STATEFILE",
+            payload: res.data[0]
+        })
+    }, messages.handleResponseError)
+    return req
+}
+
 export function getTemplates(page, pagesize, search) {
     dispatch({ type: "LOADING_TEMPLATES"})
     const req = axios.get(`/api/terraform/template/?page=${page}&page_size=${pagesize}&search=${search}`)

estate/assets/js/components/TerraformNamespaceAddFileModal.jsx

@@ -113,13 +113,13 @@ let mapDispatchToProps = (dispatch, ownProps) => {
         addFileToNamespace: (payload) => {
             var req = api.addFileToNamespace(payload)
             req.then((res) => {
-                ownProps.history.push( urljoin(ownProps.url, "/file/", res.data.slug, "/") )
+                ownProps.history.push( urljoin(ownProps.url, "/file/", res.data.slug + "/") )
             })
         },
         addTemplateToNamespace: (payload) => {
             var req = api.addTemplateToNamespace(payload)
             req.then((res) => {
-                ownProps.history.push( urljoin(ownProps.url, "/template/", res.data.slug, "/") )
+                ownProps.history.push( urljoin(ownProps.url, "/template/", res.data.slug + "/") )
             })
         }
     }

estate/assets/js/components/TerraformNamespaceItem.jsx

@@ -33,6 +33,7 @@ class TerraformNamespaceItem extends React.Component {
             const namespace = nextProps.namespace
             nextProps.getPlan(namespace.pk)
             nextProps.getApply(namespace.pk)
+            nextProps.getState(namespace.pk)
             this.mergeFiles(namespace)
             this.mergeTemplates(namespace)
         }
@@ -404,6 +405,18 @@ class TerraformNamespaceItem extends React.Component {
             </pre>
         )
     }
+    createStatePane() {
+        var data = this.props.stateObject
+        var output = join(data.content, "")
+        return (
+            <div>
+                <p>If you need to edit this please contact an administrator</p>
+                <pre className={"col-xs-12 "}>
+                    {output}
+                </pre>
+            </div>
+        )
+    }
     render() {
         if (this.props.namespace == null) {
             return null
@@ -449,6 +462,9 @@ class TerraformNamespaceItem extends React.Component {
                                 <ReactTooltip id="terraform_apply" className="ReactTooltipHoverDelay" delayHide={10} effect='solid'/>
                             </div>
                         </li>
+                        <li>
+                            <NavLink className="col-xs-11" to={ urljoin(url, "/state") } exact activeStyle={{fontWeight: "bold", color: "white", backgroundColor: "#337ab7"}} onClick={this.props.getState.bind(this, namespace.pk)}>Statefile</NavLink>
+                        </li>
                     </ul>
                     <ul className="nav nav-sidebar">
                         <li>
@@ -480,6 +496,7 @@ class TerraformNamespaceItem extends React.Component {
                     <div className="row">
                         <Route path={`${url}/plan`} render={this.createPlanPane.bind(this)} />
                         <Route path={`${url}/apply`} render={this.createApplyPane.bind(this)} />
+                        <Route path={`${url}/state`} render={this.createStatePane.bind(this)} />
                         <Route path={`${url}/file/:file`} render={this.createFilePane.bind(this)} />
                         <Route path={`${url}/template/:template`} render={this.createTemplatePane.bind(this)} />
                     </div>
@@ -509,8 +526,10 @@ const mapStateToProps = (state, ownProps) => {
         updateTemplateOfTemplateInstance: terraform.updateTemplateOfTemplateInstance,
         planOutput: state.terraform.planOutput,
         applyOutput: state.terraform.applyOutput,
+        stateObject: state.terraform.stateObject,
         getPlan: terraform.getPlanForNamespace,
         getApply: terraform.getApplyForNamespace,
+        getState: terraform.getStateForNamespace,
     }
 }
 const mapDispatchToProps = (dispatch, ownProps) => {

estate/assets/js/components/TerraformNamespaceList.jsx

@@ -1,5 +1,6 @@
 import { connect } from "react-redux"
 import DashboardListView from "./DashboardListView"
+import urljoin from "url-join"
 import * as api from "../api/terraform"
 
 const TerraformNamespacesTableColumns = [
@@ -51,7 +52,7 @@ let mapDispatchToProps = (dispatch, ownProps) => {
         createObject: (payload) => {
             const req = api.createNamespace(payload)
             req.then((res) => {
-                ownProps.history.push("./namespaces/" + res.data.slug)
+                ownProps.history.push(  urljoin(ownProps.location.pathname, res.data.slug + "/"))
             })
         }
     }

estate/assets/js/components/TerraformTemplateList.jsx

@@ -1,5 +1,6 @@
 import { connect } from "react-redux"
 import DashboardListView from "./DashboardListView"
+import urljoin from "url-join"
 import * as api from "../api/terraform"
 
 const TerraformTemplatesTableColumns = [
@@ -58,7 +59,7 @@ let mapDispatchToProps = (dispatch, ownProps) => {
         createObject: (payload) => {
             const req = api.createTemplate(payload)
             req.then((res) => {
-                ownProps.history.push("./templates/" + res.data.slug)
+                ownProps.history.push(  urljoin(ownProps.location.pathname, res.data.slug + "/") )
             })
         }
     }

estate/assets/js/reducers/terraform.js

@@ -12,6 +12,7 @@ var initialState = {
     namespacesPages: 0,
     planOutput: "",
     applyOutput: "",
+    stateObject: {},
     files: [],
     templates: [],
     renderedTemplate: "{}",
@@ -73,6 +74,11 @@ export default createReducer(initialState, {
         return state
     },
 
+    ["UPDATE_STATEFILE"]: (state, action) => {
+        state = set(["stateObject"])(action.payload)(state)
+        return state
+    },
+
     ["UPDATE_FILE"]: (state, action) => {
         var index = findIndex(state.file, {"pk": action.payload.pk})
         if (index != -1){

estate/core/HotDockerExecutor.py

@@ -147,7 +147,8 @@ def run(self):
             end = datetime.datetime.now()
             self.duration = end - start
             self.finish()
-            shutil.rmtree(self.workdir)
+            if os.path.exists(self.workdir):
+                shutil.rmtree(self.workdir)
 
     def get_escrow(self, escrow_id):
         escrow_api = os.environ.get("ESCROW_API_URI")
@@ -193,20 +194,22 @@ def write_execute_files(self):
 
     def pull_image(self):
         if self.streamer is not None:
-            self.streamer.log("Pulling docker image: {0}\n".format(self.docker_image))
+            self.streamer.log("Pulling docker image this may take a while...\n")
+        os.makedirs(self.workdir)
         command = ["docker", "pull", self.docker_image]
         exit_code, output = self.execute_command(command)
         if exit_code != 0:
-            raise Exception("".join(output + "\nExit Code: {0}".format(exit_code)))
+            raise Exception(output)
 
     def execute_command(self, command, capture=False):
         output = []
         stream_index = 0
-        LOG.info("[HotDockerExecutor] Running command: {0}".format(" ".join(command)))
+        LOG.info("[HotDockerExecutor] Running command: '{0}' in directory '{1}'".format(" ".join(command), self.workdir))
         process = subprocess.Popen(
             command,
             stderr=subprocess.STDOUT,
             stdout=subprocess.PIPE,
+            cwd=self.workdir,
         )
         if capture and self.streamer is not None:
             self.streamer.log("Started Execution @ {0}\n".format(datetime.datetime.utcnow()))
@@ -222,10 +225,10 @@ def execute_command(self, command, capture=False):
                 self.streamer.log(new_line)
         process.communicate()
         exit_code = process.poll()
+        if capture and self.streamer is not None:
+            self.streamer.log("\nExit Code: {0}\n".format(exit_code))
         if capture and self.streamer is not None:
             self.streamer.log("Completed Execution @ {0}".format(datetime.datetime.utcnow()), running=False, exit_code=exit_code)
-        if self.streamer is not None:
-            self.streamer.log("\nExit Code: {0}".format(exit_code))
         return exit_code, "".join(output)
 
     # These are the user implementation points in the overall run flow

estate/core/models/base.py

@@ -22,5 +22,8 @@ class EstateAbstractBase(PermanentModel, TimeStampedModel, TitleDescriptionModel
     class Meta(TimeStampedModel.Meta):
         abstract = True
 
+    def __unicode__(self):
+        return self.title
+
     def __repr__(self):
         return "<%s:%s pk:%i>" % (self.__class__.__name__, self.title, self.pk)

estate/terraform/admin/__init__.py

@@ -2,3 +2,4 @@
 from .namespace import * # NOQA
 from .file import * # NOQA
 from .template import * # NOQA
+from .state import * # NOQA

estate/terraform/admin/state.py

@@ -0,0 +1,15 @@
+from __future__ import absolute_import
+from django.contrib import admin
+from django.apps import apps
+
+State = apps.get_model('terraform.State')
+
+
+class StateAdmin(admin.ModelAdmin):
+    list_display = ['pk', 'title', 'description', 'modified']
+    list_filter = ['title']
+    search_fields = ['slug', 'title']
+    list_per_page = 10
+
+
+admin.site.register(State, StateAdmin)

estate/terraform/migrations/0007_historicalstate_state.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.7 on 2017-08-31 21:38
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+import estate.core.models.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('terraform', '0006_auto_20170804_1357'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='HistoricalState',
+            fields=[
+                ('id', models.IntegerField(auto_created=True, blank=True, db_index=True, verbose_name='ID')),
+                ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
+                ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
+                ('title', models.CharField(max_length=255, verbose_name='title')),
+                ('description', models.TextField(blank=True, null=True, verbose_name='description')),
+                ('deleted', models.DateTimeField(blank=True, default=None, editable=False, null=True)),
+                ('content', models.TextField(blank=True, verbose_name='content')),
+                ('history_id', models.AutoField(primary_key=True, serialize=False)),
+                ('history_date', models.DateTimeField()),
+                ('history_change_reason', models.CharField(max_length=100, null=True)),
+                ('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
+                ('history_user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)),
+                ('namespace', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='terraform.Namespace')),
+            ],
+            options={
+                'ordering': ('-history_date', '-history_id'),
+                'get_latest_by': 'history_date',
+                'verbose_name': 'historical state',
+            },
+        ),
+        migrations.CreateModel(
+            name='State',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
+                ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
+                ('title', models.CharField(max_length=255, verbose_name='title')),
+                ('description', models.TextField(blank=True, null=True, verbose_name='description')),
+                ('deleted', models.DateTimeField(blank=True, default=None, editable=False, null=True)),
+                ('slug', estate.core.models.fields.SoftDeleteAwareAutoSlugField(blank=True, editable=False, populate_from=b'title', verbose_name='slug')),
+                ('content', models.TextField(blank=True, verbose_name='content')),
+                ('namespace', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='state', to='terraform.Namespace')),
+            ],
+            options={
+                'ordering': ('-modified', '-created'),
+                'abstract': False,
+                'get_latest_by': 'modified',
+            },
+        ),
+    ]

estate/terraform/models/__init__.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
 from .file import * # NOQA
 from .template import * # NOQA
-from .namespace import * # NOQA
+from .namespace import * # NOQA
+from .state import * # NOQA

estate/terraform/models/state.py

@@ -0,0 +1,13 @@
+import logging
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+from ...core.models.base import EstateAbstractBase, HistoricalRecordsWithoutDelete
+
+LOG = logging.getLogger(__name__)
+
+
+class State(EstateAbstractBase):
+    content = models.TextField(_("content"), blank=True)
+    namespace = models.ForeignKey("terraform.Namespace", related_name="state")
+
+    history = HistoricalRecordsWithoutDelete(excluded_fields=['slug'])

estate/terraform/terraform.py

@@ -40,10 +40,11 @@ def save_plan(self, plan_hash, plan_data):
 
 class Terraform(HotDockerExecutor):
 
-    def __init__(self, action, namespace, plan_hash=None):
+    def __init__(self, action, namespace, plan_hash=None, state_obj=None):
         self.action = action
         self.namespace = namespace
         self.plan_hash = plan_hash
+        self.state_obj = state_obj
         config = {
             "docker_image": settings.TERRAFORM_DOCKER_IMAGE,
             "name": self.namespace.slug,
@@ -67,43 +68,57 @@ def run(self, *args, **kwargs):
         super(Terraform, self).run(*args, **kwargs)
 
     def write_files(self):
-        if self.streamer is not None:
-            self.streamer.log("Preparing Namespace '{0}' for action '{1}'\n".format(self.namespace.title, self.action))
         LOG.info("[Terraform] Preparing Namespace '{0}' for action '{1}'".format(self.namespace.title, self.action))
         if self.action == "plan":
+            if self.state_obj:
+                LOG.info("[Terraform] Writing terraform statefile")
+                path = os.path.join(self.workdir, "terraform.tfstate")
+                with open(path, "wb") as f:
+                    f.write(self.state_obj.content)
             for item in self.namespace.terraform_files:
                 path = os.path.join(self.workdir, str(item.pk) + "_" + item.slug + ".tf")
                 has_ext = HAS_EXT.search(item.title)
                 if has_ext:
                     path = os.path.join(self.workdir, item.title)
-                if self.streamer is not None:
-                    self.streamer.log("Writing terraform file: {0}\n".format(item.title))
                 LOG.info("[Terraform] Writing file: {0}".format(path))
                 with open(path, "wb") as f:
                     f.write(item.content)
         if self.action == "apply":
             if self.plan_hash is None:
                 raise Exception("Unable to perform action 'apply' no plan was found!")
+            if self.state_obj:
+                LOG.info("[Terraform] Writing terraform statefile")
+                path = os.path.join(self.workdir, "terraform.tfstate")
+                with open(path, "wb") as f:
+                    f.write(self.state_obj.content)
             path = os.path.join(self.workdir, "plan.tar.gz")
             plan_data = self.streamer.get_plan(self.plan_hash)
             if plan_data is None:
                 raise Exception("Unable to find plan data!")
             with open(path, "wb") as f:
                 f.write(plan_data)
-            exit_code, _ = self.execute_command(["tar", "-xzvf", path, "-C", self.workdir], self.workdir)
+            exit_code, _ = self.execute_command(["tar", "-xzvf", path, "-C", self.workdir])
             if exit_code != 0:
                 raise Exception("Unable to unpack plan file!")
 
     def finish(self):
         if self.action == "plan" and self.exit_code == 2:
             path = os.path.join(self.workdir, "plan.tar.gz")
-            exit_code, _ = self.execute_command(["tar", "-czvf", path, "./plan"], self.workdir)
+            exit_code, output = self.execute_command(["tar", "-czvf", path, "./plan"])
             if exit_code != 0:
-                raise Exception("Unable to save plan file!")
+                raise Exception("Unable to save plan file!\n" + output)
             with open(path, "rb") as f:
                 plan_data = f.read()
             plan_hash = hashlib.md5(plan_data).hexdigest()
             self.streamer.save_plan(plan_hash, plan_data)
+        if self.action == "apply" and self.exit_code == 0:
+            path = os.path.join(self.workdir, "terraform.tfstate")
+            exit_code, output = self.execute_command(["cat", path])
+            if exit_code != 0:
+                raise Exception("Failed to save the terraform state!\n" + output)
+            LOG.info("[Terraform] Saving Terraform State - \n" + output)
+            self.state_obj.content = output
+            self.state_obj.save()
 
     def get_stream(self):
         return self.streamer.get()

estate/terraform/urls.py

@@ -8,6 +8,7 @@
 router.register(r"template", views.TemplateApiView)
 router.register(r"templateinstance", views.TemplateInstanceApiView)
 router.register(r"namespace", views.NamespaceApiView)
+router.register(r"state", views.StateApiView)
 router.include_root_view = True
 
 urlpatterns = [

estate/terraform/views/__init__.py

@@ -2,3 +2,4 @@
 from .file import FileApiView # NOQA
 from .template import TemplateApiView, TemplateInstanceApiView # NOQA
 from .namespace import NamespaceApiView # NOQA
+from .state import StateApiView # NOQA