Added Feature to generate report in JSON format

core/display_results.py

@@ -1,12 +1,16 @@
 import json
 import logging
 import os
-
 from jinja2 import Environment, PackageLoader
 from tinydb import Query
-
 from core.utility import object_id_to_directory_name
 
+try:
+    to_unicode = unicode
+except NameError:
+    to_unicode = str
+
+
 logging.basicConfig(filename="log.txt")
 env = Environment(loader=PackageLoader('assets', 'templates'), extensions=['jinja2.ext.do'])
 
@@ -20,6 +24,10 @@ def pretty_print(dict):
 
 env.filters['pretty_print'] = pretty_print
 
+class Dump(object):
+    def __init__(self,obj):
+        for attr in dir(obj):
+            print "obj.%s = %s" % (attr, getattr(obj, attr))
 
 def display_results(db, projectId):
     def add_to_dropdown(category):
@@ -30,13 +38,29 @@ def add_to_dropdown(category):
         return rules
 
     def generate_entities_page(category, header, fields, dropdowns):
+        reports = []
         try:
             records = db.table(category).all()
             template = env.get_template("entity_template.html")
             entity_file_path = "Report Output/" + object_id_to_directory_name(projectId) + "/Information/" + category + "s.html"
             entity_dir_path = os.path.dirname(entity_file_path)
             if not os.path.isdir(entity_dir_path):
                 os.makedirs(entity_dir_path)
+
+            for r in records:
+                data = dict()
+                i = dict()
+
+                # cast record from tinydb document to dict
+                for k, v in r.iteritems():
+                    data[k] = v
+                i['project'] = projectId
+                i['check'] = "configs"
+                i['data'] = data
+                i['category'] = category
+                i['type'] = 'PASS'
+                reports.append(i)
+
             file = open(entity_file_path, "w+")
             file_content = None
             try:
@@ -49,7 +73,10 @@ def generate_entities_page(category, header, fields, dropdowns):
         except Exception as e:
             print("Error generating output file '%s': %s" % (entity_file_path, e))
 
+        return reports
+
     def generate_findings_page(category, header, fields, dropdowns):
+        reports = []
         try:
             for rule_title in dropdowns[category]:
                 rule = db.table("Rule").get(Query().title == rule_title)
@@ -61,21 +88,33 @@ def generate_findings_page(category, header, fields, dropdowns):
                 finding_dir_path = os.path.dirname(finding_file_path)
                 if not os.path.isdir(finding_dir_path):
                     os.makedirs(finding_dir_path)
+
+                i = dict()
+                i['project'] = projectId
+                i['check'] = rule_title
+                i['data'] = findings
+                i['category'] = category
+                i['type'] = 'WARNING'
+                reports.append(i)
+
                 file = open(finding_file_path, "w+")
                 file_content = None
                 try:
                     file_content = template.render(
                         **{"records": findings, "dropdowns": dropdowns, "header": header, "fields": fields,
                            "text": rule['title']})
                 except Exception as e:
-                    print("Error rendering output file '%s': %s" % (entity_file_path, e))
+                    print("Error rendering output file '%s': %s" % (finding_file_path, e))
                 if file_content:
                     file.write(file_content)
                 file.close()
+
         except KeyError as ke:
             pass
         except Exception as e:
-            print("Error generating output file '%s': %s" % (entity_file_path, e))
+            print("Error generating output file '%s': %s" % (finding_file_path, e))
+
+        return reports
 
     # The following would place only categories with findings in the navbar
     # for finding in findings_table.all():
@@ -85,15 +124,39 @@ def generate_findings_page(category, header, fields, dropdowns):
 
     # The header is the name of the field that will be bolded
     def generate_pages(category, header, fields, dropdowns):
+
         try:
-            generate_findings_page(category, header, fields, dropdowns)
+            findings_report = generate_findings_page(category, header, fields, dropdowns)
         except Exception as e:
             logging.exception("findings page")
         try:
-            generate_entities_page(category, header, fields, dropdowns)
+            entities_report = generate_entities_page(category, header, fields, dropdowns)
         except Exception as e:
             logging.exception("entities page")
 
+        reports = []
+        for i in findings_report:
+            reports.append(i)
+        for i in entities_report:
+            reports.append(i)
+        return reports
+
+    def process_log(report):
+        if len(report) > 0:
+            current_report = []
+            for i in report:
+                report_path = "Report Output/" + i['project'] + "/reports.json"
+                if os.path.exists(report_path):
+                    with open(report_path, 'r') as log:
+                        current_report = json.load(log)
+                        current_report.append(i)
+                    with open(report_path, 'w') as log:
+                        json.dump(current_report, log)
+                else:
+                    current_report.append(i)
+                    with open(report_path, 'w') as log:
+                        json.dump(current_report, log)
+
     # Navbar dropdowns
     dropdowns = {}
     categories = set([])
@@ -102,17 +165,19 @@ def generate_pages(category, header, fields, dropdowns):
         categories.add(rule['category'])
     for category in categories:
         dropdowns[category] = add_to_dropdown(category)
-    generate_pages("Bucket", "name", ["selfLink", "location", "storageClass", "acls", "defacls"], dropdowns)
-    generate_pages("Firewall", "name",
+    process_log(generate_pages("Bucket", "name", ["selfLink", "location", "storageClass", "acls", "defacls"], dropdowns))
+
+    process_log(generate_pages("Firewall", "name",
                    ["network", "sourceRanges", "sourceTags","direction", "destinationRanges", "allowed", "description", "targetTags",
-                    "affectedInstances"], dropdowns)
-    generate_pages("Network", "selfLink", ["name", "description", "firewallRules", "members", "subnetworks"], dropdowns)
-    generate_pages("Subnet", "selfLink", ["name", "region", "network", "ipCidrRange", "gatewayAddress", "enableFlowLogs", "privateIpGoogleAccess"], dropdowns)
-    generate_pages("Role", "role", ["members"], dropdowns)
-    generate_pages("Compute Engine", "name", ["selfLink", "machineType", "status", "startRestricted", "serviceAccounts", "networkInterfaces", "disks", "tags"], dropdowns)
-    generate_pages("SQL Instance", "name", 
+                    "affectedInstances"], dropdowns))
+    process_log(generate_pages("Network", "selfLink", ["name", "description", "firewallRules", "members", "subnetworks"], dropdowns))
+    process_log(generate_pages("Subnet", "selfLink", ["name", "region", "network", "ipCidrRange", "gatewayAddress", "enableFlowLogs", "privateIpGoogleAccess"], dropdowns))
+    process_log(generate_pages("Role", "role", ["members"], dropdowns))
+    process_log(generate_pages("Compute Engine", "name", ["selfLink", "machineType", "status", "startRestricted", "serviceAccounts", "networkInterfaces", "disks", "tags"], dropdowns))
+    process_log(generate_pages("SQL Instance", "name",
                    ["selfLink", "connectionName", "databaseVersion", "ipAddress", ["settings", "dataDiskSizeGb"], ["settings", "backupConfiguration"],
-                    ["settings", "ipConfiguration"], "serviceAccountEmailAddress", "gceZone" ], dropdowns)
-    generate_pages("Service Account", "displayName", ["email", "name", "keys", "iam_policies", "uniqueId", "roles"], dropdowns)
-    generate_pages("Address", "address", ["addressType","name","purpose","subnetwork"], dropdowns)
-    generate_pages("Cluster", "name", ["nodeConfig", "addonsConfig","loggingService","privateClusterConfig","enablePrivateNodes","podSecurityPolicyConfig"], dropdowns)
+                    ["settings", "ipConfiguration"], "serviceAccountEmailAddress", "gceZone" ], dropdowns))
+    process_log(generate_pages("Service Account", "displayName", ["email", "name", "keys", "iam_policies", "uniqueId", "roles"], dropdowns))
+    process_log(generate_pages("Address", "address", ["addressType","name","purpose","subnetwork"], dropdowns))
+    process_log(generate_pages("Cluster", "name", ["nodeConfig", "addonsConfig","loggingService","privateClusterConfig","enablePrivateNodes","podSecurityPolicyConfig"], dropdowns))
+

gscout.py

@@ -152,6 +152,7 @@ def main():
             print("Error: %s" % (msg))
             logging.exception(msg)
     try:
+        from core import config
         if args.project_name :
             list_projects(project_or_org='project-name', specifier=args.project_name)
         elif args.project_id :
@@ -171,4 +172,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()