diff --git a/doc/sql/nightmare.sql b/doc/sql/nightmare.sql index 001a611..31f16cf 100644 --- a/doc/sql/nightmare.sql +++ b/doc/sql/nightmare.sql @@ -70,6 +70,7 @@ CREATE TABLE `crashes` ( `total_samples` int(11) NOT NULL, `additional` mediumtext, `crash_hash` varchar(48), + `status` int(1) DEFAULT 0 PRIMARY KEY (`crash_id`) ) ENGINE=InnoDB AUTO_INCREMENT=826 DEFAULT CHARSET=latin1; /*!40101 SET character_set_client = @saved_cs_client */; @@ -252,7 +253,12 @@ CREATE TABLE `triggers` ( LOCK TABLES `config` WRITE; /*!40000 ALTER TABLE `config` DISABLE KEYS */; -INSERT INTO `config` VALUES (7,'SAMPLES_PATH','/home/joxean/Documentos/research/nightmare/results',NULL,NULL),(8,'TEMPLATES_PATH','/home/joxean/Documentos/research/nightmare/samples',NULL,NULL),(9,'NIGHTMARE_PATH','/home/joxean/Documentos/research/nightmare',NULL,NULL),(10,'QUEUE_HOST','localhost',NULL,NULL),(11,'QUEUE_PORT','11300',NULL,NULL),(12,'TEMPORARY_PATH','/tmp/',NULL,NULL); +INSERT INTO `config` VALUES + (1,'NIGHTMARE_PATH','/nightmare/',NULL,NULL), + (2,'WORKING_PATH','/nightmare/data/',NULL,NULL), + (3,'TEMPORARY_PATH','/tmp/',NULL,NULL), + (4,'QUEUE_HOST','localhost',NULL,NULL), + (5,'QUEUE_PORT','11300',NULL,NULL); /*!40000 ALTER TABLE `config` ENABLE KEYS */; UNLOCK TABLES; /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; diff --git a/fuzzers/generic_fuzzer.py b/fuzzers/generic_fuzzer.py old mode 100755 new mode 100644 index 965b397..c17d072 --- a/fuzzers/generic_fuzzer.py +++ b/fuzzers/generic_fuzzer.py @@ -11,6 +11,7 @@ import json import base64 import tempfile +import zlib import ConfigParser from multiprocessing import Process, cpu_count @@ -75,7 +76,6 @@ def read_configuration(self): try: self.tube_name = parser.get(self.section, 'tube') except: - raise raise Exception("No tube specified in the configuration file for section %s" % self.section) try: @@ -198,7 +198,7 @@ def launch_sample(self, buf): os.system(self.pre_command) crash = None - for i in range(0,3): + for i in range(0, 3): try: crash = self.launch_debugger(self.timeout, self.command, filename) break @@ -222,14 +222,23 @@ def fuzz(self): value = self.q.stats_tube(self.tube_name)["current-jobs-ready"] debug("Total of %d job(s) in queue" % value) job = self.q.reserve() - buf, temp_file = json.loads(job.body) - buf = base64.b64decode(buf) + d = json.loads(job.body) + sample = zlib.decompress(base64.b64decode(d['sample'])) + temp_file = d['temp_file'] + template_hash = d['template_hash'] debug("Launching sample %s..." % os.path.basename(temp_file)) - if self.launch_sample(buf): + if self.launch_sample(sample): log("We have a crash, moving to %s queue..." % self.crash_tube) crash = self.crash_info - d = {temp_file:self.crash_info} + + d = { + "temp_file": temp_file, + "crash_info": crash, + "template_hash": template_hash, + "data": None + } + self.crash_q.put(json.dumps(d)) self.crash_info = None diff --git a/runtime/nfp_engine.py b/runtime/nfp_engine.py old mode 100755 new mode 100644 index b95dc73..5bec607 --- a/runtime/nfp_engine.py +++ b/runtime/nfp_engine.py @@ -52,8 +52,8 @@ def read_config(self): log("Configuration value %s is %s" % (row.name, row.value)) # Create the corresponding directory if it doesn't exists - if not os.path.exists(self.config["SAMPLES_PATH"]): - os.makedirs(self.config["SAMPLES_PATH"]) + if not os.path.exists(self.config["WORKING_PATH"]): + os.makedirs(self.config["WORKING_PATH"]) # In Linux, it's recommended to use /dev/shm for speed improvements if not "TEMPORARY_PATH" in self.config: @@ -92,10 +92,9 @@ def get_project_engines(self): return res def read_random_file(self, folder): - basepath = os.path.join(self.config["TEMPLATES_PATH"], folder) - files = os.listdir(basepath) + files = os.listdir(folder) filename = random.choice(files) - return os.path.join(basepath, filename) + return os.path.join(folder, filename) def get_command(self, cmd, filename, subfolder): cmd = cmd.replace("%INPUT%", '"%s"' % filename) @@ -108,15 +107,16 @@ def get_command(self, cmd, filename, subfolder): return cmd, temp_file def create_sample(self, pe): - subfolder = pe.subfolder + template_folder = os.path.join(self.config["WORKING_PATH"], pe.subfolder, "templates") tube_prefix = pe.tube_prefix command = pe.command project_id = pe.project_id mutation_engine_id = pe.mutation_engine_id - filename = self.read_random_file(subfolder) + filename = self.read_random_file(template_folder) + template_hash = os.path.basename(filename) debug("Random template file %s" % filename) - cmd, temp_file = self.get_command(command, filename, subfolder) + cmd, temp_file = self.get_command(command, filename, template_folder) log("Generating mutated file %s" % temp_file) debug("*** Command: %s" % cmd) os.system(cmd) @@ -126,8 +126,14 @@ def create_sample(self, pe): log("Putting it in queue and updating statistics...") buf = file(temp_file, "rb").read() q = get_queue(watch=False, name="%s-samples" % tube_prefix) - json_buf = json.dumps([base64.b64encode(buf), temp_file]) - q.put(json_buf) + + data = { + 'sample': base64.b64encode(zlib.compress(buf)), + 'temp_file': temp_file, + 'template_hash': template_hash + } + + q.put(json.dumps(data)) self.update_statistics(project_id, mutation_engine_id) self.update_iteration(project_id) except: @@ -181,15 +187,13 @@ def update_statistics(self, project_id, mutation_engine_id): where = "statistic_id = $id" total = self.db.update("statistics", total=row.total+1, iteration=row.iteration+1, where=where, vars=vars) - def queue_is_full(self, prefix, maximum): - tube_name = "%s-samples" % prefix + def queue_is_full(self, tube_name, maximum): q = get_queue(watch=True, name=tube_name) value = q.stats_tube(tube_name)["current-jobs-ready"] debug("Total of %d job(s) in queue" % value) return value > maximum-1 - def get_pending_elements(self, prefix, maximum): - tube_name = "%s-samples" % prefix + def get_pending_elements(self, tube_name, maximum): q = get_queue(watch=True, name=tube_name) value = q.stats_tube(tube_name)["current-jobs-ready"] debug("Total of %d job(s) in queue" % value) @@ -213,16 +217,16 @@ def remove_obsolete_files(self): log("Error removing temporary file: %s" % str(sys.exc_info()[1])) job.delete() - def calculate_crash_hash(self, data): + def calculate_crash_hash(self, crash_info): crash_hash = [] - if "additional" in data: - if "stack trace" in data["additional"]: - st = data["additional"]["stack trace"] + if "additional" in crash_info: + if "stack trace" in crash_info["additional"]: + st = crash_info["additional"]["stack trace"] last = max(map(int, st.keys())) # First element in the crash hash contains the last 3 nibbles # of the $PC. - tmp = hex(data["pc"]) + tmp = hex(crash_info["pc"]) crash_hash = [tmp[len(tmp)-3:]] # Next elements, will be the last 3 nibbles of each address in @@ -261,17 +265,21 @@ def should_store_crash(self, project_id, crash_hash): return False return True - def insert_crash(self, project_id, temp_file, data): - has_file = "has_file" in data - crash_path = os.path.join(self.config["SAMPLES_PATH"], "crashes") - if not os.path.exists(temp_file) and not has_file: + def insert_crash(self, project_id, subfolder, d): + samples_path = os.path.join(self.config["WORKING_PATH"], subfolder, "samples") + temp_file = d['temp_file'] + crash_info = d['crash_info'] + template_hash = d['template_hash'] + data = d['data'] + + if data is None and not os.path.exists(temp_file): log("Test case file %s does not exists!!!!" % temp_file) return False - elif has_file: + elif data is not None: # There is no file path but, rather, a whole zlib compressed file # encoded in base64 so, create a temporary file and write to it # the decoded base64 and decompressed zlib stream of data. - buf = temp_file + buf = data temp_file = tempfile.mktemp(dir=self.config["TEMPORARY_PATH"]) try: @@ -281,11 +289,12 @@ def insert_crash(self, project_id, temp_file, data): os.remove(temp_file) raise - buf = open(temp_file, "rb").read() - file_hash = sha1(buf).hexdigest() - new_path = os.path.join(crash_path, file_hash) + with open(temp_file, "rb") as f: + buf = f.read() - sample_id = self.db.insert("samples", sample_hash=file_hash) + file_hash = sha1(buf).hexdigest() + new_path = os.path.join(samples_path, file_hash) + sample_id = self.db.insert("samples", sample_hash=file_hash, template_hash=template_hash) what = "count(*) cnt" vars = {"id":project_id} @@ -294,35 +303,39 @@ def insert_crash(self, project_id, temp_file, data): row = res[0] total = row.cnt - crash_hash = self.calculate_crash_hash(data) + crash_hash = self.calculate_crash_hash(crash_info) store_crash = self.should_store_crash(project_id, crash_hash) if store_crash: log("Saving test file %s" % new_path) - shutil.move(temp_file, new_path) + shutil.copy(temp_file, new_path) if os.path.exists(temp_file + ".diff"): - shutil.move(temp_file + ".diff", new_path + ".diff") + shutil.copy(temp_file + ".diff", new_path + ".diff") with self.db.transaction(): - log("Inserting crash $PC 0x%08x Signal %s Exploitability %s Hash %s" % (data["pc"], data["signal"], data["exploitable"], crash_hash)) - if data["disasm"] is not None: - disasm = "%08x %s" % (data["disasm"][0], data["disasm"][1]) + log("Inserting crash $PC 0x%08x Signal %s Exploitability %s Hash %s" % + (crash_info["pc"], crash_info["signal"], crash_info["exploitable"], crash_hash)) + if crash_info["disasm"] is not None: + disasm = "%08x %s" % (crash_info["disasm"][0], crash_info["disasm"][1]) else: disasm = "None" - additional_info = json.dumps(data["additional"]) + additional_info = json.dumps(crash_info["additional"]) if store_crash: self.db.insert("crashes", project_id=project_id, sample_id=sample_id, - program_counter=data["pc"], crash_signal=data["signal"], - exploitability=data["exploitable"], - disassembly=disasm, total_samples=total, - additional = str(additional_info), - crash_hash = crash_hash) + program_counter=crash_info["pc"], crash_signal=crash_info["signal"], + exploitability=crash_info["exploitable"], + disassembly=disasm, total_samples=total, + additional=str(additional_info), + crash_hash=crash_hash, status=0) log("Crash stored") else: log("Ignoring and removing already existing crash with hash %s" % crash_hash) - os.remove(temp_file) + if os.path.isfile(temp_file): + os.remove(temp_file) + if os.path.isfile(temp_file + ".diff"): + os.remove(temp_file + ".diff") self.reset_iteration(project_id) @@ -331,8 +344,29 @@ def reset_iteration(self, project_id): where = "project_id = $project_id and mutation_engine_id = -1" self.db.update("statistics", iteration=0, where=where, vars=vars) + def add_templates(self): + what = "project_id, name, subfolder" + res = self.db.select("projects", what=what, where="enabled = 1") + + for row in res: + project_folder = os.path.join(self.config["WORKING_PATH"], row['subfolder']) + input_folder = os.path.join(project_folder, "input") + + for i in os.listdir(input_folder): + i_file = os.path.join(input_folder, i) + with open(i_file, "rb") as f: + buf = f.read() + file_hash = sha1(buf).hexdigest() + template = os.path.join(project_folder, "templates", file_hash) + + if not os.path.isfile(template): + log("Adding sample %s to project %s" % (file_hash, row['name'])) + os.rename(i_file, template) + else: + os.remove(i_file) + def find_crashes(self): - what = "project_id, tube_prefix" + what = "project_id, subfolder, tube_prefix" res = self.db.select("projects", what=what, where="enabled = 1") for row in res: @@ -340,15 +374,15 @@ def find_crashes(self): q = get_queue(watch=True, name=tube_name) while q.stats_tube(tube_name)["current-jobs-ready"] > 0: job = q.reserve() - crash_info = json.loads(job.body) - temp_file = crash_info.keys()[0] - crash_data = crash_info.values()[0] - self.insert_crash(row.project_id, temp_file, crash_data) + d = json.loads(job.body) + self.insert_crash(row.project_id, row.subfolder, d) job.delete() def generate(self): log("Starting generator...") while 1: + debug("Add templates...") + self.add_templates() debug("Finding crashes...") self.find_crashes() debug("Checking files to remove...") @@ -359,10 +393,11 @@ def generate(self): for pe in project_engines: tube_prefix = pe.tube_prefix + tube_name = "%s-samples" % tube_prefix maximum = pe.maximum_samples - if not self.queue_is_full(tube_prefix, maximum): - for i in range(self.get_pending_elements(tube_prefix, maximum)): - if self.queue_is_full(tube_prefix, maximum): + if not self.queue_is_full(tube_name, maximum): + for i in range(self.get_pending_elements(tube_name, maximum)): + if self.queue_is_full(tube_name, maximum): break line = "Creating sample for %s from folder %s for tube %s mutator %s" diff --git a/runtime/nightmare_frontend.py b/runtime/nightmare_frontend.py index 5d4538b..88713a3 100755 --- a/runtime/nightmare_frontend.py +++ b/runtime/nightmare_frontend.py @@ -10,7 +10,7 @@ import sys import web import json - +import shutil from hashlib import sha1 from zipfile import ZipFile @@ -135,29 +135,19 @@ def POST(self): f = register_form() return render.login(f) - i = web.input(samples_path="", templates_path="", nightmare_path="", \ - temporary_path="") - if i.samples_path == "" or i.templates_path == "" or \ - i.nightmare_path == "" or i.temporary_path == "": + i = web.input(working_path="", nightmare_path="", temporary_path="") + if i.working_path == "" or i.nightmare_path == "" or i.temporary_path == "": render.error("Invalid samples, templates, temporary or nightmare path") db = init_web_db() with db.transaction(): - sql = "select 1 from config where name = 'SAMPLES_PATH'" - res = list(db.query(sql)) - if len(res) > 0: - sql = "update config set value = $value where name = 'SAMPLES_PATH'" - else: - sql = "insert into config (name, value) values ('SAMPLES_PATH', $value)" - db.query(sql, vars={"value":i.samples_path}) - - sql = "select 1 from config where name = 'TEMPLATES_PATH'" + sql = "select 1 from config where name = 'WORKING_PATH'" res = list(db.query(sql)) if len(res) > 0: - sql = "update config set value = $value where name = 'TEMPLATES_PATH'" + sql = "update config set value = $value where name = 'WORKING_PATH'" else: - sql = "insert into config (name, value) values ('TEMPLATES_PATH', $value)" - db.query(sql, vars={"value":i.templates_path}) + sql = "insert into config (name, value) values ('WORKING_PATH', $value)" + db.query(sql, vars={"value":i.working_path}) sql = "select 1 from config where name = 'NIGHTMARE_PATH'" res = list(db.query(sql)) @@ -201,22 +191,18 @@ def GET(self): db = init_web_db() sql = """select name, value from config - where name in ('SAMPLES_PATH', 'TEMPLATES_PATH', 'NIGHTMARE_PATH', - 'TEMPORARY_PATH', 'QUEUE_HOST', 'QUEUE_PORT')""" + where name in ('WORKING_PATH', 'NIGHTMARE_PATH', 'TEMPORARY_PATH', 'QUEUE_HOST', 'QUEUE_PORT')""" res = db.query(sql) - samples_path = "" - templates_path = "" + working_path = "" nightmare_path = "" temporary_path = "" queue_host = "localhost" queue_port = 11300 for row in res: name, value = row.name, row.value - if name == 'SAMPLES_PATH': - samples_path = value - elif name == 'TEMPLATES_PATH': - templates_path = value + if name == 'WORKING_PATH': + working_path = value elif name == 'NIGHTMARE_PATH': nightmare_path = value elif name == 'TEMPORARY_PATH': @@ -226,8 +212,7 @@ def GET(self): elif name == 'QUEUE_PORT': queue_port = value - return render.config(samples_path, templates_path, temporary_path, - nightmare_path, queue_host, queue_port) + return render.config(working_path, nightmare_path, temporary_path, queue_host, queue_port) #----------------------------------------------------------------------- class users: @@ -273,6 +258,11 @@ def POST(self): ignore_duplicates = 0 db = init_web_db() + sql = """select value from config where name in ('WORKING_PATH')""" + res = db.query(sql) + res = list(res) + working_path = res[0]['value'] + with db.transaction(): db.insert("projects", name=i.name, description=i.description, subfolder=i.subfolder, tube_prefix=i.tube_prefix, @@ -281,6 +271,19 @@ def POST(self): date=web.SQLLiteral("CURRENT_DATE"), ignore_duplicates=ignore_duplicates) + project_folder = os.path.join(working_path, i.subfolder) + if not os.path.exists(project_folder): + os.makedirs(project_folder) + + if not os.path.exists(os.path.join(project_folder, "samples")): + os.makedirs(os.path.join(project_folder, "samples")) + + if not os.path.exists(os.path.join(project_folder, "templates")): + os.makedirs(os.path.join(project_folder, "templates")) + + if not os.path.exists(os.path.join(project_folder, "input")): + os.makedirs(os.path.join(project_folder, "input")) + return web.redirect("/projects") #----------------------------------------------------------------------- @@ -317,6 +320,34 @@ def POST(self): ignore_duplicates = 0 db = init_web_db() + + sql = """select value from config where name in ('WORKING_PATH')""" + res = db.query(sql) + res = list(res) + working_path = res[0]['value'] + + what = """project_id, name, description, subfolder, tube_prefix, + maximum_samples, enabled, date, archived, + maximum_iteration, ignore_duplicates """ + where = "project_id = $project_id" + vars = {"project_id":i.id} + res = db.select("projects", what=what, where=where, vars=vars) + res = list(res) + + old_path = os.path.join(working_path, res[0]['subfolder']) + new_path = os.path.join(working_path, i.subfolder) + print old_path, new_path + if os.path.isfile(old_path) and old_path != new_path: + shutil.move(old_path, new_path) + elif old_path != new_path: + os.makedirs(new_path) + os.makedirs(os.path.join(new_path, "samples")) + os.makedirs(os.path.join(new_path, "templates")) + os.makedirs(os.path.join(new_path, "input")) + + if len(res) == 0: + return render.error("Invalid project identifier") + with db.transaction(): enabled = i.enabled == "on" archived = i.archived == "on" @@ -362,6 +393,22 @@ def POST(self): return render.error("You must check the \"I'm sure\" field.") db = init_web_db() + + sql = """select value from config where name in ('WORKING_PATH')""" + res = db.query(sql) + res = list(res) + working_path = res[0]['value'] + + what = """project_id, name, description, subfolder, tube_prefix, + maximum_samples, enabled, date, archived, + maximum_iteration, ignore_duplicates """ + where = "project_id = $project_id" + vars = {"project_id":i.id} + res = db.select("projects", what=what, where=where, vars=vars) + res = list(res) + + shutil.rmtree(os.path.join(working_path, res[0]['subfolder'])) + with db.transaction(): vars={"project_id":i.id} where = "project_id=$project_id" @@ -632,9 +679,9 @@ def GET(self): if i.no_field not in valid_fields: return render.error("Invalid field %s" % i.no_field) - sql += " ORDER BY c.%s DESC" % (i.sortValue) + sql += " ORDER BY %s DESC" % (i.sortValue) else: - sql += " ORDER BY c.date DESC" + sql += " ORDER BY date DESC" res = db.query(sql) results = {} @@ -783,26 +830,31 @@ def GET(self): is_diff = False db = init_web_db() - what = "sample_hash" - where = "sample_id = $id" - vars = {"id":i.id} - res = db.select("samples", what=what, where=where, vars=vars) + print i.id + res = db.query("""SELECT t1.sample_hash, + t3.subfolder + FROM samples t1 + JOIN crashes t2 + ON t1.sample_id = t2.sample_id + JOIN projects t3 + ON t3.project_id = t2.project_id + WHERE t1.sample_id = %s""", (i.id,)) res = list(res) if len(res) == 0: return render.error("Invalid crash identifier") row = res[0] sample_hash = row.sample_hash + subfolder = row.subfolder - res = db.select("config", what="value", where="name='SAMPLES_PATH'") + res = db.select("config", what="value", where="name='WORKING_PATH'") res = list(res) if len(res) == 0: - return render.error("Invalid configuration value for 'SAMPLES_PATH'") - row = res[0] + return render.error("Invalid configuration value for 'WORKING_PATH'") + working_path = res[0].value - path = os.path.join(row.value, "crashes") - path = os.path.join(path, sample_hash) + path = os.path.join(working_path, subfolder, "samples", sample_hash) if not os.path.exists(path): - return render.error("Crash sample does not exists! %s" % path) + return render.error("Crash sample does not exist! %s" % path) if is_diff: if not os.path.exists(path + ".diff"): @@ -947,7 +999,7 @@ def POST(self): #----------------------------------------------------------------------- def find_original_file(db, id): - + # ToDo - Currently broken. Correct this to handle project folder. vars = {"id":id} where = "sample_id = $id" res = db.select("samples", what="sample_hash", where=where, vars=vars) @@ -956,10 +1008,10 @@ def find_original_file(db, id): raise Exception("Invalid crash identifier") sample_hash = res[0].sample_hash - res = db.select("config", what="value", where="name='SAMPLES_PATH'") + res = db.select("config", what="value", where="name='WORKING_PATH'") res = list(res) if len(res) == 0: - raise Exception("Invalid configuration value for 'SAMPLES_PATH'") + raise Exception("Invalid configuration value for 'WORKING_PATH'") path = os.path.join(res[0].value, "crashes") path = os.path.join(path, sample_hash) @@ -1100,12 +1152,12 @@ def get_sample_files(db, i, crash_id): row = res[0] sample_hash = row.sample_hash - res = db.select("config", what="value", where="name = 'SAMPLES_PATH'") + res = db.select("config", what="value", where="name = 'WORKING_PATH'") res = list(res) if len(res) == 0: - return render.error("Invalid configuration value for 'SAMPLES_PATH'") + return render.error("Invalid configuration value for 'WORKING_PATH'") - path = os.path.join(res[0].value, "crashes") + path = os.path.join(res[0].value, "samples") path = os.path.join(path, sample_hash) print path if not os.path.exists(path): diff --git a/runtime/templates/config.html b/runtime/templates/config.html index e4b9c4e..426e425 100644 --- a/runtime/templates/config.html +++ b/runtime/templates/config.html @@ -1,4 +1,4 @@ -$def with (samples_path, templates_path, temporary_path, nightmare_path, queue_host, queue_port) +$def with (working_path, nightmare_path, temporary_path, queue_host, queue_port) @@ -31,15 +31,10 @@

Configuration

- - + + - - - - -
Samples path:Working path: Path where all the crashing samples will be stored for later analysis.
Templates path:Path where template files, packets, etc... will be read from.
Temporary path: