Make the sandbox actually use a restricted environment

Janzert · Janzert · commit 80b8556ed79b · 2011-05-02T17:32:36.000-04:00
diff --git a/setup/worker_server_info.py.template b/setup/worker_server_info.py.template
@@ -3,6 +3,7 @@ server_info = {{
   "mail_name": "AI Contest",
   "mail_password": "",
   "root_path": "{contest_root}",
+  "repo_path": "{repo_dir}",
   "maps_path": "{map_dir}",
   "submissions_path": "{compiled_dir}",
   "api_base_url": "{api_url}",
diff --git a/setup/worker_setup.py b/setup/worker_setup.py
@@ -130,6 +130,7 @@ def setup_contest_files(opts):
     with open(si_filename, 'r') as si_file:
         si_template = si_file.read()
     si_contents = si_template.format(contest_root=contest_root,
+            repo_dir=opts.local_repo,
             map_dir=map_dir, compiled_dir=compiled_dir,
             api_url=opts.api_url, api_key=opts.api_key)
     with CD(worker_dir):
@@ -173,6 +174,8 @@ def create_jail_group(options):
     """ Create user group for jail users and set limits on it """
     if not file_contains("/etc/group", "^jailusers"):
         run_cmd("groupadd jailusers")
+        run_cmd("groupadd jailkeeper")
+        run_cmd("usermod -a -G jailkeeper %s" % (options.username,))
     limits_conf = "/etc/security/limits.conf"
     if not file_contains(limits_conf, "@jailusers"):
         # limit jailuser processes to:
@@ -205,6 +208,8 @@ def create_jail_user(username):
     home_dir = os.path.join(jail_dir, "home/home/jailuser")
     os.makedirs(home_dir)
     run_cmd("chown %s:jailusers %s" % (username, home_dir))
+    run_cmd("chown :jailkeeper %s" % (jail_dir,))
+    run_cmd("chmod g=rwx %s" % (jail_dir,))
     fs_line = "unionfs-fuse#%s=rw:%s=ro:%s=ro %s fuse cow,allow_other 0 0" % (
             os.path.join(jail_dir, "scratch"),
             os.path.join(jail_dir, "home"),
diff --git a/worker/engine.py b/worker/engine.py
@@ -208,6 +208,7 @@ def run_game(game, botcmds, options, gameid=0):
         for bot in bots:
             if bot.is_alive:
                 bot.kill()
+            bot.release()
 
         # close all the open files
         if stream_log:
diff --git a/worker/jail_own.c b/worker/jail_own.c
@@ -44,7 +44,8 @@ int own_dir_tree(const char *base_dir, uid_t owner, gid_t group) {
             fprintf(stderr, "Could not change owner on '%s'\n", child_path);
             error = 1;
         } else {
-            printf("Changed owner on: %s\n", child_path);
+            // Uncomment for verbose output on every file ownership change
+            // printf("Changed owner on: %s\n", child_path);
         }
         if (entry->d_type == DT_DIR) {
             error = own_dir_tree(child_path, owner, group) ? 1 : error;
@@ -107,8 +108,10 @@ int main(int argc, char *argv[]) {
     int status = 0;
     printf("Changing ownership to uid %d, gid %d\n", owner_uid, owner_gid);
     if (chown(base_path, owner_uid, owner_gid)) {
-        printf("Could not change owner on %s\n", base_path);
+        fprintf(stderr, "Could not change owner on %s\n", base_path);
         status = EXIT_CHOWN_FAIL;
+    } else {
+        printf("Changed owner on: %s\n", base_path);
     }
     if (own_dir_tree(base_path, owner_uid, owner_gid)) {
         status = EXIT_CHOWN_FAIL;
diff --git a/worker/sandbox.py b/worker/sandbox.py
@@ -9,6 +9,99 @@
 from Queue import Queue, Empty
 from threading import Thread
 
+try:
+    from server_info import server_info
+    _SECURE_DEFAULT = True
+except ImportError:
+    _SECURE_DEFAULT = False
+
+class JailError(StandardError):
+    pass
+
+class _Jail(object):
+    def __init__(self):
+        self.locked = False
+        jail_base = "/srv/chroot"
+        all_jails = os.listdir(jail_base)
+        all_jails = [j for j in all_jails if j.startswith("jailuser")]
+        for jail in all_jails:
+            lock_dir = os.path.join(jail_base, jail, "locked")
+            try:
+                os.mkdir(lock_dir)
+            except OSError:
+                # if the directory could not be created, that should mean the
+                # jail is already locked and in use
+                continue
+            with open(os.path.join(lock_dir, "lock.pid"), "w") as pid_file:
+                pid_file.write(str(os.getpid()))
+            self.locked = True
+            self.name = jail
+            break
+        else:
+            raise JailError("Could not find an unlocked jail")
+        self.jchown = os.path.join(server_info["root_path"],
+                server_info["repo_path"], "worker/jail_own")
+        self.base_dir = os.path.join(jail_base, jail)
+        self.number = int(jail[len("jailuser"):])
+        self.chroot_cmd = "sudo -u {0} schroot -u {0} -c {0} -d {1} ".format(
+                self.name, "/home/jailuser")
+
+    def __del__(self):
+        if self.locked:
+            raise JailError("Jail object for %s freed without being released"
+                    % (self.name))
+
+    def release(self):
+        if not self.locked:
+            raise JailError("Attempt to release jail that is already unlocked")
+        lock_dir = os.path.join(self.base_dir, "locked")
+        pid_filename = os.path.join(lock_dir, "lock.pid")
+        with open(pid_filename, 'r') as pid_file:
+            lock_pid = int(pid_file.read())
+            if lock_pid != os.getpid():
+                # if we ever get here something has gone seriously wrong
+                # most likely the jail locking mechanism has failed
+                raise JailError("Jail released by different pid, name %s, lock_pid %d, release_pid %d"
+                        % (self.name, lock_pid, os.getpid()))
+        os.unlink(pid_filename)
+        os.rmdir(lock_dir)
+        self.locked = False
+
+    def prepare_with(self, command_dir):
+        if os.system("%s c %d" % (self.jchown, self.number)) != 0:
+            raise JailError("Error returned from jail_own c %d in prepare"
+                    % (self.number,))
+        scratch_dir = os.path.join(self.base_dir, "scratch")
+        if os.system("rm -rf %s" % (scratch_dir,)) != 0:
+            raise JailError("Could not remove old scratch area from jail %d"
+                    % (self.number,))
+        home_dir = os.path.join(scratch_dir, "home/jailuser")
+        os.makedirs(home_dir)
+        if os.system("cp -r %s %s" % (command_dir, home_dir)) != 0:
+            raise JailError("Error copying working directory '%s' to jail %d"
+                    % (command_dir, self.number))
+        if os.system("%s j %d" % (self.jchown, self.number)) != 0:
+            raise JailError("Error returned from jail_own j %d in prepare"
+                    % (self.number,))
+
+    def signal(self, signal):
+        if not self.locked:
+            raise JailError("Attempt to send %s to unlocked jail" % (signal,))
+        if os.system("sudo -u {0} killall -{1} -u {0}".format(
+                self.name, signal)) != 0:
+            raise JailError("Error returned from jail kill for %s"
+                    % (self.name,))
+
+    def kill(self):
+        self.signal("KILL")
+
+    def pause(self):
+        self.signal("STOP")
+
+    def resume(self):
+        self.signal("CONT")
+
+
 def _monitor_input_channel(sandbox):
     while sandbox.is_alive:
         try:
@@ -31,7 +124,8 @@ class Sandbox:
 
     """
 
-    def __init__(self, working_directory, shell_command, stderr=None):
+    def __init__(self, working_directory, shell_command, stderr=None,
+            secure=_SECURE_DEFAULT):
         """Initialize a new sandbox and invoke the given shell command inside.
 
         working_directory: the directory in which the shell command should
@@ -40,15 +134,26 @@ def __init__(self, working_directory, shell_command, stderr=None):
                            shell command is executed.
         shell_command: the shell command to launch inside the sandbox.
         stderr: where the bot's stderr output should be written out to
+                defaults to keeping the current stderr for the child process
+        secure: really use a jail or just run the command directly
+                defaults to True when a server_info module is found, False
+                otherwise
 
         """
-        shell_command = shell_command.replace('\\','/')
         self.is_alive = False
         self.command_process = None
         self.stdout_queue = Queue()
-        self.stderr_queue = Queue()
 
-        self.command_process = subprocess.Popen(shlex.split(shell_command),
+        if secure:
+            self.jail = _Jail()
+            self.jail.prepare_with(working_directory)
+            shell_command = self.jail.chroot_cmd + shell_command
+            working_directory = None
+        else:
+            self.jail = None
+
+        shell_command = shlex.split(shell_command.replace('\\','/'))
+        self.command_process = subprocess.Popen(shell_command,
                                                 stdin=subprocess.PIPE,
                                                 stdout=subprocess.PIPE,
                                                 stderr=stderr,
@@ -57,41 +162,62 @@ def __init__(self, working_directory, shell_command, stderr=None):
         stdout_monitor = Thread(target=_monitor_input_channel, args=(self,))
         stdout_monitor.start()
 
+    def release(self):
+        """Release the sandbox for further use
+
+        If running in a jail unlocks and releases the jail for reuse by others.
+        Must be called exactly once after Sandbox.kill has been called.
+
+        """
+        if self.is_alive:
+            raise JailError("Jail released while still alive")
+        if self.jail:
+            self.jail.release()
+
     def kill(self):
-        """ Shuts down the sandbox.
+        """Shuts down the sandbox.
 
         Shuts down the sandbox, cleaning up any spawned processes, threads, and
         other resources. The shell command running inside the sandbox may be
         suddenly terminated.
 
         """
         if self.is_alive:
-            try:
-                self.command_process.kill()
-                self.command_process.wait()
-            except OSError:
-                pass
+            if self.jail:
+                self.jail.kill()
+            else:
+                try:
+                    self.command_process.kill()
+                    self.command_process.wait()
+                except OSError:
+                    pass
             self.is_alive = False
 
     def pause(self):
         """Pause the process by sending a SIGSTOP to the child
 
         This method is a no-op on Windows
         """
-        try:
-            self.command_process.send_signal(signal.SIGSTOP)
-        except (ValueError, AttributeError):
-            pass
+        if self.jail:
+            self.jail.pause()
+        else:
+            try:
+                self.command_process.send_signal(signal.SIGSTOP)
+            except (ValueError, AttributeError):
+                pass
 
     def resume(self):
         """Resume the process by sending a SIGCONT to the child
 
         This method is a no-op on Windows
         """
-        try:
-            self.command_process.send_signal(signal.SIGCONT)
-        except (ValueError, AttributeError):
-            pass
+        if self.jail:
+            self.jail.resume()
+        else:
+            try:
+                self.command_process.send_signal(signal.SIGCONT)
+            except (ValueError, AttributeError):
+                pass
 
     def write(self, str):
         """Write str to stdin of the process being run"""
@@ -146,12 +272,18 @@ def main():
     parser.add_option("-r", "--receive-delay", action="store",
             dest="resp_delay", type="float", default=0.01,
             help="Time in seconds to sleep before checking for a response line")
+    parser.add_option("-j", "--jail", action="store_true", dest="secure",
+            default=_SECURE_DEFAULT,
+            help="Run in a secure jail")
+    parser.add_option("-o", "--open", action="store_false", dest="secure",
+            help="Run without using a secure jail")
     options, args = parser.parse_args()
     if len(args) == 0:
         parser.error("Must include a command to run.\
                 \nRun with --help for more information.")
 
-    sandbox = Sandbox(options.working_dir, " ".join(args))
+    sandbox = Sandbox(options.working_dir, " ".join(args),
+            secure=options.secure)
     for line in options.send_lines:
         if not sandbox.write_line(line):
             print >> sys.stderr, "Could not send line '%s'" % (line,)
