diff --git a/libcontainer/README.md b/libcontainer/README.md index d2a7d7889b0..70480e75920 100644 --- a/libcontainer/README.md +++ b/libcontainer/README.md @@ -82,6 +82,7 @@ config := &configs.Config{ {Type: configs.NEWPID}, {Type: configs.NEWUSER}, {Type: configs.NEWNET}, + {Type: configs.NEWCGROUP}, }), Cgroups: &configs.Cgroup{ Name: "test-container", diff --git a/libcontainer/SPEC.md b/libcontainer/SPEC.md index e5894c6429d..12da753b0af 100644 --- a/libcontainer/SPEC.md +++ b/libcontainer/SPEC.md @@ -21,16 +21,17 @@ Minimum requirements: ### Namespaces -| Flag | Enabled | -| ------------ | ------- | -| CLONE_NEWPID | 1 | -| CLONE_NEWUTS | 1 | -| CLONE_NEWIPC | 1 | -| CLONE_NEWNET | 1 | -| CLONE_NEWNS | 1 | -| CLONE_NEWUSER | 1 | - -Namespaces are created for the container via the `clone` syscall. +| Flag | Enabled | +| --------------- | ------- | +| CLONE_NEWPID | 1 | +| CLONE_NEWUTS | 1 | +| CLONE_NEWIPC | 1 | +| CLONE_NEWNET | 1 | +| CLONE_NEWNS | 1 | +| CLONE_NEWUSER | 1 | +| CLONE_NEWCGROUP | 1 | + +Namespaces are created for the container via the `unshare` syscall. ### Filesystem diff --git a/libcontainer/cgroups/utils.go b/libcontainer/cgroups/utils.go index 52fc87eb3e6..2f42b6d980f 100644 --- a/libcontainer/cgroups/utils.go +++ b/libcontainer/cgroups/utils.go @@ -17,7 +17,7 @@ import ( ) const ( - cgroupNamePrefix = "name=" + CgroupNamePrefix = "name=" CgroupProcesses = "cgroup.procs" ) @@ -139,8 +139,8 @@ func getCgroupMountsHelper(ss map[string]bool, mi io.Reader, all bool) ([]Mount, if !ss[opt] { continue } - if strings.HasPrefix(opt, cgroupNamePrefix) { - m.Subsystems = append(m.Subsystems, opt[len(cgroupNamePrefix):]) + if strings.HasPrefix(opt, CgroupNamePrefix) { + m.Subsystems = append(m.Subsystems, opt[len(CgroupNamePrefix):]) } else { m.Subsystems = append(m.Subsystems, opt) } @@ -294,7 +294,7 @@ func getControllerPath(subsystem string, cgroups map[string]string) (string, err return p, nil } - if p, ok := cgroups[cgroupNamePrefix+subsystem]; ok { + if p, ok := cgroups[CgroupNamePrefix+subsystem]; ok { return p, nil } diff --git a/libcontainer/configs/namespaces_syscall.go b/libcontainer/configs/namespaces_syscall.go index fb4b8522223..4a3a0e2c2bf 100644 --- a/libcontainer/configs/namespaces_syscall.go +++ b/libcontainer/configs/namespaces_syscall.go @@ -8,13 +8,17 @@ func (n *Namespace) Syscall() int { return namespaceInfo[n.Type] } +// This is not yet in the Go stdlib. +const syscall_CLONE_NEWCGROUP = (1 << 25) + var namespaceInfo = map[NamespaceType]int{ - NEWNET: syscall.CLONE_NEWNET, - NEWNS: syscall.CLONE_NEWNS, - NEWUSER: syscall.CLONE_NEWUSER, - NEWIPC: syscall.CLONE_NEWIPC, - NEWUTS: syscall.CLONE_NEWUTS, - NEWPID: syscall.CLONE_NEWPID, + NEWNET: syscall.CLONE_NEWNET, + NEWNS: syscall.CLONE_NEWNS, + NEWUSER: syscall.CLONE_NEWUSER, + NEWIPC: syscall.CLONE_NEWIPC, + NEWUTS: syscall.CLONE_NEWUTS, + NEWPID: syscall.CLONE_NEWPID, + NEWCGROUP: syscall_CLONE_NEWCGROUP, } // CloneFlags parses the container's Namespaces options to set the correct diff --git a/libcontainer/configs/namespaces_unix.go b/libcontainer/configs/namespaces_unix.go index 8beba9d300c..37a01afa403 100644 --- a/libcontainer/configs/namespaces_unix.go +++ b/libcontainer/configs/namespaces_unix.go @@ -9,12 +9,13 @@ import ( ) const ( - NEWNET NamespaceType = "NEWNET" - NEWPID NamespaceType = "NEWPID" - NEWNS NamespaceType = "NEWNS" - NEWUTS NamespaceType = "NEWUTS" - NEWIPC NamespaceType = "NEWIPC" - NEWUSER NamespaceType = "NEWUSER" + NEWNET NamespaceType = "NEWNET" + NEWPID NamespaceType = "NEWPID" + NEWNS NamespaceType = "NEWNS" + NEWUTS NamespaceType = "NEWUTS" + NEWIPC NamespaceType = "NEWIPC" + NEWUSER NamespaceType = "NEWUSER" + NEWCGROUP NamespaceType = "NEWCGROUP" ) var ( @@ -37,6 +38,8 @@ func NsName(ns NamespaceType) string { return "user" case NEWUTS: return "uts" + case NEWCGROUP: + return "cgroup" } return "" } @@ -70,6 +73,7 @@ func NamespaceTypes() []NamespaceType { NEWUTS, NEWIPC, NEWUSER, + NEWCGROUP, } } diff --git a/libcontainer/configs/validate/validator.go b/libcontainer/configs/validate/validator.go index f076f506a24..bfcdf181626 100644 --- a/libcontainer/configs/validate/validator.go +++ b/libcontainer/configs/validate/validator.go @@ -37,6 +37,9 @@ func (v *ConfigValidator) Validate(config *configs.Config) error { if err := v.usernamespace(config); err != nil { return err } + if err := v.cgroupnamespace(config); err != nil { + return err + } if err := v.sysctl(config); err != nil { return err } @@ -107,6 +110,15 @@ func (v *ConfigValidator) usernamespace(config *configs.Config) error { return nil } +func (v *ConfigValidator) cgroupnamespace(config *configs.Config) error { + if config.Namespaces.Contains(configs.NEWCGROUP) { + if _, err := os.Stat("/proc/self/ns/cgroup"); os.IsNotExist(err) { + return fmt.Errorf("cgroup namespaces aren't enabled in the kernel") + } + } + return nil +} + // sysctl validates that the specified sysctl keys are valid or not. // /proc/sys isn't completely namespaced and depending on which namespaces // are specified, a subset of sysctls are permitted. diff --git a/libcontainer/container_linux.go b/libcontainer/container_linux.go index 9f0043fb0cf..a087742d90f 100644 --- a/libcontainer/container_linux.go +++ b/libcontainer/container_linux.go @@ -1293,6 +1293,7 @@ func (c *linuxContainer) orderNamespacePaths(namespaces map[configs.NamespaceTyp configs.NEWNET, configs.NEWPID, configs.NEWNS, + configs.NEWCGROUP, } // Remove namespaces that we don't need to join. diff --git a/libcontainer/integration/exec_test.go b/libcontainer/integration/exec_test.go index d9e3f5f1f5a..5b7d2098374 100644 --- a/libcontainer/integration/exec_test.go +++ b/libcontainer/integration/exec_test.go @@ -1694,3 +1694,60 @@ func TestTmpfsCopyUp(t *testing.T) { t.Fatalf("/etc/passwd not copied up as expected: %v", outputLs) } } + +func TestCGROUPPrivate(t *testing.T) { + if _, err := os.Stat("/proc/self/ns/cgroup"); os.IsNotExist(err) { + t.Skip("cgroupns is unsupported") + } + if testing.Short() { + return + } + + rootfs, err := newRootfs() + ok(t, err) + defer remove(rootfs) + + l, err := os.Readlink("/proc/1/ns/cgroup") + ok(t, err) + + config := newTemplateConfig(rootfs) + config.Namespaces.Add(configs.NEWCGROUP, "") + buffers, exitCode, err := runContainer(config, "", "readlink", "/proc/self/ns/cgroup") + ok(t, err) + + if exitCode != 0 { + t.Fatalf("exit code not 0. code %d stderr %q", exitCode, buffers.Stderr) + } + + if actual := strings.Trim(buffers.Stdout.String(), "\n"); actual == l { + t.Fatalf("cgroup link should be private to the container but equals host %q %q", actual, l) + } +} + +func TestCGROUPHost(t *testing.T) { + if _, err := os.Stat("/proc/self/ns/cgroup"); os.IsNotExist(err) { + t.Skip("cgroupns is unsupported") + } + if testing.Short() { + return + } + + rootfs, err := newRootfs() + ok(t, err) + defer remove(rootfs) + + l, err := os.Readlink("/proc/1/ns/cgroup") + ok(t, err) + + config := newTemplateConfig(rootfs) + buffers, exitCode, err := runContainer(config, "", "readlink", "/proc/self/ns/cgroup") + ok(t, err) + + if exitCode != 0 { + t.Fatalf("exit code not 0. code %d stderr %q", exitCode, buffers.Stderr) + } + + if actual := strings.Trim(buffers.Stdout.String(), "\n"); actual != l { + t.Fatalf("cgroup link not equal to host link %q %q", actual, l) + } +} diff --git a/libcontainer/nsenter/nsexec.c b/libcontainer/nsenter/nsexec.c index 51bd1e3eccc..a10db319ab2 100644 --- a/libcontainer/nsenter/nsexec.c +++ b/libcontainer/nsenter/nsexec.c @@ -40,6 +40,12 @@ enum sync_t { SYNC_ERR = 0xFF, /* Fatal error, no turning back. The error code follows. */ }; +/* + * Synchronisation value for cgroup namespace setup. + * The same constant is defined in process_linux.go as "createCgroupns". + */ +#define CREATECGROUPNS 0x80 + /* longjmp() arguments. */ #define JUMP_PARENT 0x00 #define JUMP_CHILD 0xA0 @@ -570,6 +576,17 @@ void nsexec(void) kill(child, SIGKILL); bail("failed to sync with child: write(SYNC_RECVPID_ACK)"); } + + /* Send the init_func pid back to our parent. */ + len = snprintf(buf, JSON_MAX, "{\"pid\": %d}\n", child); + if (len < 0) { + kill(child, SIGKILL); + bail("unable to generate JSON for child pid"); + } + if (write(pipenum, buf, len) != len) { + kill(child, SIGKILL); + bail("unable to send child pid to bootstrapper"); + } } break; case SYNC_CHILD_READY: @@ -614,17 +631,6 @@ void nsexec(void) } } - /* Send the init_func pid back to our parent. */ - len = snprintf(buf, JSON_MAX, "{\"pid\": %d}\n", child); - if (len < 0) { - kill(child, SIGKILL); - bail("unable to generate JSON for child pid"); - } - if (write(pipenum, buf, len) != len) { - kill(child, SIGKILL); - bail("unable to send child pid to bootstrapper"); - } - exit(0); } @@ -640,6 +646,7 @@ void nsexec(void) case JUMP_CHILD: { pid_t child; enum sync_t s; + uint32_t actual_flags = config.cloneflags; /* We're in a child and thus need to tell the parent if we die. */ syncfd = sync_child_pipe[0]; @@ -667,7 +674,9 @@ void nsexec(void) * some old kernel versions where clone(CLONE_PARENT | CLONE_NEWPID) * was broken, so we'll just do it the long way anyway. */ - if (unshare(config.cloneflags) < 0) + if (actual_flags & CLONE_NEWCGROUP) + actual_flags &= ~CLONE_NEWCGROUP; + if (unshare(actual_flags) < 0) bail("failed to unshare namespaces"); /* @@ -777,6 +786,19 @@ void nsexec(void) if (setgroups(0, NULL) < 0) bail("setgroups failed"); + /* ... wait until our topmost parent has finished cgroup setup in p.manager.Apply() ... */ + if (config.cloneflags & CLONE_NEWCGROUP) { + uint8_t value; + if (read(pipenum, &value, sizeof(value)) != sizeof(value)) + bail("read synchronisation value failed"); + if (value == CREATECGROUPNS) { + if (unshare(CLONE_NEWCGROUP) < 0) + bail("failed to unshare cgroup namespace"); + } + else + bail("received unknown synchronisation value"); + } + s = SYNC_CHILD_READY; if (write(syncfd, &s, sizeof(s)) != sizeof(s)) bail("failed to sync with patent: write(SYNC_CHILD_READY)"); diff --git a/libcontainer/process_linux.go b/libcontainer/process_linux.go index 50e90757e1c..1036775b63e 100644 --- a/libcontainer/process_linux.go +++ b/libcontainer/process_linux.go @@ -19,6 +19,10 @@ import ( "github.com/opencontainers/runc/libcontainer/utils" ) +// Synchronisation value for cgroup namespace setup. +// The same constant is defined in nsexec.c as "CREATECGROUPNS". +const createCgroupns byte = (1 << 7) + type parentProcess interface { // pid returns the pid for the running process. pid() int @@ -224,12 +228,17 @@ func (p *initProcess) externalDescriptors() []string { return p.fds } -// execSetns runs the process that executes C code to perform the setns calls -// because setns support requires the C process to fork off a child and perform the setns -// before the go runtime boots, we wait on the process to die and receive the child's pid -// over the provided pipe. -// This is called by initProcess.start function -func (p *initProcess) execSetns() error { +// getChildPid receives the final child's pid over the provided pipe. +func (p *initProcess) getChildPid() (int, error) { + var pid *pid + if err := json.NewDecoder(p.parentPipe).Decode(&pid); err != nil { + p.cmd.Wait() + return -1, err + } + return pid.Pid, nil +} + +func (p *initProcess) waitForChildExit(childPid int) error { status, err := p.cmd.Process.Wait() if err != nil { p.cmd.Wait() @@ -239,12 +248,7 @@ func (p *initProcess) execSetns() error { p.cmd.Wait() return &exec.ExitError{ProcessState: status} } - var pid *pid - if err := json.NewDecoder(p.parentPipe).Decode(&pid); err != nil { - p.cmd.Wait() - return err - } - process, err := os.FindProcess(pid.Pid) + process, err := os.FindProcess(childPid) if err != nil { return err } @@ -266,22 +270,35 @@ func (p *initProcess) start() error { if _, err := io.Copy(p.parentPipe, p.bootstrapData); err != nil { return err } - if err := p.execSetns(); err != nil { - return newSystemErrorWithCause(err, "running exec setns process for init") + childPid, err := p.getChildPid() + if err != nil { + return newSystemErrorWithCause(err, "getting the final child's pid from pipe") } // Save the standard descriptor names before the container process // can potentially move them (e.g., via dup2()). If we don't do this now, // we won't know at checkpoint time which file descriptor to look up. - fds, err := getPipeFds(p.pid()) + fds, err := getPipeFds(childPid) if err != nil { - return newSystemErrorWithCausef(err, "getting pipe fds for pid %d", p.pid()) + return newSystemErrorWithCausef(err, "getting pipe fds for pid %d", childPid) } p.setExternalDescriptors(fds) // Do this before syncing with child so that no children // can escape the cgroup - if err := p.manager.Apply(p.pid()); err != nil { + if err := p.manager.Apply(childPid); err != nil { return newSystemErrorWithCause(err, "applying cgroup configuration for process") } + // Now it's time to setup cgroup namesapce + if p.config.Config.Namespaces.Contains(configs.NEWCGROUP) && p.config.Config.Namespaces.PathOf(configs.NEWCGROUP) == "" { + if _, err := p.parentPipe.Write([]byte{createCgroupns}); err != nil { + return newSystemErrorWithCause(err, "sending synchronization value to init process") + } + } + + // Wait for our first child to exit + if err := p.waitForChildExit(childPid); err != nil { + return newSystemErrorWithCause(err, "waiting for our first child to exit") + } + defer func() { if err != nil { // TODO: should not be the responsibility to call here diff --git a/libcontainer/rootfs_linux.go b/libcontainer/rootfs_linux.go index 2635fd6f99c..ad19a293872 100644 --- a/libcontainer/rootfs_linux.go +++ b/libcontainer/rootfs_linux.go @@ -45,6 +45,7 @@ func prepareRootfs(pipe io.ReadWriter, config *configs.Config) (err error) { return newSystemErrorWithCause(err, "preparing rootfs") } + hasCgroupns := config.Namespaces.Contains(configs.NEWCGROUP) setupDev := needsSetupDev(config) for _, m := range config.Mounts { for _, precmd := range m.PremountCmds { @@ -52,8 +53,7 @@ func prepareRootfs(pipe io.ReadWriter, config *configs.Config) (err error) { return newSystemErrorWithCause(err, "running premount command") } } - - if err := mountToRootfs(m, config.Rootfs, config.MountLabel); err != nil { + if err := mountToRootfs(m, config.Rootfs, config.MountLabel, hasCgroupns); err != nil { return newSystemErrorWithCausef(err, "mounting %q to rootfs %q at %q", m.Source, config.Rootfs, m.Destination) } @@ -150,7 +150,7 @@ func mountCmd(cmd configs.Command) error { return nil } -func mountToRootfs(m *configs.Mount, rootfs, mountLabel string) error { +func mountToRootfs(m *configs.Mount, rootfs, mountLabel string, enableCgroupns bool) error { var ( dest = m.Destination ) @@ -282,12 +282,33 @@ func mountToRootfs(m *configs.Mount, rootfs, mountLabel string) error { Data: "mode=755", PropagationFlags: m.PropagationFlags, } - if err := mountToRootfs(tmpfs, rootfs, mountLabel); err != nil { + if err := mountToRootfs(tmpfs, rootfs, mountLabel, enableCgroupns); err != nil { return err } for _, b := range binds { - if err := mountToRootfs(b, rootfs, mountLabel); err != nil { - return err + if enableCgroupns { + subsystemPath := filepath.Join(rootfs, b.Destination) + if err := os.MkdirAll(subsystemPath, 0755); err != nil { + return err + } + flags := defaultMountFlags + if m.Flags&syscall.MS_RDONLY != 0 { + flags = flags | syscall.MS_RDONLY + } + cgroupmount := &configs.Mount{ + Source: "cgroup", + Device: "cgroup", + Destination: subsystemPath, + Flags: flags, + Data: filepath.Base(subsystemPath), + } + if err := mountNewCgroup(cgroupmount); err != nil { + return err + } + } else { + if err := mountToRootfs(b, rootfs, mountLabel, enableCgroupns); err != nil { + return err + } } } for _, mc := range merged { @@ -810,3 +831,18 @@ func mountPropagate(m *configs.Mount, rootfs string, mountLabel string) error { } return nil } + +func mountNewCgroup(m *configs.Mount) error { + var ( + data = m.Data + source = m.Source + ) + if data == "systemd" { + data = cgroups.CgroupNamePrefix + data + source = "systemd" + } + if err := syscall.Mount(source, m.Destination, m.Device, uintptr(m.Flags), data); err != nil { + return err + } + return nil +} diff --git a/libcontainer/specconv/spec_linux.go b/libcontainer/specconv/spec_linux.go index f99be9f4544..65ae9390b4a 100644 --- a/libcontainer/specconv/spec_linux.go +++ b/libcontainer/specconv/spec_linux.go @@ -27,6 +27,7 @@ var namespaceMapping = map[specs.LinuxNamespaceType]configs.NamespaceType{ specs.UserNamespace: configs.NEWUSER, specs.IPCNamespace: configs.NEWIPC, specs.UTSNamespace: configs.NEWUTS, + specs.CgroupNamespace: configs.NEWCGROUP, } var mountPropagationMapping = map[string]int{