libcontainer: cgroups: add intel_rdt support in runc

libcontainer/SPEC.md

@@ -143,6 +143,7 @@ system resources like cpu, memory, and device access.
 | freezer    | 1       |
 | hugetlb    | 1       |
 | pids       | 1       |
+| intel_rdt  | 1       |
 
 
 All cgroup subsystem are joined so that statistics can be collected from

libcontainer/cgroups/fs/apply_raw.go

@@ -30,6 +30,7 @@ var (
 		&NetPrioGroup{},
 		&PerfEventGroup{},
 		&FreezerGroup{},
+		&IntelRdtGroup{},
 	}
 	CgroupProcesses  = "cgroup.procs"
 	HugePageSizes, _ = cgroups.GetHugePageSize()

libcontainer/cgroups/fs/intel_rdt.go

@@ -0,0 +1,75 @@
+// +build linux
+
+package fs
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type IntelRdtGroup struct {
+}
+
+func (s *IntelRdtGroup) Name() string {
+	return "intel_rdt"
+}
+
+func (s *IntelRdtGroup) Apply(d *cgroupData) error {
+	dir, err := d.join("intel_rdt")
+	if err != nil {
+		if !cgroups.IsNotFound(err) {
+			return err
+		}
+		// We will not return err here when:
+		// 1. The h/w platform doesn't support Intel RDT/CAT feature,
+		//    intel_rdt cgroup is not enabled in kernel.
+		// 2. intel_rdt cgroup is not mounted
+		return nil
+	}
+
+	if err := s.Set(dir, d.config); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *IntelRdtGroup) Set(path string, cgroup *configs.Cgroup) error {
+	// The valid CBM (capacity bitmask) is a *contiguous bits set* and
+	// number of bits that can be set is less than the max bit. The max
+	// bits in the CBM is varied among supported Intel platforms.
+	//
+	// By default the child cgroups inherit the CBM from parent. The CBM
+	// in a child cgroup should be a subset of the CBM in parent. Kernel
+	// will check if it is valid when writing.
+	//
+	// e.g., 0xfffff in root cgroup indicates the max bits of CBM is 20
+	// bits, which mapping to entire L3 cache capacity. Some valid CBM
+	// values to Set in children cgroup: 0xf, 0xf0, 0x3ff, 0x1f00 and etc.
+	if cgroup.Resources.IntelRdtL3Cbm != 0 {
+		l3CbmStr := fmt.Sprintf("0x%s", strconv.FormatUint(cgroup.Resources.IntelRdtL3Cbm, 16))
+		if err := writeFile(path, "intel_rdt.l3_cbm", l3CbmStr); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *IntelRdtGroup) Remove(d *cgroupData) error {
+	return removePath(d.path("intel_rdt"))
+}
+
+func (s *IntelRdtGroup) GetStats(path string, stats *cgroups.Stats) error {
+	value, err := getCgroupParamUintHex(path, "intel_rdt.l3_cbm")
+	if err != nil {
+		return fmt.Errorf("failed to parse intel_rdt.l3_cbm - %s", err)
+	}
+
+	stats.IntelRdtStats.L3Cbm = value
+
+	return nil
+}

libcontainer/cgroups/fs/intel_rdt_test.go

@@ -0,0 +1,62 @@
+// +build linux
+
+package fs
+
+import (
+	"strconv"
+	"testing"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+)
+
+func TestIntelRdtSetL3Cbm(t *testing.T) {
+	helper := NewCgroupTestUtil("intel_rdt", t)
+	defer helper.cleanup()
+
+	const (
+		l3CbmBefore = 0xf
+		l3CbmAfter  = 0xf0
+	)
+
+	helper.writeFileContents(map[string]string{
+		"intel_rdt.l3_cbm": strconv.FormatUint(l3CbmBefore, 16),
+	})
+
+	helper.CgroupData.config.Resources.IntelRdtL3Cbm = l3CbmAfter
+	intelrdt := &IntelRdtGroup{}
+	if err := intelrdt.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamUintHex(helper.CgroupPath, "intel_rdt.l3_cbm")
+	if err != nil {
+		t.Fatalf("Failed to parse intel_rdt.l3_cbm - %s", err)
+	}
+
+	if value != l3CbmAfter {
+		t.Fatal("Got the wrong value, set intel_rdt.l3_cbm failed.")
+	}
+}
+
+func TestIntelRdtStats(t *testing.T) {
+	helper := NewCgroupTestUtil("intel_rdt", t)
+	defer helper.cleanup()
+
+	const (
+		l3CbmContents = 0x1f00
+	)
+
+	helper.writeFileContents(map[string]string{
+		"intel_rdt.l3_cbm": strconv.FormatUint(l3CbmContents, 16),
+	})
+
+	intelrdt := &IntelRdtGroup{}
+	stats := *cgroups.NewStats()
+	if err := intelrdt.GetStats(helper.CgroupPath, &stats); err != nil {
+		t.Fatal(err)
+	}
+
+	if stats.IntelRdtStats.L3Cbm != l3CbmContents {
+		t.Fatalf("Expected '0x%x', got '0x%x' for intel_rdt.l3_cbm", l3CbmContents, stats.IntelRdtStats.L3Cbm)
+	}
+}

libcontainer/cgroups/fs/utils.go

@@ -68,6 +68,22 @@ func getCgroupParamUint(cgroupPath, cgroupFile string) (uint64, error) {
 	return res, nil
 }
 
+// Gets a single hex uint64 value from the specified cgroup file.
+func getCgroupParamUintHex(cgroupPath, cgroupFile string) (uint64, error) {
+	fileName := filepath.Join(cgroupPath, cgroupFile)
+	contents, err := ioutil.ReadFile(fileName)
+	if err != nil {
+		return 0, err
+	}
+
+	hexStr := strings.TrimSpace(strings.TrimPrefix(string(contents), "0x"))
+	res, err := parseUint(hexStr, 16, 64)
+	if err != nil {
+		return res, fmt.Errorf("unable to parse %q as a uint from Cgroup file %q", string(contents), fileName)
+	}
+	return res, nil
+}
+
 // Gets a string value from the specified cgroup file
 func getCgroupParamString(cgroupPath, cgroupFile string) (string, error) {
 	contents, err := ioutil.ReadFile(filepath.Join(cgroupPath, cgroupFile))

libcontainer/cgroups/stats.go

@@ -84,13 +84,18 @@ type HugetlbStats struct {
 	Failcnt uint64 `json:"failcnt"`
 }
 
+type IntelRdtStats struct {
+	L3Cbm uint64 `json:"l3_cbm,omitempty"`
+}
+
 type Stats struct {
 	CpuStats    CpuStats    `json:"cpu_stats,omitempty"`
 	MemoryStats MemoryStats `json:"memory_stats,omitempty"`
 	PidsStats   PidsStats   `json:"pids_stats,omitempty"`
 	BlkioStats  BlkioStats  `json:"blkio_stats,omitempty"`
 	// the map is in the format "size of hugepage: stats of the hugepage"
-	HugetlbStats map[string]HugetlbStats `json:"hugetlb_stats,omitempty"`
+	HugetlbStats  map[string]HugetlbStats `json:"hugetlb_stats,omitempty"`
+	IntelRdtStats IntelRdtStats           `json:"intel_rdt_stats,omitempty"`
 }
 
 func NewStats() *Stats {

libcontainer/cgroups/systemd/apply_systemd.go

@@ -63,6 +63,7 @@ var subsystems = subsystemSet{
 	&fs.NetPrioGroup{},
 	&fs.NetClsGroup{},
 	&fs.NameGroup{GroupName: "name=systemd"},
+	&fs.IntelRdtGroup{},
 }
 
 const (
@@ -251,7 +252,7 @@ func (m *Manager) Apply(pid int) error {
 		return err
 	}
 
-	// we need to manually join the freezer, net_cls, net_prio, pids and cpuset cgroup in systemd
+	// we need to manually join the freezer, net_cls, net_prio cpuset and intel_rdt cgroup in systemd
 	// because it does not currently support it via the dbus api.
 	if err := joinFreezer(c, pid); err != nil {
 		return err
@@ -279,6 +280,11 @@ func (m *Manager) Apply(pid int) error {
 	if err := joinPerfEvent(c, pid); err != nil {
 		return err
 	}
+
+	if err := joinIntelRdt(c, pid); err != nil {
+		return err
+	}
+
 	// FIXME: Systemd does have `BlockIODeviceWeight` property, but we got problem
 	// using that (at least on systemd 208, see https://github.com/opencontainers/runc/libcontainer/pull/354),
 	// so use fs work around for now.
@@ -603,3 +609,24 @@ func joinPerfEvent(c *configs.Cgroup, pid int) error {
 	}
 	return nil
 }
+
+func joinIntelRdt(c *configs.Cgroup, pid int) error {
+	path, err := join(c, "intel_rdt", pid)
+	if err != nil {
+		if !cgroups.IsNotFound(err) {
+			return err
+		}
+		// We will not return err here when:
+		// 1. The h/w platform doesn't support Intel RDT/CAT feature,
+		//    intel_rdt cgroup is not enabled in kernel.
+		// 2. intel_rdt cgroup is not mounted
+		return nil
+	}
+
+	IntelRdt, err := subsystems.Get("intel_rdt")
+	if err != nil {
+		return err
+	}
+
+	return IntelRdt.Set(path, c)
+}

libcontainer/configs/cgroup_unix.go

@@ -118,4 +118,7 @@ type Resources struct {
 
 	// Set class identifier for container's network packets
 	NetClsClassid string `json:"net_cls_classid"`
+
+	// L3 cache capacity bitmask (CBM) for container
+	IntelRdtL3Cbm uint64 `json:"intel_rdt_l3_cbm"`
 }

spec.go

@@ -480,6 +480,10 @@ func createCgroupConfig(name string, spec *specs.LinuxSpec) (*configs.Cgroup, er
 			})
 		}
 	}
+	if r.IntelRdt != nil {
+		c.Resources.IntelRdtL3Cbm = *r.IntelRdt.L3Cbm
+	}
+
 	return c, nil
 }