libct/cg: add swapOnlyUsage in MemoryStats

This field reports swap-only usage. For cgroupv1, `Usage` and `Failcnt` are set by subtracting memory usage from memory+swap usage. For cgroupv2, `Usage`, `Limit`, and `MaxUsage` are set. This commit also export `MaxUsage` of memory under cgroupv2 mode, using `memory.peak` introduced in kernel 5.19. Signed-off-by: Heran Yang <[email protected]>
rotnihr389 · Oct 25, 2023 · 104b8dc · 104b8dc
1 parent a68529c
commit 104b8dc
Show file tree

Hide file tree

Showing 6 changed files with 49 additions and 8 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -31,6 +31,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
    This aligns cgroupv2 root usage more closely with cgroupv1 reporting.
    Additionally, report root swap usage as sum of swap and memory usage,
    aligned with v1 and existing non-root v2 reporting. (#3933)
+ * Add `swapOnlyUsage` in `MemoryStats`. This field reports swap-only usage.
+   For cgroupv1, `Usage` and `Failcnt` are set by subtracting memory usage
+   from memory+swap usage. For cgroupv2, `Usage`, `Limit`, and `MaxUsage`
+   are set. (#4010)
 
 ### Fixed
 

diff --git a/libcontainer/cgroups/fs/memory.go b/libcontainer/cgroups/fs/memory.go
@@ -170,6 +170,10 @@ func (s *MemoryGroup) GetStats(path string, stats *cgroups.Stats) error {
 		return err
 	}
 	stats.MemoryStats.SwapUsage = swapUsage
+	stats.MemoryStats.SwapOnlyUsage = cgroups.MemoryData{
+		Usage:   swapUsage.Usage - memoryUsage.Usage,
+		Failcnt: swapUsage.Failcnt - memoryUsage.Failcnt,
+	}
 	kernelUsage, err := getMemoryData(path, "kmem")
 	if err != nil {
 		return err

diff --git a/libcontainer/cgroups/fs/memory_test.go b/libcontainer/cgroups/fs/memory_test.go
@@ -249,12 +249,13 @@ func TestMemoryStats(t *testing.T) {
 		t.Fatal(err)
 	}
 	expectedStats := cgroups.MemoryStats{
-		Cache:        512,
-		Usage:        cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192},
-		SwapUsage:    cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192},
-		KernelUsage:  cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192},
-		Stats:        map[string]uint64{"cache": 512, "rss": 1024},
-		UseHierarchy: true,
+		Cache:         512,
+		Usage:         cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192},
+		SwapUsage:     cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192},
+		SwapOnlyUsage: cgroups.MemoryData{Usage: 0, MaxUsage: 0, Failcnt: 0, Limit: 0},
+		KernelUsage:   cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192},
+		Stats:         map[string]uint64{"cache": 512, "rss": 1024},
+		UseHierarchy:  true,
 		PageUsageByNUMA: cgroups.PageUsageByNUMA{
 			PageUsageByNUMAInner: cgroups.PageUsageByNUMAInner{
 				Total:       cgroups.PageStats{Total: 44611, Nodes: map[uint8]uint64{0: 32631, 1: 7501, 2: 1982, 3: 2497}},

diff --git a/libcontainer/cgroups/fs2/memory.go b/libcontainer/cgroups/fs2/memory.go
@@ -105,25 +105,30 @@ func statMemory(dirPath string, stats *cgroups.Stats) error {
 	memoryUsage, err := getMemoryDataV2(dirPath, "")
 	if err != nil {
 		if errors.Is(err, unix.ENOENT) && dirPath == UnifiedMountpoint {
-			// The root cgroup does not have memory.{current,max}
+			// The root cgroup does not have memory.{current,max,peak}
 			// so emulate those using data from /proc/meminfo and
 			// /sys/fs/cgroup/memory.stat
 			return rootStatsFromMeminfo(stats)
 		}
 		return err
 	}
 	stats.MemoryStats.Usage = memoryUsage
-	swapUsage, err := getMemoryDataV2(dirPath, "swap")
+	swapOnlyUsage, err := getMemoryDataV2(dirPath, "swap")
 	if err != nil {
 		return err
 	}
+	stats.MemoryStats.SwapOnlyUsage = swapOnlyUsage
+	swapUsage := swapOnlyUsage
 	// As cgroup v1 reports SwapUsage values as mem+swap combined,
 	// while in cgroup v2 swap values do not include memory,
 	// report combined mem+swap for v1 compatibility.
 	swapUsage.Usage += memoryUsage.Usage
 	if swapUsage.Limit != math.MaxUint64 {
 		swapUsage.Limit += memoryUsage.Limit
 	}
+	// The `MaxUsage` of mem+swap cannot simply combine mem with
+	// swap. So set it to 0 for v1 compatibility.
+	swapUsage.MaxUsage = 0
 	stats.MemoryStats.SwapUsage = swapUsage
 
 	return nil
@@ -138,6 +143,7 @@ func getMemoryDataV2(path, name string) (cgroups.MemoryData, error) {
 	}
 	usage := moduleName + ".current"
 	limit := moduleName + ".max"
+	maxUsage := moduleName + ".peak"
 
 	value, err := fscommon.GetCgroupParamUint(path, usage)
 	if err != nil {
@@ -157,6 +163,14 @@ func getMemoryDataV2(path, name string) (cgroups.MemoryData, error) {
 	}
 	memoryData.Limit = value
 
+	// `memory.peak` since kernel 5.19
+	// `memory.swap.peak` since kernel 6.5
+	value, err = fscommon.GetCgroupParamUint(path, maxUsage)
+	if err != nil && !os.IsNotExist(err) {
+		return cgroups.MemoryData{}, err
+	}
+	memoryData.MaxUsage = value
+
 	return memoryData, nil
 }
 

diff --git a/libcontainer/cgroups/fs2/memory_test.go b/libcontainer/cgroups/fs2/memory_test.go
@@ -94,6 +94,10 @@ func TestStatMemoryPodCgroup(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	if err := os.WriteFile(filepath.Join(fakeCgroupDir, "memory.peak"), []byte("987654321"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
 	gotStats := cgroups.NewStats()
 
 	// use a fake root path to trigger the pod cgroup lookup.
@@ -107,6 +111,18 @@ func TestStatMemoryPodCgroup(t *testing.T) {
 	if gotStats.MemoryStats.Usage.Usage != expectedUsageBytes {
 		t.Errorf("parsed cgroupv2 memory.stat doesn't match expected result: \ngot %#v\nexpected %#v\n", gotStats.MemoryStats.Usage.Usage, expectedUsageBytes)
 	}
+
+	// result should be "memory.max"
+	var expectedLimitBytes uint64 = 999999999
+	if gotStats.MemoryStats.Usage.Limit != expectedLimitBytes {
+		t.Errorf("parsed cgroupv2 memory.stat doesn't match expected result: \ngot %#v\nexpected %#v\n", gotStats.MemoryStats.Usage.Limit, expectedLimitBytes)
+	}
+
+	// result should be "memory.peak"
+	var expectedMaxUsageBytes uint64 = 987654321
+	if gotStats.MemoryStats.Usage.MaxUsage != expectedMaxUsageBytes {
+		t.Errorf("parsed cgroupv2 memory.stat doesn't match expected result: \ngot %#v\nexpected %#v\n", gotStats.MemoryStats.Usage.MaxUsage, expectedMaxUsageBytes)
+	}
 }
 
 func TestRootStatsFromMeminfo(t *testing.T) {

diff --git a/libcontainer/cgroups/stats.go b/libcontainer/cgroups/stats.go
@@ -91,6 +91,8 @@ type MemoryStats struct {
 	Usage MemoryData `json:"usage,omitempty"`
 	// usage of memory + swap
 	SwapUsage MemoryData `json:"swap_usage,omitempty"`
+	// usage of swap only
+	SwapOnlyUsage MemoryData `json:"swap_only_usage,omitempty"`
 	// usage of kernel memory
 	KernelUsage MemoryData `json:"kernel_usage,omitempty"`
 	// usage of kernel TCP memory