mm/memcg: add wmark-step support

Signed-off-by: Winston Wen <wentao@uniontech.com>
Change-Id: I07fcfcced59279861e27440426000f6d6b42f36f

Author: winnscode

include/linux/memcontrol.h

@@ -256,6 +256,10 @@ struct mem_cgroup {
 	/* OOM-Killer disable */
 	int		oom_kill_disable;
 
+#ifdef CONFIG_WMARK_STEP
+	int		wmark_step;
+#endif
+
 	/* memory.events and memory.events.local */
 	struct cgroup_file events_file;
 	struct cgroup_file events_local_file;

mm/Kconfig

@@ -1282,6 +1282,13 @@ config LOCK_MM_AND_FIND_VMA
 	bool
 	depends on !STACK_GROWSUP
 
+config WMARK_STEP
+	bool "WaterMark step"
+	depends on MEMCG
+	default y
+	help
+	  Watermark step.
+
 source "mm/damon/Kconfig"
 
 endmenu

mm/internal.h

@@ -913,8 +913,24 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone,
 #define ALLOC_HIGHATOMIC	0x200 /* Allows access to MIGRATE_HIGHATOMIC */
 #define ALLOC_KSWAPD		0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */
 
+#ifdef CONFIG_WMARK_STEP
+#define ALLOC_WMARK_STEP		0x1000 /* Allow use user-reserve */
+#endif
+
+#ifndef CONFIG_WMARK_STEP
 /* Flags that allow allocations below the min watermark. */
 #define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM)
+#else
+#define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM|ALLOC_WMARK_STEP)
+#endif
+
+#ifdef CONFIG_WMARK_STEP
+extern unsigned int wmark_step_enable;
+extern unsigned int wmark_step_max;
+extern unsigned int wmark_step_kthread;
+extern unsigned int wmark_step_size;
+int should_use_wmark_step(void);
+#endif
 
 enum ttu_flags;
 struct tlbflush_unmap_batch;

mm/memcontrol.c

@@ -5388,6 +5388,9 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
 	if (parent) {
 		WRITE_ONCE(memcg->swappiness, mem_cgroup_swappiness(parent));
 		WRITE_ONCE(memcg->oom_kill_disable, READ_ONCE(parent->oom_kill_disable));
+#ifdef CONFIG_WMARK_STEP
+		WRITE_ONCE(memcg->wmark_step, READ_ONCE(parent->wmark_step));
+#endif
 
 		page_counter_init(&memcg->memory, &parent->memory);
 		page_counter_init(&memcg->swap, &parent->swap);
@@ -6735,6 +6738,71 @@ static ssize_t memory_oom_group_write(struct kernfs_open_file *of,
 	return nbytes;
 }
 
+#ifdef CONFIG_WMARK_STEP
+int should_use_wmark_step(void)
+{
+	struct mem_cgroup *memcg;
+
+	if (!wmark_step_enable)
+		return 0;
+
+	if (!in_task())
+		return 0;
+
+	if (mem_cgroup_disabled())
+		return 0;
+
+	if (current->flags & PF_KTHREAD)
+		return wmark_step_kthread;
+
+	memcg = get_mem_cgroup_from_mm(current->mm);
+	return memcg->wmark_step;
+}
+
+static int memory_wmark_step_show(struct seq_file *m, void *v)
+{
+	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
+
+	seq_printf(m, "%d\n", READ_ONCE(memcg->wmark_step));
+
+	return 0;
+}
+
+static ssize_t memory_wmark_step_write(struct kernfs_open_file *of,
+				      char *buf, size_t nbytes, loff_t off)
+{
+	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
+	struct mem_cgroup *iter;
+	int ret, wmark_step;
+	// kuid_t root_uid;
+	// const struct cred *cred;
+
+	BUG_ON(mem_cgroup_is_root(memcg));
+
+	/* Temporarily keep the code for future use */
+	//root_uid = make_kuid(&init_user_ns, 0);
+	//cred = current_cred();
+	//if (!uid_eq(cred->uid, root_uid))
+	//	return -EPERM;
+
+	buf = strstrip(buf);
+	if (!buf)
+		return -EINVAL;
+
+	ret = kstrtoint(buf, 0, &wmark_step);
+	if (ret)
+		return ret;
+
+	if (wmark_step < 0 || wmark_step > wmark_step_max)
+		return -EINVAL;
+
+	for_each_mem_cgroup_tree(iter, memcg)
+		WRITE_ONCE(iter->wmark_step, wmark_step);
+
+	return nbytes;
+}
+#endif
+
 static ssize_t memory_reclaim(struct kernfs_open_file *of, char *buf,
 			      size_t nbytes, loff_t off)
 {
@@ -6840,6 +6908,14 @@ static struct cftype memory_files[] = {
 		.seq_show = memory_oom_group_show,
 		.write = memory_oom_group_write,
 	},
+#ifdef CONFIG_WMARK_STEP
+	{
+		.name = "wmark_step",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.seq_show = memory_wmark_step_show,
+		.write = memory_wmark_step_write,
+	},
+#endif
 	{
 		.name = "reclaim",
 		.flags = CFTYPE_NS_DELEGATABLE,
@@ -7416,6 +7492,66 @@ static int __init cgroup_memory(char *s)
 }
 __setup("cgroup.memory=", cgroup_memory);
 
+#ifdef CONFIG_WMARK_STEP
+static ssize_t wmark_step_kthread_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "%u\n", wmark_step_kthread);
+}
+
+static ssize_t wmark_step_kthread_store(struct kobject *kobj, struct kobj_attribute *attr,
+				const char *buf, size_t len)
+{
+	unsigned int kthread;
+
+	if (kstrtouint(buf, 0, &kthread))
+		return -EINVAL;
+
+	if (kthread > 5)
+		return -EINVAL;
+
+	wmark_step_kthread = kthread;
+
+	return len;
+}
+
+static struct kobj_attribute wmark_step_kthread_attr = __ATTR_RW(wmark_step_kthread);
+
+static ssize_t wmark_step_enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "%u\n", wmark_step_enable);
+}
+
+static ssize_t wmark_step_enabled_store(struct kobject *kobj, struct kobj_attribute *attr,
+				const char *buf, size_t len)
+{
+	unsigned int enable;
+
+	if (kstrtouint(buf, 0, &enable))
+		return -EINVAL;
+
+	if (enable > 1)
+		return -EINVAL;
+
+	wmark_step_enable = enable;
+
+	return len;
+}
+
+static struct kobj_attribute wmark_step_enabled_attr = __ATTR(enabled, 0644,
+	wmark_step_enabled_show, wmark_step_enabled_store);
+
+static struct attribute *wmark_step_attrs[] = {
+	&wmark_step_enabled_attr.attr,
+	&wmark_step_kthread_attr.attr,
+	NULL
+};
+
+static const struct attribute_group wmark_step_attr_group = {
+	.name = "wmark_step",
+	.attrs = wmark_step_attrs,
+};
+#endif
+
 /*
  * subsys_initcall() for memory controller.
  *
@@ -7454,6 +7590,11 @@ static int __init mem_cgroup_init(void)
 		soft_limit_tree.rb_tree_per_node[node] = rtpn;
 	}
 
+#ifdef CONFIG_WMARK_STEP
+	if (sysfs_create_group(mm_kobj, &wmark_step_attr_group))
+		pr_warn("wmark-step: failed to create sysfs group\n");
+#endif
+
 	return 0;
 }
 subsys_initcall(mem_cgroup_init);

mm/page_alloc.c

@@ -204,6 +204,13 @@ EXPORT_SYMBOL(node_states);
 
 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
 
+#ifdef CONFIG_WMARK_STEP
+unsigned int wmark_step_enable = 0;
+unsigned int wmark_step_max = 5;
+unsigned int wmark_step_kthread = 3;
+unsigned int wmark_step_size = 16 * 1024 * 1024;
+#endif
+
 /*
  * A cached value of the page's pageblock's migratetype, used when the page is
  * put on a pcplist. Used to avoid the pageblock migratetype lookup when
@@ -2860,11 +2867,35 @@ bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
 {
 	long min = mark;
 	int o;
+#ifdef CONFIG_WMARK_STEP
+	long pages_wmark_step = (wmark_step_max * wmark_step_size) >> PAGE_SHIFT;
+	int wmark_step;
+#endif
 
 	/* free_pages may go negative - that's OK */
 	free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags);
 
+
 	if (unlikely(alloc_flags & ALLOC_RESERVES)) {
+#ifdef CONFIG_WMARK_STEP
+		if (zone_idx(z) == ZONE_NORMAL) {
+			wmark_step = should_use_wmark_step();
+			if (wmark_step < 0 || wmark_step > wmark_step_max) {
+				pr_err("uosste: wmark_step error: %d\n", wmark_step);
+				goto skip;
+			}
+
+			if (min <= pages_wmark_step) {
+				pr_err("uosste: min[%ld] < pages_wmark_step[%ld]\n",
+						min, pages_wmark_step);
+				goto skip;
+			}
+
+			min -= (wmark_step * wmark_step_size) >> PAGE_SHIFT;
+		}
+skip:
+#endif
+
 		/*
 		 * __GFP_HIGH allows access to 50% of the min reserve as well
 		 * as OOM.
@@ -3713,6 +3744,11 @@ static inline unsigned int
 gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order)
 {
 	unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
+#ifdef CONFIG_WMARK_STEP
+	/* Only enabled when ALLOC_WMARK_MIN */
+	if (should_use_wmark_step())
+		alloc_flags |= ALLOC_WMARK_STEP;
+#endif
 
 	/*
 	 * __GFP_HIGH is assumed to be the same as ALLOC_MIN_RESERVE
@@ -3731,6 +3767,7 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order)
 	alloc_flags |= (__force int)
 		(gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM));
 
+
 	if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) {
 		/*
 		 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
@@ -5656,6 +5693,9 @@ static void setup_per_zone_lowmem_reserve(void)
 static void __setup_per_zone_wmarks(void)
 {
 	unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
+#ifdef CONFIG_WMARK_STEP
+	unsigned long pages_wmark_step = (wmark_step_max * wmark_step_size) >> PAGE_SHIFT;
+#endif
 	unsigned long lowmem_pages = 0;
 	struct zone *zone;
 	unsigned long flags;
@@ -5693,6 +5733,12 @@ static void __setup_per_zone_wmarks(void)
 			 * proportionate to the zone's size.
 			 */
 			zone->_watermark[WMARK_MIN] = tmp;
+
+#ifdef CONFIG_WMARK_STEP
+			/* Put all wmark step in zone normal */
+			if (zone_idx(zone) == ZONE_NORMAL)
+				zone->_watermark[WMARK_MIN] += pages_wmark_step;
+#endif
 		}
 
 		/*