mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-13 16:36:53 +07:00
3e32cb2e0a
Memory is internally accounted in bytes, using spinlock-protected 64-bit counters, even though the smallest accounting delta is a page. The counter interface is also convoluted and does too many things. Introduce a new lockless word-sized page counter API, then change all memory accounting over to it. The translation from and to bytes then only happens when interfacing with userspace. The removed locking overhead is noticable when scaling beyond the per-cpu charge caches - on a 4-socket machine with 144-threads, the following test shows the performance differences of 288 memcgs concurrently running a page fault benchmark: vanilla: 18631648.500498 task-clock (msec) # 140.643 CPUs utilized ( +- 0.33% ) 1,380,638 context-switches # 0.074 K/sec ( +- 0.75% ) 24,390 cpu-migrations # 0.001 K/sec ( +- 8.44% ) 1,843,305,768 page-faults # 0.099 M/sec ( +- 0.00% ) 50,134,994,088,218 cycles # 2.691 GHz ( +- 0.33% ) <not supported> stalled-cycles-frontend <not supported> stalled-cycles-backend 8,049,712,224,651 instructions # 0.16 insns per cycle ( +- 0.04% ) 1,586,970,584,979 branches # 85.176 M/sec ( +- 0.05% ) 1,724,989,949 branch-misses # 0.11% of all branches ( +- 0.48% ) 132.474343877 seconds time elapsed ( +- 0.21% ) lockless: 12195979.037525 task-clock (msec) # 133.480 CPUs utilized ( +- 0.18% ) 832,850 context-switches # 0.068 K/sec ( +- 0.54% ) 15,624 cpu-migrations # 0.001 K/sec ( +- 10.17% ) 1,843,304,774 page-faults # 0.151 M/sec ( +- 0.00% ) 32,811,216,801,141 cycles # 2.690 GHz ( +- 0.18% ) <not supported> stalled-cycles-frontend <not supported> stalled-cycles-backend 9,999,265,091,727 instructions # 0.30 insns per cycle ( +- 0.10% ) 2,076,759,325,203 branches # 170.282 M/sec ( +- 0.12% ) 1,656,917,214 branch-misses # 0.08% of all branches ( +- 0.55% ) 91.369330729 seconds time elapsed ( +- 0.45% ) On top of improved scalability, this also gets rid of the icky long long types in the very heart of memcg, which is great for 32 bit and also makes the code a lot more readable. Notable differences between the old and new API: - res_counter_charge() and res_counter_charge_nofail() become page_counter_try_charge() and page_counter_charge() resp. to match the more common kernel naming scheme of try_do()/do() - res_counter_uncharge_until() is only ever used to cancel a local counter and never to uncharge bigger segments of a hierarchy, so it's replaced by the simpler page_counter_cancel() - res_counter_set_limit() is replaced by page_counter_limit(), which expects its callers to serialize against themselves - res_counter_memparse_write_strategy() is replaced by page_counter_limit(), which rounds down to the nearest page size - rather than up. This is more reasonable for explicitely requested hard upper limits. - to keep charging light-weight, page_counter_try_charge() charges speculatively, only to roll back if the result exceeds the limit. Because of this, a failing bigger charge can temporarily lock out smaller charges that would otherwise succeed. The error is bounded to the difference between the smallest and the biggest possible charge size, so for memcg, this means that a failing THP charge can send base page charges into reclaim upto 2MB (4MB) before the limit would have been reached. This should be acceptable. [akpm@linux-foundation.org: add includes for WARN_ON_ONCE and memparse] [akpm@linux-foundation.org: add includes for WARN_ON_ONCE, memparse, strncmp, and PAGE_SIZE] Signed-off-by: Johannes Weiner <hannes@cmpxchg.org> Acked-by: Michal Hocko <mhocko@suse.cz> Acked-by: Vladimir Davydov <vdavydov@parallels.com> Cc: Tejun Heo <tj@kernel.org> Cc: David Rientjes <rientjes@google.com> Cc: Stephen Rothwell <sfr@canb.auug.org.au> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
230 lines
5.4 KiB
C
230 lines
5.4 KiB
C
#include <net/tcp.h>
|
|
#include <net/tcp_memcontrol.h>
|
|
#include <net/sock.h>
|
|
#include <net/ip.h>
|
|
#include <linux/nsproxy.h>
|
|
#include <linux/memcontrol.h>
|
|
#include <linux/module.h>
|
|
|
|
int tcp_init_cgroup(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
|
|
{
|
|
/*
|
|
* The root cgroup does not use page_counters, but rather,
|
|
* rely on the data already collected by the network
|
|
* subsystem
|
|
*/
|
|
struct mem_cgroup *parent = parent_mem_cgroup(memcg);
|
|
struct page_counter *counter_parent = NULL;
|
|
struct cg_proto *cg_proto, *parent_cg;
|
|
|
|
cg_proto = tcp_prot.proto_cgroup(memcg);
|
|
if (!cg_proto)
|
|
return 0;
|
|
|
|
cg_proto->sysctl_mem[0] = sysctl_tcp_mem[0];
|
|
cg_proto->sysctl_mem[1] = sysctl_tcp_mem[1];
|
|
cg_proto->sysctl_mem[2] = sysctl_tcp_mem[2];
|
|
cg_proto->memory_pressure = 0;
|
|
cg_proto->memcg = memcg;
|
|
|
|
parent_cg = tcp_prot.proto_cgroup(parent);
|
|
if (parent_cg)
|
|
counter_parent = &parent_cg->memory_allocated;
|
|
|
|
page_counter_init(&cg_proto->memory_allocated, counter_parent);
|
|
percpu_counter_init(&cg_proto->sockets_allocated, 0, GFP_KERNEL);
|
|
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(tcp_init_cgroup);
|
|
|
|
void tcp_destroy_cgroup(struct mem_cgroup *memcg)
|
|
{
|
|
struct cg_proto *cg_proto;
|
|
|
|
cg_proto = tcp_prot.proto_cgroup(memcg);
|
|
if (!cg_proto)
|
|
return;
|
|
|
|
percpu_counter_destroy(&cg_proto->sockets_allocated);
|
|
}
|
|
EXPORT_SYMBOL(tcp_destroy_cgroup);
|
|
|
|
static int tcp_update_limit(struct mem_cgroup *memcg, unsigned long nr_pages)
|
|
{
|
|
struct cg_proto *cg_proto;
|
|
int i;
|
|
int ret;
|
|
|
|
cg_proto = tcp_prot.proto_cgroup(memcg);
|
|
if (!cg_proto)
|
|
return -EINVAL;
|
|
|
|
ret = page_counter_limit(&cg_proto->memory_allocated, nr_pages);
|
|
if (ret)
|
|
return ret;
|
|
|
|
for (i = 0; i < 3; i++)
|
|
cg_proto->sysctl_mem[i] = min_t(long, nr_pages,
|
|
sysctl_tcp_mem[i]);
|
|
|
|
if (nr_pages == PAGE_COUNTER_MAX)
|
|
clear_bit(MEMCG_SOCK_ACTIVE, &cg_proto->flags);
|
|
else {
|
|
/*
|
|
* The active bit needs to be written after the static_key
|
|
* update. This is what guarantees that the socket activation
|
|
* function is the last one to run. See sock_update_memcg() for
|
|
* details, and note that we don't mark any socket as belonging
|
|
* to this memcg until that flag is up.
|
|
*
|
|
* We need to do this, because static_keys will span multiple
|
|
* sites, but we can't control their order. If we mark a socket
|
|
* as accounted, but the accounting functions are not patched in
|
|
* yet, we'll lose accounting.
|
|
*
|
|
* We never race with the readers in sock_update_memcg(),
|
|
* because when this value change, the code to process it is not
|
|
* patched in yet.
|
|
*
|
|
* The activated bit is used to guarantee that no two writers
|
|
* will do the update in the same memcg. Without that, we can't
|
|
* properly shutdown the static key.
|
|
*/
|
|
if (!test_and_set_bit(MEMCG_SOCK_ACTIVATED, &cg_proto->flags))
|
|
static_key_slow_inc(&memcg_socket_limit_enabled);
|
|
set_bit(MEMCG_SOCK_ACTIVE, &cg_proto->flags);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
enum {
|
|
RES_USAGE,
|
|
RES_LIMIT,
|
|
RES_MAX_USAGE,
|
|
RES_FAILCNT,
|
|
};
|
|
|
|
static DEFINE_MUTEX(tcp_limit_mutex);
|
|
|
|
static ssize_t tcp_cgroup_write(struct kernfs_open_file *of,
|
|
char *buf, size_t nbytes, loff_t off)
|
|
{
|
|
struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
|
|
unsigned long nr_pages;
|
|
int ret = 0;
|
|
|
|
buf = strstrip(buf);
|
|
|
|
switch (of_cft(of)->private) {
|
|
case RES_LIMIT:
|
|
/* see memcontrol.c */
|
|
ret = page_counter_memparse(buf, &nr_pages);
|
|
if (ret)
|
|
break;
|
|
mutex_lock(&tcp_limit_mutex);
|
|
ret = tcp_update_limit(memcg, nr_pages);
|
|
mutex_unlock(&tcp_limit_mutex);
|
|
break;
|
|
default:
|
|
ret = -EINVAL;
|
|
break;
|
|
}
|
|
return ret ?: nbytes;
|
|
}
|
|
|
|
static u64 tcp_cgroup_read(struct cgroup_subsys_state *css, struct cftype *cft)
|
|
{
|
|
struct mem_cgroup *memcg = mem_cgroup_from_css(css);
|
|
struct cg_proto *cg_proto = tcp_prot.proto_cgroup(memcg);
|
|
u64 val;
|
|
|
|
switch (cft->private) {
|
|
case RES_LIMIT:
|
|
if (!cg_proto)
|
|
return PAGE_COUNTER_MAX;
|
|
val = cg_proto->memory_allocated.limit;
|
|
val *= PAGE_SIZE;
|
|
break;
|
|
case RES_USAGE:
|
|
if (!cg_proto)
|
|
val = atomic_long_read(&tcp_memory_allocated);
|
|
else
|
|
val = page_counter_read(&cg_proto->memory_allocated);
|
|
val *= PAGE_SIZE;
|
|
break;
|
|
case RES_FAILCNT:
|
|
if (!cg_proto)
|
|
return 0;
|
|
val = cg_proto->memory_allocated.failcnt;
|
|
break;
|
|
case RES_MAX_USAGE:
|
|
if (!cg_proto)
|
|
return 0;
|
|
val = cg_proto->memory_allocated.watermark;
|
|
val *= PAGE_SIZE;
|
|
break;
|
|
default:
|
|
BUG();
|
|
}
|
|
return val;
|
|
}
|
|
|
|
static ssize_t tcp_cgroup_reset(struct kernfs_open_file *of,
|
|
char *buf, size_t nbytes, loff_t off)
|
|
{
|
|
struct mem_cgroup *memcg;
|
|
struct cg_proto *cg_proto;
|
|
|
|
memcg = mem_cgroup_from_css(of_css(of));
|
|
cg_proto = tcp_prot.proto_cgroup(memcg);
|
|
if (!cg_proto)
|
|
return nbytes;
|
|
|
|
switch (of_cft(of)->private) {
|
|
case RES_MAX_USAGE:
|
|
page_counter_reset_watermark(&cg_proto->memory_allocated);
|
|
break;
|
|
case RES_FAILCNT:
|
|
cg_proto->memory_allocated.failcnt = 0;
|
|
break;
|
|
}
|
|
|
|
return nbytes;
|
|
}
|
|
|
|
static struct cftype tcp_files[] = {
|
|
{
|
|
.name = "kmem.tcp.limit_in_bytes",
|
|
.write = tcp_cgroup_write,
|
|
.read_u64 = tcp_cgroup_read,
|
|
.private = RES_LIMIT,
|
|
},
|
|
{
|
|
.name = "kmem.tcp.usage_in_bytes",
|
|
.read_u64 = tcp_cgroup_read,
|
|
.private = RES_USAGE,
|
|
},
|
|
{
|
|
.name = "kmem.tcp.failcnt",
|
|
.private = RES_FAILCNT,
|
|
.write = tcp_cgroup_reset,
|
|
.read_u64 = tcp_cgroup_read,
|
|
},
|
|
{
|
|
.name = "kmem.tcp.max_usage_in_bytes",
|
|
.private = RES_MAX_USAGE,
|
|
.write = tcp_cgroup_reset,
|
|
.read_u64 = tcp_cgroup_read,
|
|
},
|
|
{ } /* terminate */
|
|
};
|
|
|
|
static int __init tcp_memcontrol_init(void)
|
|
{
|
|
WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys, tcp_files));
|
|
return 0;
|
|
}
|
|
__initcall(tcp_memcontrol_init);
|