mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-08 01:16:41 +07:00
3812c8c8f3
The memcg OOM handling is incredibly fragile and can deadlock. When a task fails to charge memory, it invokes the OOM killer and loops right there in the charge code until it succeeds. Comparably, any other task that enters the charge path at this point will go to a waitqueue right then and there and sleep until the OOM situation is resolved. The problem is that these tasks may hold filesystem locks and the mmap_sem; locks that the selected OOM victim may need to exit. For example, in one reported case, the task invoking the OOM killer was about to charge a page cache page during a write(), which holds the i_mutex. The OOM killer selected a task that was just entering truncate() and trying to acquire the i_mutex: OOM invoking task: mem_cgroup_handle_oom+0x241/0x3b0 mem_cgroup_cache_charge+0xbe/0xe0 add_to_page_cache_locked+0x4c/0x140 add_to_page_cache_lru+0x22/0x50 grab_cache_page_write_begin+0x8b/0xe0 ext3_write_begin+0x88/0x270 generic_file_buffered_write+0x116/0x290 __generic_file_aio_write+0x27c/0x480 generic_file_aio_write+0x76/0xf0 # takes ->i_mutex do_sync_write+0xea/0x130 vfs_write+0xf3/0x1f0 sys_write+0x51/0x90 system_call_fastpath+0x18/0x1d OOM kill victim: do_truncate+0x58/0xa0 # takes i_mutex do_last+0x250/0xa30 path_openat+0xd7/0x440 do_filp_open+0x49/0xa0 do_sys_open+0x106/0x240 sys_open+0x20/0x30 system_call_fastpath+0x18/0x1d The OOM handling task will retry the charge indefinitely while the OOM killed task is not releasing any resources. A similar scenario can happen when the kernel OOM killer for a memcg is disabled and a userspace task is in charge of resolving OOM situations. In this case, ALL tasks that enter the OOM path will be made to sleep on the OOM waitqueue and wait for userspace to free resources or increase the group's limit. But a userspace OOM handler is prone to deadlock itself on the locks held by the waiting tasks. For example one of the sleeping tasks may be stuck in a brk() call with the mmap_sem held for writing but the userspace handler, in order to pick an optimal victim, may need to read files from /proc/<pid>, which tries to acquire the same mmap_sem for reading and deadlocks. This patch changes the way tasks behave after detecting a memcg OOM and makes sure nobody loops or sleeps with locks held: 1. When OOMing in a user fault, invoke the OOM killer and restart the fault instead of looping on the charge attempt. This way, the OOM victim can not get stuck on locks the looping task may hold. 2. When OOMing in a user fault but somebody else is handling it (either the kernel OOM killer or a userspace handler), don't go to sleep in the charge context. Instead, remember the OOMing memcg in the task struct and then fully unwind the page fault stack with -ENOMEM. pagefault_out_of_memory() will then call back into the memcg code to check if the -ENOMEM came from the memcg, and then either put the task to sleep on the memcg's OOM waitqueue or just restart the fault. The OOM victim can no longer get stuck on any lock a sleeping task may hold. Debugged by Michal Hocko. Signed-off-by: Johannes Weiner <hannes@cmpxchg.org> Reported-by: azurIt <azurit@pobox.sk> Acked-by: Michal Hocko <mhocko@suse.cz> Cc: David Rientjes <rientjes@google.com> Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
692 lines
20 KiB
C
692 lines
20 KiB
C
/*
|
|
* linux/mm/oom_kill.c
|
|
*
|
|
* Copyright (C) 1998,2000 Rik van Riel
|
|
* Thanks go out to Claus Fischer for some serious inspiration and
|
|
* for goading me into coding this file...
|
|
* Copyright (C) 2010 Google, Inc.
|
|
* Rewritten by David Rientjes
|
|
*
|
|
* The routines in this file are used to kill a process when
|
|
* we're seriously out of memory. This gets called from __alloc_pages()
|
|
* in mm/page_alloc.c when we really run out of memory.
|
|
*
|
|
* Since we won't call these routines often (on a well-configured
|
|
* machine) this file will double as a 'coding guide' and a signpost
|
|
* for newbie kernel hackers. It features several pointers to major
|
|
* kernel subsystems and hints as to where to find out what things do.
|
|
*/
|
|
|
|
#include <linux/oom.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/err.h>
|
|
#include <linux/gfp.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/swap.h>
|
|
#include <linux/timex.h>
|
|
#include <linux/jiffies.h>
|
|
#include <linux/cpuset.h>
|
|
#include <linux/export.h>
|
|
#include <linux/notifier.h>
|
|
#include <linux/memcontrol.h>
|
|
#include <linux/mempolicy.h>
|
|
#include <linux/security.h>
|
|
#include <linux/ptrace.h>
|
|
#include <linux/freezer.h>
|
|
#include <linux/ftrace.h>
|
|
#include <linux/ratelimit.h>
|
|
|
|
#define CREATE_TRACE_POINTS
|
|
#include <trace/events/oom.h>
|
|
|
|
int sysctl_panic_on_oom;
|
|
int sysctl_oom_kill_allocating_task;
|
|
int sysctl_oom_dump_tasks = 1;
|
|
static DEFINE_SPINLOCK(zone_scan_lock);
|
|
|
|
#ifdef CONFIG_NUMA
|
|
/**
|
|
* has_intersects_mems_allowed() - check task eligiblity for kill
|
|
* @tsk: task struct of which task to consider
|
|
* @mask: nodemask passed to page allocator for mempolicy ooms
|
|
*
|
|
* Task eligibility is determined by whether or not a candidate task, @tsk,
|
|
* shares the same mempolicy nodes as current if it is bound by such a policy
|
|
* and whether or not it has the same set of allowed cpuset nodes.
|
|
*/
|
|
static bool has_intersects_mems_allowed(struct task_struct *tsk,
|
|
const nodemask_t *mask)
|
|
{
|
|
struct task_struct *start = tsk;
|
|
|
|
do {
|
|
if (mask) {
|
|
/*
|
|
* If this is a mempolicy constrained oom, tsk's
|
|
* cpuset is irrelevant. Only return true if its
|
|
* mempolicy intersects current, otherwise it may be
|
|
* needlessly killed.
|
|
*/
|
|
if (mempolicy_nodemask_intersects(tsk, mask))
|
|
return true;
|
|
} else {
|
|
/*
|
|
* This is not a mempolicy constrained oom, so only
|
|
* check the mems of tsk's cpuset.
|
|
*/
|
|
if (cpuset_mems_allowed_intersects(current, tsk))
|
|
return true;
|
|
}
|
|
} while_each_thread(start, tsk);
|
|
|
|
return false;
|
|
}
|
|
#else
|
|
static bool has_intersects_mems_allowed(struct task_struct *tsk,
|
|
const nodemask_t *mask)
|
|
{
|
|
return true;
|
|
}
|
|
#endif /* CONFIG_NUMA */
|
|
|
|
/*
|
|
* The process p may have detached its own ->mm while exiting or through
|
|
* use_mm(), but one or more of its subthreads may still have a valid
|
|
* pointer. Return p, or any of its subthreads with a valid ->mm, with
|
|
* task_lock() held.
|
|
*/
|
|
struct task_struct *find_lock_task_mm(struct task_struct *p)
|
|
{
|
|
struct task_struct *t = p;
|
|
|
|
do {
|
|
task_lock(t);
|
|
if (likely(t->mm))
|
|
return t;
|
|
task_unlock(t);
|
|
} while_each_thread(p, t);
|
|
|
|
return NULL;
|
|
}
|
|
|
|
/* return true if the task is not adequate as candidate victim task. */
|
|
static bool oom_unkillable_task(struct task_struct *p,
|
|
const struct mem_cgroup *memcg, const nodemask_t *nodemask)
|
|
{
|
|
if (is_global_init(p))
|
|
return true;
|
|
if (p->flags & PF_KTHREAD)
|
|
return true;
|
|
|
|
/* When mem_cgroup_out_of_memory() and p is not member of the group */
|
|
if (memcg && !task_in_mem_cgroup(p, memcg))
|
|
return true;
|
|
|
|
/* p may not have freeable memory in nodemask */
|
|
if (!has_intersects_mems_allowed(p, nodemask))
|
|
return true;
|
|
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* oom_badness - heuristic function to determine which candidate task to kill
|
|
* @p: task struct of which task we should calculate
|
|
* @totalpages: total present RAM allowed for page allocation
|
|
*
|
|
* The heuristic for determining which task to kill is made to be as simple and
|
|
* predictable as possible. The goal is to return the highest value for the
|
|
* task consuming the most memory to avoid subsequent oom failures.
|
|
*/
|
|
unsigned long oom_badness(struct task_struct *p, struct mem_cgroup *memcg,
|
|
const nodemask_t *nodemask, unsigned long totalpages)
|
|
{
|
|
long points;
|
|
long adj;
|
|
|
|
if (oom_unkillable_task(p, memcg, nodemask))
|
|
return 0;
|
|
|
|
p = find_lock_task_mm(p);
|
|
if (!p)
|
|
return 0;
|
|
|
|
adj = (long)p->signal->oom_score_adj;
|
|
if (adj == OOM_SCORE_ADJ_MIN) {
|
|
task_unlock(p);
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* The baseline for the badness score is the proportion of RAM that each
|
|
* task's rss, pagetable and swap space use.
|
|
*/
|
|
points = get_mm_rss(p->mm) + p->mm->nr_ptes +
|
|
get_mm_counter(p->mm, MM_SWAPENTS);
|
|
task_unlock(p);
|
|
|
|
/*
|
|
* Root processes get 3% bonus, just like the __vm_enough_memory()
|
|
* implementation used by LSMs.
|
|
*/
|
|
if (has_capability_noaudit(p, CAP_SYS_ADMIN))
|
|
adj -= 30;
|
|
|
|
/* Normalize to oom_score_adj units */
|
|
adj *= totalpages / 1000;
|
|
points += adj;
|
|
|
|
/*
|
|
* Never return 0 for an eligible task regardless of the root bonus and
|
|
* oom_score_adj (oom_score_adj can't be OOM_SCORE_ADJ_MIN here).
|
|
*/
|
|
return points > 0 ? points : 1;
|
|
}
|
|
|
|
/*
|
|
* Determine the type of allocation constraint.
|
|
*/
|
|
#ifdef CONFIG_NUMA
|
|
static enum oom_constraint constrained_alloc(struct zonelist *zonelist,
|
|
gfp_t gfp_mask, nodemask_t *nodemask,
|
|
unsigned long *totalpages)
|
|
{
|
|
struct zone *zone;
|
|
struct zoneref *z;
|
|
enum zone_type high_zoneidx = gfp_zone(gfp_mask);
|
|
bool cpuset_limited = false;
|
|
int nid;
|
|
|
|
/* Default to all available memory */
|
|
*totalpages = totalram_pages + total_swap_pages;
|
|
|
|
if (!zonelist)
|
|
return CONSTRAINT_NONE;
|
|
/*
|
|
* Reach here only when __GFP_NOFAIL is used. So, we should avoid
|
|
* to kill current.We have to random task kill in this case.
|
|
* Hopefully, CONSTRAINT_THISNODE...but no way to handle it, now.
|
|
*/
|
|
if (gfp_mask & __GFP_THISNODE)
|
|
return CONSTRAINT_NONE;
|
|
|
|
/*
|
|
* This is not a __GFP_THISNODE allocation, so a truncated nodemask in
|
|
* the page allocator means a mempolicy is in effect. Cpuset policy
|
|
* is enforced in get_page_from_freelist().
|
|
*/
|
|
if (nodemask && !nodes_subset(node_states[N_MEMORY], *nodemask)) {
|
|
*totalpages = total_swap_pages;
|
|
for_each_node_mask(nid, *nodemask)
|
|
*totalpages += node_spanned_pages(nid);
|
|
return CONSTRAINT_MEMORY_POLICY;
|
|
}
|
|
|
|
/* Check this allocation failure is caused by cpuset's wall function */
|
|
for_each_zone_zonelist_nodemask(zone, z, zonelist,
|
|
high_zoneidx, nodemask)
|
|
if (!cpuset_zone_allowed_softwall(zone, gfp_mask))
|
|
cpuset_limited = true;
|
|
|
|
if (cpuset_limited) {
|
|
*totalpages = total_swap_pages;
|
|
for_each_node_mask(nid, cpuset_current_mems_allowed)
|
|
*totalpages += node_spanned_pages(nid);
|
|
return CONSTRAINT_CPUSET;
|
|
}
|
|
return CONSTRAINT_NONE;
|
|
}
|
|
#else
|
|
static enum oom_constraint constrained_alloc(struct zonelist *zonelist,
|
|
gfp_t gfp_mask, nodemask_t *nodemask,
|
|
unsigned long *totalpages)
|
|
{
|
|
*totalpages = totalram_pages + total_swap_pages;
|
|
return CONSTRAINT_NONE;
|
|
}
|
|
#endif
|
|
|
|
enum oom_scan_t oom_scan_process_thread(struct task_struct *task,
|
|
unsigned long totalpages, const nodemask_t *nodemask,
|
|
bool force_kill)
|
|
{
|
|
if (task->exit_state)
|
|
return OOM_SCAN_CONTINUE;
|
|
if (oom_unkillable_task(task, NULL, nodemask))
|
|
return OOM_SCAN_CONTINUE;
|
|
|
|
/*
|
|
* This task already has access to memory reserves and is being killed.
|
|
* Don't allow any other task to have access to the reserves.
|
|
*/
|
|
if (test_tsk_thread_flag(task, TIF_MEMDIE)) {
|
|
if (unlikely(frozen(task)))
|
|
__thaw_task(task);
|
|
if (!force_kill)
|
|
return OOM_SCAN_ABORT;
|
|
}
|
|
if (!task->mm)
|
|
return OOM_SCAN_CONTINUE;
|
|
|
|
/*
|
|
* If task is allocating a lot of memory and has been marked to be
|
|
* killed first if it triggers an oom, then select it.
|
|
*/
|
|
if (oom_task_origin(task))
|
|
return OOM_SCAN_SELECT;
|
|
|
|
if (task->flags & PF_EXITING && !force_kill) {
|
|
/*
|
|
* If this task is not being ptraced on exit, then wait for it
|
|
* to finish before killing some other task unnecessarily.
|
|
*/
|
|
if (!(task->group_leader->ptrace & PT_TRACE_EXIT))
|
|
return OOM_SCAN_ABORT;
|
|
}
|
|
return OOM_SCAN_OK;
|
|
}
|
|
|
|
/*
|
|
* Simple selection loop. We chose the process with the highest
|
|
* number of 'points'. Returns -1 on scan abort.
|
|
*
|
|
* (not docbooked, we don't want this one cluttering up the manual)
|
|
*/
|
|
static struct task_struct *select_bad_process(unsigned int *ppoints,
|
|
unsigned long totalpages, const nodemask_t *nodemask,
|
|
bool force_kill)
|
|
{
|
|
struct task_struct *g, *p;
|
|
struct task_struct *chosen = NULL;
|
|
unsigned long chosen_points = 0;
|
|
|
|
rcu_read_lock();
|
|
do_each_thread(g, p) {
|
|
unsigned int points;
|
|
|
|
switch (oom_scan_process_thread(p, totalpages, nodemask,
|
|
force_kill)) {
|
|
case OOM_SCAN_SELECT:
|
|
chosen = p;
|
|
chosen_points = ULONG_MAX;
|
|
/* fall through */
|
|
case OOM_SCAN_CONTINUE:
|
|
continue;
|
|
case OOM_SCAN_ABORT:
|
|
rcu_read_unlock();
|
|
return (struct task_struct *)(-1UL);
|
|
case OOM_SCAN_OK:
|
|
break;
|
|
};
|
|
points = oom_badness(p, NULL, nodemask, totalpages);
|
|
if (points > chosen_points) {
|
|
chosen = p;
|
|
chosen_points = points;
|
|
}
|
|
} while_each_thread(g, p);
|
|
if (chosen)
|
|
get_task_struct(chosen);
|
|
rcu_read_unlock();
|
|
|
|
*ppoints = chosen_points * 1000 / totalpages;
|
|
return chosen;
|
|
}
|
|
|
|
/**
|
|
* dump_tasks - dump current memory state of all system tasks
|
|
* @memcg: current's memory controller, if constrained
|
|
* @nodemask: nodemask passed to page allocator for mempolicy ooms
|
|
*
|
|
* Dumps the current memory state of all eligible tasks. Tasks not in the same
|
|
* memcg, not in the same cpuset, or bound to a disjoint set of mempolicy nodes
|
|
* are not shown.
|
|
* State information includes task's pid, uid, tgid, vm size, rss, nr_ptes,
|
|
* swapents, oom_score_adj value, and name.
|
|
*/
|
|
static void dump_tasks(const struct mem_cgroup *memcg, const nodemask_t *nodemask)
|
|
{
|
|
struct task_struct *p;
|
|
struct task_struct *task;
|
|
|
|
pr_info("[ pid ] uid tgid total_vm rss nr_ptes swapents oom_score_adj name\n");
|
|
rcu_read_lock();
|
|
for_each_process(p) {
|
|
if (oom_unkillable_task(p, memcg, nodemask))
|
|
continue;
|
|
|
|
task = find_lock_task_mm(p);
|
|
if (!task) {
|
|
/*
|
|
* This is a kthread or all of p's threads have already
|
|
* detached their mm's. There's no need to report
|
|
* them; they can't be oom killed anyway.
|
|
*/
|
|
continue;
|
|
}
|
|
|
|
pr_info("[%5d] %5d %5d %8lu %8lu %7lu %8lu %5hd %s\n",
|
|
task->pid, from_kuid(&init_user_ns, task_uid(task)),
|
|
task->tgid, task->mm->total_vm, get_mm_rss(task->mm),
|
|
task->mm->nr_ptes,
|
|
get_mm_counter(task->mm, MM_SWAPENTS),
|
|
task->signal->oom_score_adj, task->comm);
|
|
task_unlock(task);
|
|
}
|
|
rcu_read_unlock();
|
|
}
|
|
|
|
static void dump_header(struct task_struct *p, gfp_t gfp_mask, int order,
|
|
struct mem_cgroup *memcg, const nodemask_t *nodemask)
|
|
{
|
|
task_lock(current);
|
|
pr_warning("%s invoked oom-killer: gfp_mask=0x%x, order=%d, "
|
|
"oom_score_adj=%hd\n",
|
|
current->comm, gfp_mask, order,
|
|
current->signal->oom_score_adj);
|
|
cpuset_print_task_mems_allowed(current);
|
|
task_unlock(current);
|
|
dump_stack();
|
|
if (memcg)
|
|
mem_cgroup_print_oom_info(memcg, p);
|
|
else
|
|
show_mem(SHOW_MEM_FILTER_NODES);
|
|
if (sysctl_oom_dump_tasks)
|
|
dump_tasks(memcg, nodemask);
|
|
}
|
|
|
|
#define K(x) ((x) << (PAGE_SHIFT-10))
|
|
/*
|
|
* Must be called while holding a reference to p, which will be released upon
|
|
* returning.
|
|
*/
|
|
void oom_kill_process(struct task_struct *p, gfp_t gfp_mask, int order,
|
|
unsigned int points, unsigned long totalpages,
|
|
struct mem_cgroup *memcg, nodemask_t *nodemask,
|
|
const char *message)
|
|
{
|
|
struct task_struct *victim = p;
|
|
struct task_struct *child;
|
|
struct task_struct *t = p;
|
|
struct mm_struct *mm;
|
|
unsigned int victim_points = 0;
|
|
static DEFINE_RATELIMIT_STATE(oom_rs, DEFAULT_RATELIMIT_INTERVAL,
|
|
DEFAULT_RATELIMIT_BURST);
|
|
|
|
/*
|
|
* If the task is already exiting, don't alarm the sysadmin or kill
|
|
* its children or threads, just set TIF_MEMDIE so it can die quickly
|
|
*/
|
|
if (p->flags & PF_EXITING) {
|
|
set_tsk_thread_flag(p, TIF_MEMDIE);
|
|
put_task_struct(p);
|
|
return;
|
|
}
|
|
|
|
if (__ratelimit(&oom_rs))
|
|
dump_header(p, gfp_mask, order, memcg, nodemask);
|
|
|
|
task_lock(p);
|
|
pr_err("%s: Kill process %d (%s) score %d or sacrifice child\n",
|
|
message, task_pid_nr(p), p->comm, points);
|
|
task_unlock(p);
|
|
|
|
/*
|
|
* If any of p's children has a different mm and is eligible for kill,
|
|
* the one with the highest oom_badness() score is sacrificed for its
|
|
* parent. This attempts to lose the minimal amount of work done while
|
|
* still freeing memory.
|
|
*/
|
|
read_lock(&tasklist_lock);
|
|
do {
|
|
list_for_each_entry(child, &t->children, sibling) {
|
|
unsigned int child_points;
|
|
|
|
if (child->mm == p->mm)
|
|
continue;
|
|
/*
|
|
* oom_badness() returns 0 if the thread is unkillable
|
|
*/
|
|
child_points = oom_badness(child, memcg, nodemask,
|
|
totalpages);
|
|
if (child_points > victim_points) {
|
|
put_task_struct(victim);
|
|
victim = child;
|
|
victim_points = child_points;
|
|
get_task_struct(victim);
|
|
}
|
|
}
|
|
} while_each_thread(p, t);
|
|
read_unlock(&tasklist_lock);
|
|
|
|
rcu_read_lock();
|
|
p = find_lock_task_mm(victim);
|
|
if (!p) {
|
|
rcu_read_unlock();
|
|
put_task_struct(victim);
|
|
return;
|
|
} else if (victim != p) {
|
|
get_task_struct(p);
|
|
put_task_struct(victim);
|
|
victim = p;
|
|
}
|
|
|
|
/* mm cannot safely be dereferenced after task_unlock(victim) */
|
|
mm = victim->mm;
|
|
pr_err("Killed process %d (%s) total-vm:%lukB, anon-rss:%lukB, file-rss:%lukB\n",
|
|
task_pid_nr(victim), victim->comm, K(victim->mm->total_vm),
|
|
K(get_mm_counter(victim->mm, MM_ANONPAGES)),
|
|
K(get_mm_counter(victim->mm, MM_FILEPAGES)));
|
|
task_unlock(victim);
|
|
|
|
/*
|
|
* Kill all user processes sharing victim->mm in other thread groups, if
|
|
* any. They don't get access to memory reserves, though, to avoid
|
|
* depletion of all memory. This prevents mm->mmap_sem livelock when an
|
|
* oom killed thread cannot exit because it requires the semaphore and
|
|
* its contended by another thread trying to allocate memory itself.
|
|
* That thread will now get access to memory reserves since it has a
|
|
* pending fatal signal.
|
|
*/
|
|
for_each_process(p)
|
|
if (p->mm == mm && !same_thread_group(p, victim) &&
|
|
!(p->flags & PF_KTHREAD)) {
|
|
if (p->signal->oom_score_adj == OOM_SCORE_ADJ_MIN)
|
|
continue;
|
|
|
|
task_lock(p); /* Protect ->comm from prctl() */
|
|
pr_err("Kill process %d (%s) sharing same memory\n",
|
|
task_pid_nr(p), p->comm);
|
|
task_unlock(p);
|
|
do_send_sig_info(SIGKILL, SEND_SIG_FORCED, p, true);
|
|
}
|
|
rcu_read_unlock();
|
|
|
|
set_tsk_thread_flag(victim, TIF_MEMDIE);
|
|
do_send_sig_info(SIGKILL, SEND_SIG_FORCED, victim, true);
|
|
put_task_struct(victim);
|
|
}
|
|
#undef K
|
|
|
|
/*
|
|
* Determines whether the kernel must panic because of the panic_on_oom sysctl.
|
|
*/
|
|
void check_panic_on_oom(enum oom_constraint constraint, gfp_t gfp_mask,
|
|
int order, const nodemask_t *nodemask)
|
|
{
|
|
if (likely(!sysctl_panic_on_oom))
|
|
return;
|
|
if (sysctl_panic_on_oom != 2) {
|
|
/*
|
|
* panic_on_oom == 1 only affects CONSTRAINT_NONE, the kernel
|
|
* does not panic for cpuset, mempolicy, or memcg allocation
|
|
* failures.
|
|
*/
|
|
if (constraint != CONSTRAINT_NONE)
|
|
return;
|
|
}
|
|
dump_header(NULL, gfp_mask, order, NULL, nodemask);
|
|
panic("Out of memory: %s panic_on_oom is enabled\n",
|
|
sysctl_panic_on_oom == 2 ? "compulsory" : "system-wide");
|
|
}
|
|
|
|
static BLOCKING_NOTIFIER_HEAD(oom_notify_list);
|
|
|
|
int register_oom_notifier(struct notifier_block *nb)
|
|
{
|
|
return blocking_notifier_chain_register(&oom_notify_list, nb);
|
|
}
|
|
EXPORT_SYMBOL_GPL(register_oom_notifier);
|
|
|
|
int unregister_oom_notifier(struct notifier_block *nb)
|
|
{
|
|
return blocking_notifier_chain_unregister(&oom_notify_list, nb);
|
|
}
|
|
EXPORT_SYMBOL_GPL(unregister_oom_notifier);
|
|
|
|
/*
|
|
* Try to acquire the OOM killer lock for the zones in zonelist. Returns zero
|
|
* if a parallel OOM killing is already taking place that includes a zone in
|
|
* the zonelist. Otherwise, locks all zones in the zonelist and returns 1.
|
|
*/
|
|
int try_set_zonelist_oom(struct zonelist *zonelist, gfp_t gfp_mask)
|
|
{
|
|
struct zoneref *z;
|
|
struct zone *zone;
|
|
int ret = 1;
|
|
|
|
spin_lock(&zone_scan_lock);
|
|
for_each_zone_zonelist(zone, z, zonelist, gfp_zone(gfp_mask)) {
|
|
if (zone_is_oom_locked(zone)) {
|
|
ret = 0;
|
|
goto out;
|
|
}
|
|
}
|
|
|
|
for_each_zone_zonelist(zone, z, zonelist, gfp_zone(gfp_mask)) {
|
|
/*
|
|
* Lock each zone in the zonelist under zone_scan_lock so a
|
|
* parallel invocation of try_set_zonelist_oom() doesn't succeed
|
|
* when it shouldn't.
|
|
*/
|
|
zone_set_flag(zone, ZONE_OOM_LOCKED);
|
|
}
|
|
|
|
out:
|
|
spin_unlock(&zone_scan_lock);
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* Clears the ZONE_OOM_LOCKED flag for all zones in the zonelist so that failed
|
|
* allocation attempts with zonelists containing them may now recall the OOM
|
|
* killer, if necessary.
|
|
*/
|
|
void clear_zonelist_oom(struct zonelist *zonelist, gfp_t gfp_mask)
|
|
{
|
|
struct zoneref *z;
|
|
struct zone *zone;
|
|
|
|
spin_lock(&zone_scan_lock);
|
|
for_each_zone_zonelist(zone, z, zonelist, gfp_zone(gfp_mask)) {
|
|
zone_clear_flag(zone, ZONE_OOM_LOCKED);
|
|
}
|
|
spin_unlock(&zone_scan_lock);
|
|
}
|
|
|
|
/**
|
|
* out_of_memory - kill the "best" process when we run out of memory
|
|
* @zonelist: zonelist pointer
|
|
* @gfp_mask: memory allocation flags
|
|
* @order: amount of memory being requested as a power of 2
|
|
* @nodemask: nodemask passed to page allocator
|
|
* @force_kill: true if a task must be killed, even if others are exiting
|
|
*
|
|
* If we run out of memory, we have the choice between either
|
|
* killing a random task (bad), letting the system crash (worse)
|
|
* OR try to be smart about which process to kill. Note that we
|
|
* don't have to be perfect here, we just have to be good.
|
|
*/
|
|
void out_of_memory(struct zonelist *zonelist, gfp_t gfp_mask,
|
|
int order, nodemask_t *nodemask, bool force_kill)
|
|
{
|
|
const nodemask_t *mpol_mask;
|
|
struct task_struct *p;
|
|
unsigned long totalpages;
|
|
unsigned long freed = 0;
|
|
unsigned int uninitialized_var(points);
|
|
enum oom_constraint constraint = CONSTRAINT_NONE;
|
|
int killed = 0;
|
|
|
|
blocking_notifier_call_chain(&oom_notify_list, 0, &freed);
|
|
if (freed > 0)
|
|
/* Got some memory back in the last second. */
|
|
return;
|
|
|
|
/*
|
|
* If current has a pending SIGKILL or is exiting, then automatically
|
|
* select it. The goal is to allow it to allocate so that it may
|
|
* quickly exit and free its memory.
|
|
*/
|
|
if (fatal_signal_pending(current) || current->flags & PF_EXITING) {
|
|
set_thread_flag(TIF_MEMDIE);
|
|
return;
|
|
}
|
|
|
|
/*
|
|
* Check if there were limitations on the allocation (only relevant for
|
|
* NUMA) that may require different handling.
|
|
*/
|
|
constraint = constrained_alloc(zonelist, gfp_mask, nodemask,
|
|
&totalpages);
|
|
mpol_mask = (constraint == CONSTRAINT_MEMORY_POLICY) ? nodemask : NULL;
|
|
check_panic_on_oom(constraint, gfp_mask, order, mpol_mask);
|
|
|
|
if (sysctl_oom_kill_allocating_task && current->mm &&
|
|
!oom_unkillable_task(current, NULL, nodemask) &&
|
|
current->signal->oom_score_adj != OOM_SCORE_ADJ_MIN) {
|
|
get_task_struct(current);
|
|
oom_kill_process(current, gfp_mask, order, 0, totalpages, NULL,
|
|
nodemask,
|
|
"Out of memory (oom_kill_allocating_task)");
|
|
goto out;
|
|
}
|
|
|
|
p = select_bad_process(&points, totalpages, mpol_mask, force_kill);
|
|
/* Found nothing?!?! Either we hang forever, or we panic. */
|
|
if (!p) {
|
|
dump_header(NULL, gfp_mask, order, NULL, mpol_mask);
|
|
panic("Out of memory and no killable processes...\n");
|
|
}
|
|
if (p != (void *)-1UL) {
|
|
oom_kill_process(p, gfp_mask, order, points, totalpages, NULL,
|
|
nodemask, "Out of memory");
|
|
killed = 1;
|
|
}
|
|
out:
|
|
/*
|
|
* Give the killed threads a good chance of exiting before trying to
|
|
* allocate memory again.
|
|
*/
|
|
if (killed)
|
|
schedule_timeout_killable(1);
|
|
}
|
|
|
|
/*
|
|
* The pagefault handler calls here because it is out of memory, so kill a
|
|
* memory-hogging task. If any populated zone has ZONE_OOM_LOCKED set, a
|
|
* parallel oom killing is already in progress so do nothing.
|
|
*/
|
|
void pagefault_out_of_memory(void)
|
|
{
|
|
struct zonelist *zonelist;
|
|
|
|
if (mem_cgroup_oom_synchronize())
|
|
return;
|
|
|
|
zonelist = node_zonelist(first_online_node, GFP_KERNEL);
|
|
if (try_set_zonelist_oom(zonelist, GFP_KERNEL)) {
|
|
out_of_memory(NULL, 0, 0, NULL, false);
|
|
clear_zonelist_oom(zonelist, GFP_KERNEL);
|
|
}
|
|
}
|