mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-11-24 10:00:51 +07:00
5c2c2587b1
get_dev_page() enables paths like get_user_pages() to pin a dynamically mapped pfn-range (devm_memremap_pages()) while the resulting struct page objects are in use. Unlike get_page() it may fail if the device is, or is in the process of being, disabled. While the initial lookup of the range may be an expensive list walk, the result is cached to speed up subsequent lookups which are likely to be in the same mapped range. devm_memremap_pages() now requires a reference counter to be specified at init time. For pmem this means moving request_queue allocation into pmem_alloc() so the existing queue usage counter can track "device pages". ZONE_DEVICE pages always have an elevated count and will never be on an lru reclaim list. That space in 'struct page' can be redirected for other uses, but for safety introduce a poison value that will always trip __list_add() to assert. This allows half of the struct list_head storage to be reclaimed with some assurance to back up the assumption that the page count never goes to zero and a list_add() is never attempted. Signed-off-by: Dan Williams <dan.j.williams@intel.com> Tested-by: Logan Gunthorpe <logang@deltatee.com> Cc: Dave Hansen <dave@sr71.net> Cc: Matthew Wilcox <willy@linux.intel.com> Cc: Ross Zwisler <ross.zwisler@linux.intel.com> Cc: Alexander Viro <viro@zeniv.linux.org.uk> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
110 lines
2.9 KiB
C
110 lines
2.9 KiB
C
/*
|
|
* Copyright 2006, Red Hat, Inc., Dave Jones
|
|
* Released under the General Public License (GPL).
|
|
*
|
|
* This file contains the linked list implementations for
|
|
* DEBUG_LIST.
|
|
*/
|
|
|
|
#include <linux/export.h>
|
|
#include <linux/list.h>
|
|
#include <linux/bug.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/rculist.h>
|
|
|
|
static struct list_head force_poison;
|
|
void list_force_poison(struct list_head *entry)
|
|
{
|
|
entry->next = &force_poison;
|
|
entry->prev = &force_poison;
|
|
}
|
|
|
|
/*
|
|
* Insert a new entry between two known consecutive entries.
|
|
*
|
|
* This is only for internal list manipulation where we know
|
|
* the prev/next entries already!
|
|
*/
|
|
|
|
void __list_add(struct list_head *new,
|
|
struct list_head *prev,
|
|
struct list_head *next)
|
|
{
|
|
WARN(new->next == &force_poison || new->prev == &force_poison,
|
|
"list_add attempted on force-poisoned entry\n");
|
|
WARN(next->prev != prev,
|
|
"list_add corruption. next->prev should be "
|
|
"prev (%p), but was %p. (next=%p).\n",
|
|
prev, next->prev, next);
|
|
WARN(prev->next != next,
|
|
"list_add corruption. prev->next should be "
|
|
"next (%p), but was %p. (prev=%p).\n",
|
|
next, prev->next, prev);
|
|
WARN(new == prev || new == next,
|
|
"list_add double add: new=%p, prev=%p, next=%p.\n",
|
|
new, prev, next);
|
|
next->prev = new;
|
|
new->next = next;
|
|
new->prev = prev;
|
|
WRITE_ONCE(prev->next, new);
|
|
}
|
|
EXPORT_SYMBOL(__list_add);
|
|
|
|
void __list_del_entry(struct list_head *entry)
|
|
{
|
|
struct list_head *prev, *next;
|
|
|
|
prev = entry->prev;
|
|
next = entry->next;
|
|
|
|
if (WARN(next == LIST_POISON1,
|
|
"list_del corruption, %p->next is LIST_POISON1 (%p)\n",
|
|
entry, LIST_POISON1) ||
|
|
WARN(prev == LIST_POISON2,
|
|
"list_del corruption, %p->prev is LIST_POISON2 (%p)\n",
|
|
entry, LIST_POISON2) ||
|
|
WARN(prev->next != entry,
|
|
"list_del corruption. prev->next should be %p, "
|
|
"but was %p\n", entry, prev->next) ||
|
|
WARN(next->prev != entry,
|
|
"list_del corruption. next->prev should be %p, "
|
|
"but was %p\n", entry, next->prev))
|
|
return;
|
|
|
|
__list_del(prev, next);
|
|
}
|
|
EXPORT_SYMBOL(__list_del_entry);
|
|
|
|
/**
|
|
* list_del - deletes entry from list.
|
|
* @entry: the element to delete from the list.
|
|
* Note: list_empty on entry does not return true after this, the entry is
|
|
* in an undefined state.
|
|
*/
|
|
void list_del(struct list_head *entry)
|
|
{
|
|
__list_del_entry(entry);
|
|
entry->next = LIST_POISON1;
|
|
entry->prev = LIST_POISON2;
|
|
}
|
|
EXPORT_SYMBOL(list_del);
|
|
|
|
/*
|
|
* RCU variants.
|
|
*/
|
|
void __list_add_rcu(struct list_head *new,
|
|
struct list_head *prev, struct list_head *next)
|
|
{
|
|
WARN(next->prev != prev,
|
|
"list_add_rcu corruption. next->prev should be prev (%p), but was %p. (next=%p).\n",
|
|
prev, next->prev, next);
|
|
WARN(prev->next != next,
|
|
"list_add_rcu corruption. prev->next should be next (%p), but was %p. (prev=%p).\n",
|
|
next, prev->next, prev);
|
|
new->next = next;
|
|
new->prev = prev;
|
|
rcu_assign_pointer(list_next_rcu(prev), new);
|
|
next->prev = new;
|
|
}
|
|
EXPORT_SYMBOL(__list_add_rcu);
|