mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-04 14:26:41 +07:00
d72da4a4d9
Change the insert and erase code such that lockless searches are
non-fatal.
In and of itself an rbtree cannot be correctly searched while
in-modification, we can however provide weaker guarantees that will
allow the rbtree to be used in conjunction with other techniques, such
as latches; see 9b0fd802e8 ("seqcount: Add raw_write_seqcount_latch()").
For this to work we need the following guarantees from the rbtree
code:
1) a lockless reader must not see partial stores, this would allow it
to observe nodes that are invalid memory.
2) there must not be (temporary) loops in the tree structure in the
modifier's program order, this would cause a lookup which
interrupts the modifier to get stuck indefinitely.
For 1) we must use WRITE_ONCE() for all updates to the tree structure;
in particular this patch only does rb_{left,right} as those are the
only element required for simple searches.
It generates slightly worse code, probably because volatile. But in
pointer chasing heavy code a few instructions more should not matter.
For 2) I have carefully audited the code and drawn every intermediate
link state and not found a loop.
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: David Woodhouse <David.Woodhouse@intel.com>
Cc: Rik van Riel <riel@redhat.com>
Reviewed-by: Michel Lespinasse <walken@google.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
119 lines
3.9 KiB
C
119 lines
3.9 KiB
C
/*
|
|
Red Black Trees
|
|
(C) 1999 Andrea Arcangeli <andrea@suse.de>
|
|
|
|
This program is free software; you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation; either version 2 of the License, or
|
|
(at your option) any later version.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this program; if not, write to the Free Software
|
|
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
linux/include/linux/rbtree.h
|
|
|
|
To use rbtrees you'll have to implement your own insert and search cores.
|
|
This will avoid us to use callbacks and to drop drammatically performances.
|
|
I know it's not the cleaner way, but in C (not in C++) to get
|
|
performances and genericity...
|
|
|
|
See Documentation/rbtree.txt for documentation and samples.
|
|
*/
|
|
|
|
#ifndef _LINUX_RBTREE_H
|
|
#define _LINUX_RBTREE_H
|
|
|
|
#include <linux/kernel.h>
|
|
#include <linux/stddef.h>
|
|
#include <linux/rcupdate.h>
|
|
|
|
struct rb_node {
|
|
unsigned long __rb_parent_color;
|
|
struct rb_node *rb_right;
|
|
struct rb_node *rb_left;
|
|
} __attribute__((aligned(sizeof(long))));
|
|
/* The alignment might seem pointless, but allegedly CRIS needs it */
|
|
|
|
struct rb_root {
|
|
struct rb_node *rb_node;
|
|
};
|
|
|
|
|
|
#define rb_parent(r) ((struct rb_node *)((r)->__rb_parent_color & ~3))
|
|
|
|
#define RB_ROOT (struct rb_root) { NULL, }
|
|
#define rb_entry(ptr, type, member) container_of(ptr, type, member)
|
|
|
|
#define RB_EMPTY_ROOT(root) ((root)->rb_node == NULL)
|
|
|
|
/* 'empty' nodes are nodes that are known not to be inserted in an rbtree */
|
|
#define RB_EMPTY_NODE(node) \
|
|
((node)->__rb_parent_color == (unsigned long)(node))
|
|
#define RB_CLEAR_NODE(node) \
|
|
((node)->__rb_parent_color = (unsigned long)(node))
|
|
|
|
|
|
extern void rb_insert_color(struct rb_node *, struct rb_root *);
|
|
extern void rb_erase(struct rb_node *, struct rb_root *);
|
|
|
|
|
|
/* Find logical next and previous nodes in a tree */
|
|
extern struct rb_node *rb_next(const struct rb_node *);
|
|
extern struct rb_node *rb_prev(const struct rb_node *);
|
|
extern struct rb_node *rb_first(const struct rb_root *);
|
|
extern struct rb_node *rb_last(const struct rb_root *);
|
|
|
|
/* Postorder iteration - always visit the parent after its children */
|
|
extern struct rb_node *rb_first_postorder(const struct rb_root *);
|
|
extern struct rb_node *rb_next_postorder(const struct rb_node *);
|
|
|
|
/* Fast replacement of a single node without remove/rebalance/add/rebalance */
|
|
extern void rb_replace_node(struct rb_node *victim, struct rb_node *new,
|
|
struct rb_root *root);
|
|
|
|
static inline void rb_link_node(struct rb_node *node, struct rb_node *parent,
|
|
struct rb_node **rb_link)
|
|
{
|
|
node->__rb_parent_color = (unsigned long)parent;
|
|
node->rb_left = node->rb_right = NULL;
|
|
|
|
*rb_link = node;
|
|
}
|
|
|
|
static inline void rb_link_node_rcu(struct rb_node *node, struct rb_node *parent,
|
|
struct rb_node **rb_link)
|
|
{
|
|
node->__rb_parent_color = (unsigned long)parent;
|
|
node->rb_left = node->rb_right = NULL;
|
|
|
|
rcu_assign_pointer(*rb_link, node);
|
|
}
|
|
|
|
#define rb_entry_safe(ptr, type, member) \
|
|
({ typeof(ptr) ____ptr = (ptr); \
|
|
____ptr ? rb_entry(____ptr, type, member) : NULL; \
|
|
})
|
|
|
|
/**
|
|
* rbtree_postorder_for_each_entry_safe - iterate over rb_root in post order of
|
|
* given type safe against removal of rb_node entry
|
|
*
|
|
* @pos: the 'type *' to use as a loop cursor.
|
|
* @n: another 'type *' to use as temporary storage
|
|
* @root: 'rb_root *' of the rbtree.
|
|
* @field: the name of the rb_node field within 'type'.
|
|
*/
|
|
#define rbtree_postorder_for_each_entry_safe(pos, n, root, field) \
|
|
for (pos = rb_entry_safe(rb_first_postorder(root), typeof(*pos), field); \
|
|
pos && ({ n = rb_entry_safe(rb_next_postorder(&pos->field), \
|
|
typeof(*pos), field); 1; }); \
|
|
pos = n)
|
|
|
|
#endif /* _LINUX_RBTREE_H */
|