2005-04-17 05:20:36 +07:00
|
|
|
#ifndef _LINUX_FUTEX_H
|
|
|
|
|
#define _LINUX_FUTEX_H
|
|
|
|
|
|
2006-03-27 16:16:22 +07:00
|
|
|
#include <linux/sched.h>
|
|
|
|
|
|
2005-04-17 05:20:36 +07:00
|
|
|
/* Second argument to futex syscall */
|
|
|
|
|
|
|
|
|
|
|
[PATCH] FUTEX_WAKE_OP: pthread_cond_signal() speedup
ATM pthread_cond_signal is unnecessarily slow, because it wakes one waiter
(which at least on UP usually means an immediate context switch to one of
the waiter threads). This waiter wakes up and after a few instructions it
attempts to acquire the cv internal lock, but that lock is still held by
the thread calling pthread_cond_signal. So it goes to sleep and eventually
the signalling thread is scheduled in, unlocks the internal lock and wakes
the waiter again.
Now, before 2003-09-21 NPTL was using FUTEX_REQUEUE in pthread_cond_signal
to avoid this performance issue, but it was removed when locks were
redesigned to the 3 state scheme (unlocked, locked uncontended, locked
contended).
Following scenario shows why simply using FUTEX_REQUEUE in
pthread_cond_signal together with using lll_mutex_unlock_force in place of
lll_mutex_unlock is not enough and probably why it has been disabled at
that time:
The number is value in cv->__data.__lock.
thr1 thr2 thr3
0 pthread_cond_wait
1 lll_mutex_lock (cv->__data.__lock)
0 lll_mutex_unlock (cv->__data.__lock)
0 lll_futex_wait (&cv->__data.__futex, futexval)
0 pthread_cond_signal
1 lll_mutex_lock (cv->__data.__lock)
1 pthread_cond_signal
2 lll_mutex_lock (cv->__data.__lock)
2 lll_futex_wait (&cv->__data.__lock, 2)
2 lll_futex_requeue (&cv->__data.__futex, 0, 1, &cv->__data.__lock)
# FUTEX_REQUEUE, not FUTEX_CMP_REQUEUE
2 lll_mutex_unlock_force (cv->__data.__lock)
0 cv->__data.__lock = 0
0 lll_futex_wake (&cv->__data.__lock, 1)
1 lll_mutex_lock (cv->__data.__lock)
0 lll_mutex_unlock (cv->__data.__lock)
# Here, lll_mutex_unlock doesn't know there are threads waiting
# on the internal cv's lock
Now, I believe it is possible to use FUTEX_REQUEUE in pthread_cond_signal,
but it will cost us not one, but 2 extra syscalls and, what's worse, one of
these extra syscalls will be done for every single waiting loop in
pthread_cond_*wait.
We would need to use lll_mutex_unlock_force in pthread_cond_signal after
requeue and lll_mutex_cond_lock in pthread_cond_*wait after lll_futex_wait.
Another alternative is to do the unlocking pthread_cond_signal needs to do
(the lock can't be unlocked before lll_futex_wake, as that is racy) in the
kernel.
I have implemented both variants, futex-requeue-glibc.patch is the first
one and futex-wake_op{,-glibc}.patch is the unlocking inside of the kernel.
The kernel interface allows userland to specify how exactly an unlocking
operation should look like (some atomic arithmetic operation with optional
constant argument and comparison of the previous futex value with another
constant).
It has been implemented just for ppc*, x86_64 and i?86, for other
architectures I'm including just a stub header which can be used as a
starting point by maintainers to write support for their arches and ATM
will just return -ENOSYS for FUTEX_WAKE_OP. The requeue patch has been
(lightly) tested just on x86_64, the wake_op patch on ppc64 kernel running
32-bit and 64-bit NPTL and x86_64 kernel running 32-bit and 64-bit NPTL.
With the following benchmark on UP x86-64 I get:
for i in nptl-orig nptl-requeue nptl-wake_op; do echo time elf/ld.so --library-path .:$i /tmp/bench; \
for j in 1 2; do echo ( time elf/ld.so --library-path .:$i /tmp/bench ) 2>&1; done; done
time elf/ld.so --library-path .:nptl-orig /tmp/bench
real 0m0.655s user 0m0.253s sys 0m0.403s
real 0m0.657s user 0m0.269s sys 0m0.388s
time elf/ld.so --library-path .:nptl-requeue /tmp/bench
real 0m0.496s user 0m0.225s sys 0m0.271s
real 0m0.531s user 0m0.242s sys 0m0.288s
time elf/ld.so --library-path .:nptl-wake_op /tmp/bench
real 0m0.380s user 0m0.176s sys 0m0.204s
real 0m0.382s user 0m0.175s sys 0m0.207s
The benchmark is at:
http://sourceware.org/ml/libc-alpha/2005-03/txt00001.txt
Older futex-requeue-glibc.patch version is at:
http://sourceware.org/ml/libc-alpha/2005-03/txt00002.txt
Older futex-wake_op-glibc.patch version is at:
http://sourceware.org/ml/libc-alpha/2005-03/txt00003.txt
Will post a new version (just x86-64 fixes so that the patch
applies against pthread_cond_signal.S) to libc-hacker ml soon.
Attached is the kernel FUTEX_WAKE_OP patch as well as a simple-minded
testcase that will not test the atomicity of the operation, but at least
check if the threads that should have been woken up are woken up and
whether the arithmetic operation in the kernel gave the expected results.
Acked-by: Ingo Molnar <mingo@redhat.com>
Cc: Ulrich Drepper <drepper@redhat.com>
Cc: Jamie Lokier <jamie@shareable.org>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Signed-off-by: Yoichi Yuasa <yuasa@hh.iij4u.or.jp>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-07 05:16:25 +07:00
|
|
|
#define FUTEX_WAIT 0
|
|
|
|
|
#define FUTEX_WAKE 1
|
|
|
|
|
#define FUTEX_FD 2
|
|
|
|
|
#define FUTEX_REQUEUE 3
|
|
|
|
|
#define FUTEX_CMP_REQUEUE 4
|
|
|
|
|
#define FUTEX_WAKE_OP 5
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2006-03-27 16:16:22 +07:00
|
|
|
/*
|
|
|
|
|
* Support for robust futexes: the kernel cleans up held futexes at
|
|
|
|
|
* thread exit time.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Per-lock list entry - embedded in user-space locks, somewhere close
|
|
|
|
|
* to the futex field. (Note: user-space uses a double-linked list to
|
|
|
|
|
* achieve O(1) list add and remove, but the kernel only needs to know
|
|
|
|
|
* about the forward link)
|
|
|
|
|
*
|
|
|
|
|
* NOTE: this structure is part of the syscall ABI, and must not be
|
|
|
|
|
* changed.
|
|
|
|
|
*/
|
|
|
|
|
struct robust_list {
|
|
|
|
|
struct robust_list __user *next;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Per-thread list head:
|
|
|
|
|
*
|
|
|
|
|
* NOTE: this structure is part of the syscall ABI, and must only be
|
|
|
|
|
* changed if the change is first communicated with the glibc folks.
|
|
|
|
|
* (When an incompatible change is done, we'll increase the structure
|
|
|
|
|
* size, which glibc will detect)
|
|
|
|
|
*/
|
|
|
|
|
struct robust_list_head {
|
|
|
|
|
/*
|
|
|
|
|
* The head of the list. Points back to itself if empty:
|
|
|
|
|
*/
|
|
|
|
|
struct robust_list list;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* This relative offset is set by user-space, it gives the kernel
|
|
|
|
|
* the relative position of the futex field to examine. This way
|
|
|
|
|
* we keep userspace flexible, to freely shape its data-structure,
|
|
|
|
|
* without hardcoding any particular offset into the kernel:
|
|
|
|
|
*/
|
|
|
|
|
long futex_offset;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* The death of the thread may race with userspace setting
|
|
|
|
|
* up a lock's links. So to handle this race, userspace first
|
|
|
|
|
* sets this field to the address of the to-be-taken lock,
|
|
|
|
|
* then does the lock acquire, and then adds itself to the
|
|
|
|
|
* list, and then clears this field. Hence the kernel will
|
|
|
|
|
* always have full knowledge of all locks that the thread
|
|
|
|
|
* _might_ have taken. We check the owner TID in any case,
|
|
|
|
|
* so only truly owned locks will be handled.
|
|
|
|
|
*/
|
|
|
|
|
struct robust_list __user *list_op_pending;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Are there any waiters for this robust futex:
|
|
|
|
|
*/
|
|
|
|
|
#define FUTEX_WAITERS 0x80000000
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* The kernel signals via this bit that a thread holding a futex
|
|
|
|
|
* has exited without unlocking the futex. The kernel also does
|
|
|
|
|
* a FUTEX_WAKE on such futexes, after setting the bit, to wake
|
|
|
|
|
* up any possible waiters:
|
|
|
|
|
*/
|
|
|
|
|
#define FUTEX_OWNER_DIED 0x40000000
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* The rest of the robust-futex field is for the TID:
|
|
|
|
|
*/
|
2006-03-27 16:16:28 +07:00
|
|
|
#define FUTEX_TID_MASK 0x3fffffff
|
2006-03-27 16:16:22 +07:00
|
|
|
|
|
|
|
|
/*
|
2006-03-27 16:16:28 +07:00
|
|
|
* This limit protects against a deliberately circular list.
|
|
|
|
|
* (Not worth introducing an rlimit for it)
|
2006-03-27 16:16:22 +07:00
|
|
|
*/
|
2006-03-27 16:16:28 +07:00
|
|
|
#define ROBUST_LIST_LIMIT 2048
|
2006-03-27 16:16:22 +07:00
|
|
|
|
2005-04-17 05:20:36 +07:00
|
|
|
long do_futex(unsigned long uaddr, int op, int val,
|
|
|
|
|
unsigned long timeout, unsigned long uaddr2, int val2,
|
|
|
|
|
int val3);
|
|
|
|
|
|
2006-03-27 16:16:27 +07:00
|
|
|
extern int handle_futex_death(u32 __user *uaddr, struct task_struct *curr);
|
2006-03-27 16:16:22 +07:00
|
|
|
|
|
|
|
|
#ifdef CONFIG_FUTEX
|
|
|
|
|
extern void exit_robust_list(struct task_struct *curr);
|
|
|
|
|
#else
|
|
|
|
|
static inline void exit_robust_list(struct task_struct *curr)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
[PATCH] FUTEX_WAKE_OP: pthread_cond_signal() speedup
ATM pthread_cond_signal is unnecessarily slow, because it wakes one waiter
(which at least on UP usually means an immediate context switch to one of
the waiter threads). This waiter wakes up and after a few instructions it
attempts to acquire the cv internal lock, but that lock is still held by
the thread calling pthread_cond_signal. So it goes to sleep and eventually
the signalling thread is scheduled in, unlocks the internal lock and wakes
the waiter again.
Now, before 2003-09-21 NPTL was using FUTEX_REQUEUE in pthread_cond_signal
to avoid this performance issue, but it was removed when locks were
redesigned to the 3 state scheme (unlocked, locked uncontended, locked
contended).
Following scenario shows why simply using FUTEX_REQUEUE in
pthread_cond_signal together with using lll_mutex_unlock_force in place of
lll_mutex_unlock is not enough and probably why it has been disabled at
that time:
The number is value in cv->__data.__lock.
thr1 thr2 thr3
0 pthread_cond_wait
1 lll_mutex_lock (cv->__data.__lock)
0 lll_mutex_unlock (cv->__data.__lock)
0 lll_futex_wait (&cv->__data.__futex, futexval)
0 pthread_cond_signal
1 lll_mutex_lock (cv->__data.__lock)
1 pthread_cond_signal
2 lll_mutex_lock (cv->__data.__lock)
2 lll_futex_wait (&cv->__data.__lock, 2)
2 lll_futex_requeue (&cv->__data.__futex, 0, 1, &cv->__data.__lock)
# FUTEX_REQUEUE, not FUTEX_CMP_REQUEUE
2 lll_mutex_unlock_force (cv->__data.__lock)
0 cv->__data.__lock = 0
0 lll_futex_wake (&cv->__data.__lock, 1)
1 lll_mutex_lock (cv->__data.__lock)
0 lll_mutex_unlock (cv->__data.__lock)
# Here, lll_mutex_unlock doesn't know there are threads waiting
# on the internal cv's lock
Now, I believe it is possible to use FUTEX_REQUEUE in pthread_cond_signal,
but it will cost us not one, but 2 extra syscalls and, what's worse, one of
these extra syscalls will be done for every single waiting loop in
pthread_cond_*wait.
We would need to use lll_mutex_unlock_force in pthread_cond_signal after
requeue and lll_mutex_cond_lock in pthread_cond_*wait after lll_futex_wait.
Another alternative is to do the unlocking pthread_cond_signal needs to do
(the lock can't be unlocked before lll_futex_wake, as that is racy) in the
kernel.
I have implemented both variants, futex-requeue-glibc.patch is the first
one and futex-wake_op{,-glibc}.patch is the unlocking inside of the kernel.
The kernel interface allows userland to specify how exactly an unlocking
operation should look like (some atomic arithmetic operation with optional
constant argument and comparison of the previous futex value with another
constant).
It has been implemented just for ppc*, x86_64 and i?86, for other
architectures I'm including just a stub header which can be used as a
starting point by maintainers to write support for their arches and ATM
will just return -ENOSYS for FUTEX_WAKE_OP. The requeue patch has been
(lightly) tested just on x86_64, the wake_op patch on ppc64 kernel running
32-bit and 64-bit NPTL and x86_64 kernel running 32-bit and 64-bit NPTL.
With the following benchmark on UP x86-64 I get:
for i in nptl-orig nptl-requeue nptl-wake_op; do echo time elf/ld.so --library-path .:$i /tmp/bench; \
for j in 1 2; do echo ( time elf/ld.so --library-path .:$i /tmp/bench ) 2>&1; done; done
time elf/ld.so --library-path .:nptl-orig /tmp/bench
real 0m0.655s user 0m0.253s sys 0m0.403s
real 0m0.657s user 0m0.269s sys 0m0.388s
time elf/ld.so --library-path .:nptl-requeue /tmp/bench
real 0m0.496s user 0m0.225s sys 0m0.271s
real 0m0.531s user 0m0.242s sys 0m0.288s
time elf/ld.so --library-path .:nptl-wake_op /tmp/bench
real 0m0.380s user 0m0.176s sys 0m0.204s
real 0m0.382s user 0m0.175s sys 0m0.207s
The benchmark is at:
http://sourceware.org/ml/libc-alpha/2005-03/txt00001.txt
Older futex-requeue-glibc.patch version is at:
http://sourceware.org/ml/libc-alpha/2005-03/txt00002.txt
Older futex-wake_op-glibc.patch version is at:
http://sourceware.org/ml/libc-alpha/2005-03/txt00003.txt
Will post a new version (just x86-64 fixes so that the patch
applies against pthread_cond_signal.S) to libc-hacker ml soon.
Attached is the kernel FUTEX_WAKE_OP patch as well as a simple-minded
testcase that will not test the atomicity of the operation, but at least
check if the threads that should have been woken up are woken up and
whether the arithmetic operation in the kernel gave the expected results.
Acked-by: Ingo Molnar <mingo@redhat.com>
Cc: Ulrich Drepper <drepper@redhat.com>
Cc: Jamie Lokier <jamie@shareable.org>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Signed-off-by: Yoichi Yuasa <yuasa@hh.iij4u.or.jp>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-07 05:16:25 +07:00
|
|
|
#define FUTEX_OP_SET 0 /* *(int *)UADDR2 = OPARG; */
|
|
|
|
|
#define FUTEX_OP_ADD 1 /* *(int *)UADDR2 += OPARG; */
|
|
|
|
|
#define FUTEX_OP_OR 2 /* *(int *)UADDR2 |= OPARG; */
|
|
|
|
|
#define FUTEX_OP_ANDN 3 /* *(int *)UADDR2 &= ~OPARG; */
|
|
|
|
|
#define FUTEX_OP_XOR 4 /* *(int *)UADDR2 ^= OPARG; */
|
|
|
|
|
|
|
|
|
|
#define FUTEX_OP_OPARG_SHIFT 8 /* Use (1 << OPARG) instead of OPARG. */
|
|
|
|
|
|
|
|
|
|
#define FUTEX_OP_CMP_EQ 0 /* if (oldval == CMPARG) wake */
|
|
|
|
|
#define FUTEX_OP_CMP_NE 1 /* if (oldval != CMPARG) wake */
|
|
|
|
|
#define FUTEX_OP_CMP_LT 2 /* if (oldval < CMPARG) wake */
|
|
|
|
|
#define FUTEX_OP_CMP_LE 3 /* if (oldval <= CMPARG) wake */
|
|
|
|
|
#define FUTEX_OP_CMP_GT 4 /* if (oldval > CMPARG) wake */
|
|
|
|
|
#define FUTEX_OP_CMP_GE 5 /* if (oldval >= CMPARG) wake */
|
|
|
|
|
|
|
|
|
|
/* FUTEX_WAKE_OP will perform atomically
|
|
|
|
|
int oldval = *(int *)UADDR2;
|
|
|
|
|
*(int *)UADDR2 = oldval OP OPARG;
|
|
|
|
|
if (oldval CMP CMPARG)
|
|
|
|
|
wake UADDR2; */
|
|
|
|
|
|
|
|
|
|
#define FUTEX_OP(op, oparg, cmp, cmparg) \
|
|
|
|
|
(((op & 0xf) << 28) | ((cmp & 0xf) << 24) \
|
|
|
|
|
| ((oparg & 0xfff) << 12) | (cmparg & 0xfff))
|
|
|
|
|
|
2005-04-17 05:20:36 +07:00
|
|
|
#endif
|