mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-06 05:36:39 +07:00
e335e3eb82
Get rid of INLINE_SPIN_UNLOCK entirely replacing it with UNINLINE_SPIN_UNLOCK instead of the reverse meaning. Whoever wants to change the default spinlock inlining behavior and uninline the spinlocks for some weird reason, such as spinlock debugging, paravirt etc. can now all just select UNINLINE_SPIN_UNLOCK Original discussion at: https://lkml.org/lkml/2012/3/21/357 Suggested-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Ralf Baechle <ralf@linux-mips.org> Cc: Chris Metcalf <cmetcalf@tilera.com> Cc: Chris Zankel <chris@zankel.net> Cc: linux-mips@linux-mips.org Link: http://lkml.kernel.org/r/20120322095502.30866.75756.sendpatchset@codeblue [ tidied up the changelog a bit ] Signed-off-by: Ingo Molnar <mingo@kernel.org>
386 lines
9.5 KiB
C
386 lines
9.5 KiB
C
/*
|
|
* Copyright (2004) Linus Torvalds
|
|
*
|
|
* Author: Zwane Mwaikambo <zwane@fsmlabs.com>
|
|
*
|
|
* Copyright (2004, 2005) Ingo Molnar
|
|
*
|
|
* This file contains the spinlock/rwlock implementations for the
|
|
* SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
|
|
*
|
|
* Note that some architectures have special knowledge about the
|
|
* stack frames of these functions in their profile_pc. If you
|
|
* change anything significant here that could change the stack
|
|
* frame contact the architecture maintainers.
|
|
*/
|
|
|
|
#include <linux/linkage.h>
|
|
#include <linux/preempt.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/interrupt.h>
|
|
#include <linux/debug_locks.h>
|
|
#include <linux/export.h>
|
|
|
|
/*
|
|
* If lockdep is enabled then we use the non-preemption spin-ops
|
|
* even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
|
|
* not re-enabled during lock-acquire (which the preempt-spin-ops do):
|
|
*/
|
|
#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
|
|
/*
|
|
* The __lock_function inlines are taken from
|
|
* include/linux/spinlock_api_smp.h
|
|
*/
|
|
#else
|
|
#define raw_read_can_lock(l) read_can_lock(l)
|
|
#define raw_write_can_lock(l) write_can_lock(l)
|
|
/*
|
|
* We build the __lock_function inlines here. They are too large for
|
|
* inlining all over the place, but here is only one user per function
|
|
* which embedds them into the calling _lock_function below.
|
|
*
|
|
* This could be a long-held lock. We both prepare to spin for a long
|
|
* time (making _this_ CPU preemptable if possible), and we also signal
|
|
* towards that other CPU that it should break the lock ASAP.
|
|
*/
|
|
#define BUILD_LOCK_OPS(op, locktype) \
|
|
void __lockfunc __raw_##op##_lock(locktype##_t *lock) \
|
|
{ \
|
|
for (;;) { \
|
|
preempt_disable(); \
|
|
if (likely(do_raw_##op##_trylock(lock))) \
|
|
break; \
|
|
preempt_enable(); \
|
|
\
|
|
if (!(lock)->break_lock) \
|
|
(lock)->break_lock = 1; \
|
|
while (!raw_##op##_can_lock(lock) && (lock)->break_lock)\
|
|
arch_##op##_relax(&lock->raw_lock); \
|
|
} \
|
|
(lock)->break_lock = 0; \
|
|
} \
|
|
\
|
|
unsigned long __lockfunc __raw_##op##_lock_irqsave(locktype##_t *lock) \
|
|
{ \
|
|
unsigned long flags; \
|
|
\
|
|
for (;;) { \
|
|
preempt_disable(); \
|
|
local_irq_save(flags); \
|
|
if (likely(do_raw_##op##_trylock(lock))) \
|
|
break; \
|
|
local_irq_restore(flags); \
|
|
preempt_enable(); \
|
|
\
|
|
if (!(lock)->break_lock) \
|
|
(lock)->break_lock = 1; \
|
|
while (!raw_##op##_can_lock(lock) && (lock)->break_lock)\
|
|
arch_##op##_relax(&lock->raw_lock); \
|
|
} \
|
|
(lock)->break_lock = 0; \
|
|
return flags; \
|
|
} \
|
|
\
|
|
void __lockfunc __raw_##op##_lock_irq(locktype##_t *lock) \
|
|
{ \
|
|
_raw_##op##_lock_irqsave(lock); \
|
|
} \
|
|
\
|
|
void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \
|
|
{ \
|
|
unsigned long flags; \
|
|
\
|
|
/* */ \
|
|
/* Careful: we must exclude softirqs too, hence the */ \
|
|
/* irq-disabling. We use the generic preemption-aware */ \
|
|
/* function: */ \
|
|
/**/ \
|
|
flags = _raw_##op##_lock_irqsave(lock); \
|
|
local_bh_disable(); \
|
|
local_irq_restore(flags); \
|
|
} \
|
|
|
|
/*
|
|
* Build preemption-friendly versions of the following
|
|
* lock-spinning functions:
|
|
*
|
|
* __[spin|read|write]_lock()
|
|
* __[spin|read|write]_lock_irq()
|
|
* __[spin|read|write]_lock_irqsave()
|
|
* __[spin|read|write]_lock_bh()
|
|
*/
|
|
BUILD_LOCK_OPS(spin, raw_spinlock);
|
|
BUILD_LOCK_OPS(read, rwlock);
|
|
BUILD_LOCK_OPS(write, rwlock);
|
|
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_SPIN_TRYLOCK
|
|
int __lockfunc _raw_spin_trylock(raw_spinlock_t *lock)
|
|
{
|
|
return __raw_spin_trylock(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_spin_trylock);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_SPIN_TRYLOCK_BH
|
|
int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock)
|
|
{
|
|
return __raw_spin_trylock_bh(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_spin_trylock_bh);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_SPIN_LOCK
|
|
void __lockfunc _raw_spin_lock(raw_spinlock_t *lock)
|
|
{
|
|
__raw_spin_lock(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_spin_lock);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_SPIN_LOCK_IRQSAVE
|
|
unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock)
|
|
{
|
|
return __raw_spin_lock_irqsave(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_spin_lock_irqsave);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_SPIN_LOCK_IRQ
|
|
void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock)
|
|
{
|
|
__raw_spin_lock_irq(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_spin_lock_irq);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_SPIN_LOCK_BH
|
|
void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock)
|
|
{
|
|
__raw_spin_lock_bh(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_spin_lock_bh);
|
|
#endif
|
|
|
|
#ifdef CONFIG_UNINLINE_SPIN_UNLOCK
|
|
void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock)
|
|
{
|
|
__raw_spin_unlock(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_spin_unlock);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE
|
|
void __lockfunc _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)
|
|
{
|
|
__raw_spin_unlock_irqrestore(lock, flags);
|
|
}
|
|
EXPORT_SYMBOL(_raw_spin_unlock_irqrestore);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ
|
|
void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock)
|
|
{
|
|
__raw_spin_unlock_irq(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_spin_unlock_irq);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_SPIN_UNLOCK_BH
|
|
void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock)
|
|
{
|
|
__raw_spin_unlock_bh(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_spin_unlock_bh);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_READ_TRYLOCK
|
|
int __lockfunc _raw_read_trylock(rwlock_t *lock)
|
|
{
|
|
return __raw_read_trylock(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_read_trylock);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_READ_LOCK
|
|
void __lockfunc _raw_read_lock(rwlock_t *lock)
|
|
{
|
|
__raw_read_lock(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_read_lock);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_READ_LOCK_IRQSAVE
|
|
unsigned long __lockfunc _raw_read_lock_irqsave(rwlock_t *lock)
|
|
{
|
|
return __raw_read_lock_irqsave(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_read_lock_irqsave);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_READ_LOCK_IRQ
|
|
void __lockfunc _raw_read_lock_irq(rwlock_t *lock)
|
|
{
|
|
__raw_read_lock_irq(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_read_lock_irq);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_READ_LOCK_BH
|
|
void __lockfunc _raw_read_lock_bh(rwlock_t *lock)
|
|
{
|
|
__raw_read_lock_bh(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_read_lock_bh);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_READ_UNLOCK
|
|
void __lockfunc _raw_read_unlock(rwlock_t *lock)
|
|
{
|
|
__raw_read_unlock(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_read_unlock);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_READ_UNLOCK_IRQRESTORE
|
|
void __lockfunc _raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
|
|
{
|
|
__raw_read_unlock_irqrestore(lock, flags);
|
|
}
|
|
EXPORT_SYMBOL(_raw_read_unlock_irqrestore);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_READ_UNLOCK_IRQ
|
|
void __lockfunc _raw_read_unlock_irq(rwlock_t *lock)
|
|
{
|
|
__raw_read_unlock_irq(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_read_unlock_irq);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_READ_UNLOCK_BH
|
|
void __lockfunc _raw_read_unlock_bh(rwlock_t *lock)
|
|
{
|
|
__raw_read_unlock_bh(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_read_unlock_bh);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_WRITE_TRYLOCK
|
|
int __lockfunc _raw_write_trylock(rwlock_t *lock)
|
|
{
|
|
return __raw_write_trylock(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_write_trylock);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_WRITE_LOCK
|
|
void __lockfunc _raw_write_lock(rwlock_t *lock)
|
|
{
|
|
__raw_write_lock(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_write_lock);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_WRITE_LOCK_IRQSAVE
|
|
unsigned long __lockfunc _raw_write_lock_irqsave(rwlock_t *lock)
|
|
{
|
|
return __raw_write_lock_irqsave(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_write_lock_irqsave);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_WRITE_LOCK_IRQ
|
|
void __lockfunc _raw_write_lock_irq(rwlock_t *lock)
|
|
{
|
|
__raw_write_lock_irq(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_write_lock_irq);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_WRITE_LOCK_BH
|
|
void __lockfunc _raw_write_lock_bh(rwlock_t *lock)
|
|
{
|
|
__raw_write_lock_bh(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_write_lock_bh);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_WRITE_UNLOCK
|
|
void __lockfunc _raw_write_unlock(rwlock_t *lock)
|
|
{
|
|
__raw_write_unlock(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_write_unlock);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQRESTORE
|
|
void __lockfunc _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
|
|
{
|
|
__raw_write_unlock_irqrestore(lock, flags);
|
|
}
|
|
EXPORT_SYMBOL(_raw_write_unlock_irqrestore);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQ
|
|
void __lockfunc _raw_write_unlock_irq(rwlock_t *lock)
|
|
{
|
|
__raw_write_unlock_irq(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_write_unlock_irq);
|
|
#endif
|
|
|
|
#ifndef CONFIG_INLINE_WRITE_UNLOCK_BH
|
|
void __lockfunc _raw_write_unlock_bh(rwlock_t *lock)
|
|
{
|
|
__raw_write_unlock_bh(lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_write_unlock_bh);
|
|
#endif
|
|
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
|
|
void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass)
|
|
{
|
|
preempt_disable();
|
|
spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
|
|
LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_spin_lock_nested);
|
|
|
|
unsigned long __lockfunc _raw_spin_lock_irqsave_nested(raw_spinlock_t *lock,
|
|
int subclass)
|
|
{
|
|
unsigned long flags;
|
|
|
|
local_irq_save(flags);
|
|
preempt_disable();
|
|
spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
|
|
LOCK_CONTENDED_FLAGS(lock, do_raw_spin_trylock, do_raw_spin_lock,
|
|
do_raw_spin_lock_flags, &flags);
|
|
return flags;
|
|
}
|
|
EXPORT_SYMBOL(_raw_spin_lock_irqsave_nested);
|
|
|
|
void __lockfunc _raw_spin_lock_nest_lock(raw_spinlock_t *lock,
|
|
struct lockdep_map *nest_lock)
|
|
{
|
|
preempt_disable();
|
|
spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
|
|
LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
|
|
}
|
|
EXPORT_SYMBOL(_raw_spin_lock_nest_lock);
|
|
|
|
#endif
|
|
|
|
notrace int in_lock_functions(unsigned long addr)
|
|
{
|
|
/* Linker adds these: start and end of __lockfunc functions */
|
|
extern char __lock_text_start[], __lock_text_end[];
|
|
|
|
return addr >= (unsigned long)__lock_text_start
|
|
&& addr < (unsigned long)__lock_text_end;
|
|
}
|
|
EXPORT_SYMBOL(in_lock_functions);
|