mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-28 11:18:45 +07:00
c09d6a04d1
On CPUs which support the LSE atomic instructions introduced in ARMv8.1, it makes sense to use them in preference to ll/sc sequences. This patch introduces runtime patching of atomic_t and atomic64_t routines so that the call-site for the out-of-line ll/sc sequences is patched with an LSE atomic instruction when we detect that the CPU supports it. If binutils is not recent enough to assemble the LSE instructions, then the ll/sc sequences are inlined as though CONFIG_ARM64_LSE_ATOMICS=n. Reviewed-by: Catalin Marinas <catalin.marinas@arm.com> Signed-off-by: Will Deacon <will.deacon@arm.com>
353 lines
7.7 KiB
C
353 lines
7.7 KiB
C
/*
|
|
* Based on arch/arm/include/asm/atomic.h
|
|
*
|
|
* Copyright (C) 1996 Russell King.
|
|
* Copyright (C) 2002 Deep Blue Solutions Ltd.
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
* published by the Free Software Foundation.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
#ifndef __ASM_ATOMIC_LSE_H
|
|
#define __ASM_ATOMIC_LSE_H
|
|
|
|
#ifndef __ARM64_IN_ATOMIC_IMPL
|
|
#error "please don't include this file directly"
|
|
#endif
|
|
|
|
#define __LL_SC_ATOMIC(op) __LL_SC_CALL(atomic_##op)
|
|
|
|
static inline void atomic_andnot(int i, atomic_t *v)
|
|
{
|
|
register int w0 asm ("w0") = i;
|
|
register atomic_t *x1 asm ("x1") = v;
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(andnot),
|
|
" stclr %w[i], %[v]\n")
|
|
: [i] "+r" (w0), [v] "+Q" (v->counter)
|
|
: "r" (x1)
|
|
: "x30");
|
|
}
|
|
|
|
static inline void atomic_or(int i, atomic_t *v)
|
|
{
|
|
register int w0 asm ("w0") = i;
|
|
register atomic_t *x1 asm ("x1") = v;
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(or),
|
|
" stset %w[i], %[v]\n")
|
|
: [i] "+r" (w0), [v] "+Q" (v->counter)
|
|
: "r" (x1)
|
|
: "x30");
|
|
}
|
|
|
|
static inline void atomic_xor(int i, atomic_t *v)
|
|
{
|
|
register int w0 asm ("w0") = i;
|
|
register atomic_t *x1 asm ("x1") = v;
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(xor),
|
|
" steor %w[i], %[v]\n")
|
|
: [i] "+r" (w0), [v] "+Q" (v->counter)
|
|
: "r" (x1)
|
|
: "x30");
|
|
}
|
|
|
|
static inline void atomic_add(int i, atomic_t *v)
|
|
{
|
|
register int w0 asm ("w0") = i;
|
|
register atomic_t *x1 asm ("x1") = v;
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(add),
|
|
" stadd %w[i], %[v]\n")
|
|
: [i] "+r" (w0), [v] "+Q" (v->counter)
|
|
: "r" (x1)
|
|
: "x30");
|
|
}
|
|
|
|
static inline int atomic_add_return(int i, atomic_t *v)
|
|
{
|
|
register int w0 asm ("w0") = i;
|
|
register atomic_t *x1 asm ("x1") = v;
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
/* LL/SC */
|
|
" nop\n"
|
|
__LL_SC_ATOMIC(add_return),
|
|
/* LSE atomics */
|
|
" ldaddal %w[i], w30, %[v]\n"
|
|
" add %w[i], %w[i], w30")
|
|
: [i] "+r" (w0), [v] "+Q" (v->counter)
|
|
: "r" (x1)
|
|
: "x30", "memory");
|
|
|
|
return w0;
|
|
}
|
|
|
|
static inline void atomic_and(int i, atomic_t *v)
|
|
{
|
|
register int w0 asm ("w0") = i;
|
|
register atomic_t *x1 asm ("x1") = v;
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
/* LL/SC */
|
|
" nop\n"
|
|
__LL_SC_ATOMIC(and),
|
|
/* LSE atomics */
|
|
" mvn %w[i], %w[i]\n"
|
|
" stclr %w[i], %[v]")
|
|
: [i] "+r" (w0), [v] "+Q" (v->counter)
|
|
: "r" (x1)
|
|
: "x30");
|
|
}
|
|
|
|
static inline void atomic_sub(int i, atomic_t *v)
|
|
{
|
|
register int w0 asm ("w0") = i;
|
|
register atomic_t *x1 asm ("x1") = v;
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
/* LL/SC */
|
|
" nop\n"
|
|
__LL_SC_ATOMIC(sub),
|
|
/* LSE atomics */
|
|
" neg %w[i], %w[i]\n"
|
|
" stadd %w[i], %[v]")
|
|
: [i] "+r" (w0), [v] "+Q" (v->counter)
|
|
: "r" (x1)
|
|
: "x30");
|
|
}
|
|
|
|
static inline int atomic_sub_return(int i, atomic_t *v)
|
|
{
|
|
register int w0 asm ("w0") = i;
|
|
register atomic_t *x1 asm ("x1") = v;
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
/* LL/SC */
|
|
" nop\n"
|
|
__LL_SC_ATOMIC(sub_return)
|
|
" nop",
|
|
/* LSE atomics */
|
|
" neg %w[i], %w[i]\n"
|
|
" ldaddal %w[i], w30, %[v]\n"
|
|
" add %w[i], %w[i], w30")
|
|
: [i] "+r" (w0), [v] "+Q" (v->counter)
|
|
: "r" (x1)
|
|
: "x30", "memory");
|
|
|
|
return w0;
|
|
}
|
|
|
|
static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new)
|
|
{
|
|
register unsigned long x0 asm ("x0") = (unsigned long)ptr;
|
|
register int w1 asm ("w1") = old;
|
|
register int w2 asm ("w2") = new;
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
/* LL/SC */
|
|
" nop\n"
|
|
__LL_SC_ATOMIC(cmpxchg)
|
|
" nop",
|
|
/* LSE atomics */
|
|
" mov w30, %w[old]\n"
|
|
" casal w30, %w[new], %[v]\n"
|
|
" mov %w[ret], w30")
|
|
: [ret] "+r" (x0), [v] "+Q" (ptr->counter)
|
|
: [old] "r" (w1), [new] "r" (w2)
|
|
: "x30", "cc", "memory");
|
|
|
|
return x0;
|
|
}
|
|
|
|
#undef __LL_SC_ATOMIC
|
|
|
|
#define __LL_SC_ATOMIC64(op) __LL_SC_CALL(atomic64_##op)
|
|
|
|
static inline void atomic64_andnot(long i, atomic64_t *v)
|
|
{
|
|
register long x0 asm ("x0") = i;
|
|
register atomic64_t *x1 asm ("x1") = v;
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(andnot),
|
|
" stclr %[i], %[v]\n")
|
|
: [i] "+r" (x0), [v] "+Q" (v->counter)
|
|
: "r" (x1)
|
|
: "x30");
|
|
}
|
|
|
|
static inline void atomic64_or(long i, atomic64_t *v)
|
|
{
|
|
register long x0 asm ("x0") = i;
|
|
register atomic64_t *x1 asm ("x1") = v;
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(or),
|
|
" stset %[i], %[v]\n")
|
|
: [i] "+r" (x0), [v] "+Q" (v->counter)
|
|
: "r" (x1)
|
|
: "x30");
|
|
}
|
|
|
|
static inline void atomic64_xor(long i, atomic64_t *v)
|
|
{
|
|
register long x0 asm ("x0") = i;
|
|
register atomic64_t *x1 asm ("x1") = v;
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(xor),
|
|
" steor %[i], %[v]\n")
|
|
: [i] "+r" (x0), [v] "+Q" (v->counter)
|
|
: "r" (x1)
|
|
: "x30");
|
|
}
|
|
|
|
static inline void atomic64_add(long i, atomic64_t *v)
|
|
{
|
|
register long x0 asm ("x0") = i;
|
|
register atomic64_t *x1 asm ("x1") = v;
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(add),
|
|
" stadd %[i], %[v]\n")
|
|
: [i] "+r" (x0), [v] "+Q" (v->counter)
|
|
: "r" (x1)
|
|
: "x30");
|
|
}
|
|
|
|
static inline long atomic64_add_return(long i, atomic64_t *v)
|
|
{
|
|
register long x0 asm ("x0") = i;
|
|
register atomic64_t *x1 asm ("x1") = v;
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
/* LL/SC */
|
|
" nop\n"
|
|
__LL_SC_ATOMIC64(add_return),
|
|
/* LSE atomics */
|
|
" ldaddal %[i], x30, %[v]\n"
|
|
" add %[i], %[i], x30")
|
|
: [i] "+r" (x0), [v] "+Q" (v->counter)
|
|
: "r" (x1)
|
|
: "x30", "memory");
|
|
|
|
return x0;
|
|
}
|
|
|
|
static inline void atomic64_and(long i, atomic64_t *v)
|
|
{
|
|
register long x0 asm ("x0") = i;
|
|
register atomic64_t *x1 asm ("x1") = v;
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
/* LL/SC */
|
|
" nop\n"
|
|
__LL_SC_ATOMIC64(and),
|
|
/* LSE atomics */
|
|
" mvn %[i], %[i]\n"
|
|
" stclr %[i], %[v]")
|
|
: [i] "+r" (x0), [v] "+Q" (v->counter)
|
|
: "r" (x1)
|
|
: "x30");
|
|
}
|
|
|
|
static inline void atomic64_sub(long i, atomic64_t *v)
|
|
{
|
|
register long x0 asm ("x0") = i;
|
|
register atomic64_t *x1 asm ("x1") = v;
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
/* LL/SC */
|
|
" nop\n"
|
|
__LL_SC_ATOMIC64(sub),
|
|
/* LSE atomics */
|
|
" neg %[i], %[i]\n"
|
|
" stadd %[i], %[v]")
|
|
: [i] "+r" (x0), [v] "+Q" (v->counter)
|
|
: "r" (x1)
|
|
: "x30");
|
|
}
|
|
|
|
static inline long atomic64_sub_return(long i, atomic64_t *v)
|
|
{
|
|
register long x0 asm ("x0") = i;
|
|
register atomic64_t *x1 asm ("x1") = v;
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
/* LL/SC */
|
|
" nop\n"
|
|
__LL_SC_ATOMIC64(sub_return)
|
|
" nop",
|
|
/* LSE atomics */
|
|
" neg %[i], %[i]\n"
|
|
" ldaddal %[i], x30, %[v]\n"
|
|
" add %[i], %[i], x30")
|
|
: [i] "+r" (x0), [v] "+Q" (v->counter)
|
|
: "r" (x1)
|
|
: "x30", "memory");
|
|
|
|
return x0;
|
|
}
|
|
static inline long atomic64_cmpxchg(atomic64_t *ptr, long old, long new)
|
|
{
|
|
register unsigned long x0 asm ("x0") = (unsigned long)ptr;
|
|
register long x1 asm ("x1") = old;
|
|
register long x2 asm ("x2") = new;
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
/* LL/SC */
|
|
" nop\n"
|
|
__LL_SC_ATOMIC64(cmpxchg)
|
|
" nop",
|
|
/* LSE atomics */
|
|
" mov x30, %[old]\n"
|
|
" casal x30, %[new], %[v]\n"
|
|
" mov %[ret], x30")
|
|
: [ret] "+r" (x0), [v] "+Q" (ptr->counter)
|
|
: [old] "r" (x1), [new] "r" (x2)
|
|
: "x30", "cc", "memory");
|
|
|
|
return x0;
|
|
}
|
|
|
|
static inline long atomic64_dec_if_positive(atomic64_t *v)
|
|
{
|
|
register long x0 asm ("x0") = (long)v;
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
/* LL/SC */
|
|
" nop\n"
|
|
__LL_SC_ATOMIC64(dec_if_positive)
|
|
" nop\n"
|
|
" nop\n"
|
|
" nop\n"
|
|
" nop\n"
|
|
" nop",
|
|
/* LSE atomics */
|
|
"1: ldr x30, %[v]\n"
|
|
" subs %[ret], x30, #1\n"
|
|
" b.mi 2f\n"
|
|
" casal x30, %[ret], %[v]\n"
|
|
" sub x30, x30, #1\n"
|
|
" sub x30, x30, %[ret]\n"
|
|
" cbnz x30, 1b\n"
|
|
"2:")
|
|
: [ret] "+&r" (x0), [v] "+Q" (v->counter)
|
|
:
|
|
: "x30", "cc", "memory");
|
|
|
|
return x0;
|
|
}
|
|
|
|
#undef __LL_SC_ATOMIC64
|
|
|
|
#endif /* __ASM_ATOMIC_LSE_H */
|