mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-15 14:06:44 +07:00
5ce6c1f353
Atomics present the same issue with locking: release and acquire
variants need to be strengthened to meet the constraints defined
by the Linux-kernel memory consistency model [1].
Atomics present a further issue: implementations of atomics such
as atomic_cmpxchg() and atomic_add_unless() rely on LR/SC pairs,
which do not give full-ordering with .aqrl; for example, current
implementations allow the "lr-sc-aqrl-pair-vs-full-barrier" test
below to end up with the state indicated in the "exists" clause.
In order to "synchronize" LKMM and RISC-V's implementation, this
commit strengthens the implementations of the atomics operations
by replacing .rl and .aq with the use of ("lightweigth") fences,
and by replacing .aqrl LR/SC pairs in sequences such as:
0: lr.w.aqrl %0, %addr
bne %0, %old, 1f
...
sc.w.aqrl %1, %new, %addr
bnez %1, 0b
1:
with sequences of the form:
0: lr.w %0, %addr
bne %0, %old, 1f
...
sc.w.rl %1, %new, %addr /* SC-release */
bnez %1, 0b
fence rw, rw /* "full" fence */
1:
following Daniel's suggestion.
These modifications were validated with simulation of the RISC-V
memory consistency model.
C lr-sc-aqrl-pair-vs-full-barrier
{}
P0(int *x, int *y, atomic_t *u)
{
int r0;
int r1;
WRITE_ONCE(*x, 1);
r0 = atomic_cmpxchg(u, 0, 1);
r1 = READ_ONCE(*y);
}
P1(int *x, int *y, atomic_t *v)
{
int r0;
int r1;
WRITE_ONCE(*y, 1);
r0 = atomic_cmpxchg(v, 0, 1);
r1 = READ_ONCE(*x);
}
exists (u=1 /\ v=1 /\ 0:r1=0 /\ 1:r1=0)
[1] https://marc.info/?l=linux-kernel&m=151930201102853&w=2
https://groups.google.com/a/groups.riscv.org/forum/#!topic/isa-dev/hKywNHBkAXM
https://marc.info/?l=linux-kernel&m=151633436614259&w=2
Suggested-by: Daniel Lustig <dlustig@nvidia.com>
Signed-off-by: Andrea Parri <parri.andrea@gmail.com>
Cc: Palmer Dabbelt <palmer@sifive.com>
Cc: Albert Ou <albert@sifive.com>
Cc: Daniel Lustig <dlustig@nvidia.com>
Cc: Alan Stern <stern@rowland.harvard.edu>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Jade Alglave <j.alglave@ucl.ac.uk>
Cc: Luc Maranget <luc.maranget@inria.fr>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Akira Yokosawa <akiyks@gmail.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: linux-riscv@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Palmer Dabbelt <palmer@sifive.com>
384 lines
9.6 KiB
C
384 lines
9.6 KiB
C
/*
|
|
* Copyright (C) 2014 Regents of the University of California
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU General Public License
|
|
* as published by the Free Software Foundation, version 2.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*/
|
|
|
|
#ifndef _ASM_RISCV_CMPXCHG_H
|
|
#define _ASM_RISCV_CMPXCHG_H
|
|
|
|
#include <linux/bug.h>
|
|
|
|
#include <asm/barrier.h>
|
|
#include <asm/fence.h>
|
|
|
|
#define __xchg_relaxed(ptr, new, size) \
|
|
({ \
|
|
__typeof__(ptr) __ptr = (ptr); \
|
|
__typeof__(new) __new = (new); \
|
|
__typeof__(*(ptr)) __ret; \
|
|
switch (size) { \
|
|
case 4: \
|
|
__asm__ __volatile__ ( \
|
|
" amoswap.w %0, %2, %1\n" \
|
|
: "=r" (__ret), "+A" (*__ptr) \
|
|
: "r" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
case 8: \
|
|
__asm__ __volatile__ ( \
|
|
" amoswap.d %0, %2, %1\n" \
|
|
: "=r" (__ret), "+A" (*__ptr) \
|
|
: "r" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
default: \
|
|
BUILD_BUG(); \
|
|
} \
|
|
__ret; \
|
|
})
|
|
|
|
#define xchg_relaxed(ptr, x) \
|
|
({ \
|
|
__typeof__(*(ptr)) _x_ = (x); \
|
|
(__typeof__(*(ptr))) __xchg_relaxed((ptr), \
|
|
_x_, sizeof(*(ptr))); \
|
|
})
|
|
|
|
#define __xchg_acquire(ptr, new, size) \
|
|
({ \
|
|
__typeof__(ptr) __ptr = (ptr); \
|
|
__typeof__(new) __new = (new); \
|
|
__typeof__(*(ptr)) __ret; \
|
|
switch (size) { \
|
|
case 4: \
|
|
__asm__ __volatile__ ( \
|
|
" amoswap.w %0, %2, %1\n" \
|
|
RISCV_ACQUIRE_BARRIER \
|
|
: "=r" (__ret), "+A" (*__ptr) \
|
|
: "r" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
case 8: \
|
|
__asm__ __volatile__ ( \
|
|
" amoswap.d %0, %2, %1\n" \
|
|
RISCV_ACQUIRE_BARRIER \
|
|
: "=r" (__ret), "+A" (*__ptr) \
|
|
: "r" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
default: \
|
|
BUILD_BUG(); \
|
|
} \
|
|
__ret; \
|
|
})
|
|
|
|
#define xchg_acquire(ptr, x) \
|
|
({ \
|
|
__typeof__(*(ptr)) _x_ = (x); \
|
|
(__typeof__(*(ptr))) __xchg_acquire((ptr), \
|
|
_x_, sizeof(*(ptr))); \
|
|
})
|
|
|
|
#define __xchg_release(ptr, new, size) \
|
|
({ \
|
|
__typeof__(ptr) __ptr = (ptr); \
|
|
__typeof__(new) __new = (new); \
|
|
__typeof__(*(ptr)) __ret; \
|
|
switch (size) { \
|
|
case 4: \
|
|
__asm__ __volatile__ ( \
|
|
RISCV_RELEASE_BARRIER \
|
|
" amoswap.w %0, %2, %1\n" \
|
|
: "=r" (__ret), "+A" (*__ptr) \
|
|
: "r" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
case 8: \
|
|
__asm__ __volatile__ ( \
|
|
RISCV_RELEASE_BARRIER \
|
|
" amoswap.d %0, %2, %1\n" \
|
|
: "=r" (__ret), "+A" (*__ptr) \
|
|
: "r" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
default: \
|
|
BUILD_BUG(); \
|
|
} \
|
|
__ret; \
|
|
})
|
|
|
|
#define xchg_release(ptr, x) \
|
|
({ \
|
|
__typeof__(*(ptr)) _x_ = (x); \
|
|
(__typeof__(*(ptr))) __xchg_release((ptr), \
|
|
_x_, sizeof(*(ptr))); \
|
|
})
|
|
|
|
#define __xchg(ptr, new, size) \
|
|
({ \
|
|
__typeof__(ptr) __ptr = (ptr); \
|
|
__typeof__(new) __new = (new); \
|
|
__typeof__(*(ptr)) __ret; \
|
|
switch (size) { \
|
|
case 4: \
|
|
__asm__ __volatile__ ( \
|
|
" amoswap.w.aqrl %0, %2, %1\n" \
|
|
: "=r" (__ret), "+A" (*__ptr) \
|
|
: "r" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
case 8: \
|
|
__asm__ __volatile__ ( \
|
|
" amoswap.d.aqrl %0, %2, %1\n" \
|
|
: "=r" (__ret), "+A" (*__ptr) \
|
|
: "r" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
default: \
|
|
BUILD_BUG(); \
|
|
} \
|
|
__ret; \
|
|
})
|
|
|
|
#define xchg(ptr, x) \
|
|
({ \
|
|
__typeof__(*(ptr)) _x_ = (x); \
|
|
(__typeof__(*(ptr))) __xchg((ptr), _x_, sizeof(*(ptr))); \
|
|
})
|
|
|
|
#define xchg32(ptr, x) \
|
|
({ \
|
|
BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
|
|
xchg((ptr), (x)); \
|
|
})
|
|
|
|
#define xchg64(ptr, x) \
|
|
({ \
|
|
BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
|
|
xchg((ptr), (x)); \
|
|
})
|
|
|
|
/*
|
|
* Atomic compare and exchange. Compare OLD with MEM, if identical,
|
|
* store NEW in MEM. Return the initial value in MEM. Success is
|
|
* indicated by comparing RETURN with OLD.
|
|
*/
|
|
#define __cmpxchg_relaxed(ptr, old, new, size) \
|
|
({ \
|
|
__typeof__(ptr) __ptr = (ptr); \
|
|
__typeof__(*(ptr)) __old = (old); \
|
|
__typeof__(*(ptr)) __new = (new); \
|
|
__typeof__(*(ptr)) __ret; \
|
|
register unsigned int __rc; \
|
|
switch (size) { \
|
|
case 4: \
|
|
__asm__ __volatile__ ( \
|
|
"0: lr.w %0, %2\n" \
|
|
" bne %0, %z3, 1f\n" \
|
|
" sc.w %1, %z4, %2\n" \
|
|
" bnez %1, 0b\n" \
|
|
"1:\n" \
|
|
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
|
: "rJ" (__old), "rJ" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
case 8: \
|
|
__asm__ __volatile__ ( \
|
|
"0: lr.d %0, %2\n" \
|
|
" bne %0, %z3, 1f\n" \
|
|
" sc.d %1, %z4, %2\n" \
|
|
" bnez %1, 0b\n" \
|
|
"1:\n" \
|
|
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
|
: "rJ" (__old), "rJ" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
default: \
|
|
BUILD_BUG(); \
|
|
} \
|
|
__ret; \
|
|
})
|
|
|
|
#define cmpxchg_relaxed(ptr, o, n) \
|
|
({ \
|
|
__typeof__(*(ptr)) _o_ = (o); \
|
|
__typeof__(*(ptr)) _n_ = (n); \
|
|
(__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \
|
|
_o_, _n_, sizeof(*(ptr))); \
|
|
})
|
|
|
|
#define __cmpxchg_acquire(ptr, old, new, size) \
|
|
({ \
|
|
__typeof__(ptr) __ptr = (ptr); \
|
|
__typeof__(*(ptr)) __old = (old); \
|
|
__typeof__(*(ptr)) __new = (new); \
|
|
__typeof__(*(ptr)) __ret; \
|
|
register unsigned int __rc; \
|
|
switch (size) { \
|
|
case 4: \
|
|
__asm__ __volatile__ ( \
|
|
"0: lr.w %0, %2\n" \
|
|
" bne %0, %z3, 1f\n" \
|
|
" sc.w %1, %z4, %2\n" \
|
|
" bnez %1, 0b\n" \
|
|
RISCV_ACQUIRE_BARRIER \
|
|
"1:\n" \
|
|
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
|
: "rJ" (__old), "rJ" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
case 8: \
|
|
__asm__ __volatile__ ( \
|
|
"0: lr.d %0, %2\n" \
|
|
" bne %0, %z3, 1f\n" \
|
|
" sc.d %1, %z4, %2\n" \
|
|
" bnez %1, 0b\n" \
|
|
RISCV_ACQUIRE_BARRIER \
|
|
"1:\n" \
|
|
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
|
: "rJ" (__old), "rJ" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
default: \
|
|
BUILD_BUG(); \
|
|
} \
|
|
__ret; \
|
|
})
|
|
|
|
#define cmpxchg_acquire(ptr, o, n) \
|
|
({ \
|
|
__typeof__(*(ptr)) _o_ = (o); \
|
|
__typeof__(*(ptr)) _n_ = (n); \
|
|
(__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \
|
|
_o_, _n_, sizeof(*(ptr))); \
|
|
})
|
|
|
|
#define __cmpxchg_release(ptr, old, new, size) \
|
|
({ \
|
|
__typeof__(ptr) __ptr = (ptr); \
|
|
__typeof__(*(ptr)) __old = (old); \
|
|
__typeof__(*(ptr)) __new = (new); \
|
|
__typeof__(*(ptr)) __ret; \
|
|
register unsigned int __rc; \
|
|
switch (size) { \
|
|
case 4: \
|
|
__asm__ __volatile__ ( \
|
|
RISCV_RELEASE_BARRIER \
|
|
"0: lr.w %0, %2\n" \
|
|
" bne %0, %z3, 1f\n" \
|
|
" sc.w %1, %z4, %2\n" \
|
|
" bnez %1, 0b\n" \
|
|
"1:\n" \
|
|
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
|
: "rJ" (__old), "rJ" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
case 8: \
|
|
__asm__ __volatile__ ( \
|
|
RISCV_RELEASE_BARRIER \
|
|
"0: lr.d %0, %2\n" \
|
|
" bne %0, %z3, 1f\n" \
|
|
" sc.d %1, %z4, %2\n" \
|
|
" bnez %1, 0b\n" \
|
|
"1:\n" \
|
|
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
|
: "rJ" (__old), "rJ" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
default: \
|
|
BUILD_BUG(); \
|
|
} \
|
|
__ret; \
|
|
})
|
|
|
|
#define cmpxchg_release(ptr, o, n) \
|
|
({ \
|
|
__typeof__(*(ptr)) _o_ = (o); \
|
|
__typeof__(*(ptr)) _n_ = (n); \
|
|
(__typeof__(*(ptr))) __cmpxchg_release((ptr), \
|
|
_o_, _n_, sizeof(*(ptr))); \
|
|
})
|
|
|
|
#define __cmpxchg(ptr, old, new, size) \
|
|
({ \
|
|
__typeof__(ptr) __ptr = (ptr); \
|
|
__typeof__(*(ptr)) __old = (old); \
|
|
__typeof__(*(ptr)) __new = (new); \
|
|
__typeof__(*(ptr)) __ret; \
|
|
register unsigned int __rc; \
|
|
switch (size) { \
|
|
case 4: \
|
|
__asm__ __volatile__ ( \
|
|
"0: lr.w %0, %2\n" \
|
|
" bne %0, %z3, 1f\n" \
|
|
" sc.w.rl %1, %z4, %2\n" \
|
|
" bnez %1, 0b\n" \
|
|
" fence rw, rw\n" \
|
|
"1:\n" \
|
|
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
|
: "rJ" (__old), "rJ" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
case 8: \
|
|
__asm__ __volatile__ ( \
|
|
"0: lr.d %0, %2\n" \
|
|
" bne %0, %z3, 1f\n" \
|
|
" sc.d.rl %1, %z4, %2\n" \
|
|
" bnez %1, 0b\n" \
|
|
" fence rw, rw\n" \
|
|
"1:\n" \
|
|
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
|
: "rJ" (__old), "rJ" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
default: \
|
|
BUILD_BUG(); \
|
|
} \
|
|
__ret; \
|
|
})
|
|
|
|
#define cmpxchg(ptr, o, n) \
|
|
({ \
|
|
__typeof__(*(ptr)) _o_ = (o); \
|
|
__typeof__(*(ptr)) _n_ = (n); \
|
|
(__typeof__(*(ptr))) __cmpxchg((ptr), \
|
|
_o_, _n_, sizeof(*(ptr))); \
|
|
})
|
|
|
|
#define cmpxchg_local(ptr, o, n) \
|
|
(__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr))))
|
|
|
|
#define cmpxchg32(ptr, o, n) \
|
|
({ \
|
|
BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
|
|
cmpxchg((ptr), (o), (n)); \
|
|
})
|
|
|
|
#define cmpxchg32_local(ptr, o, n) \
|
|
({ \
|
|
BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
|
|
cmpxchg_relaxed((ptr), (o), (n)) \
|
|
})
|
|
|
|
#define cmpxchg64(ptr, o, n) \
|
|
({ \
|
|
BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
|
|
cmpxchg((ptr), (o), (n)); \
|
|
})
|
|
|
|
#define cmpxchg64_local(ptr, o, n) \
|
|
({ \
|
|
BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
|
|
cmpxchg_relaxed((ptr), (o), (n)); \
|
|
})
|
|
|
|
#endif /* _ASM_RISCV_CMPXCHG_H */
|