mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-11-30 11:36:40 +07:00
1077fa36f2
There are a number of situations where the mandatory barriers rmb() and wmb() are used to order memory/memory operations in the device drivers and those barriers are much heavier than they actually need to be. For example in the case of PowerPC wmb() calls the heavy-weight sync instruction when for coherent memory operations all that is really needed is an lsync or eieio instruction. This commit adds a coherent only version of the mandatory memory barriers rmb() and wmb(). In most cases this should result in the barrier being the same as the SMP barriers for the SMP case, however in some cases we use a barrier that is somewhere in between rmb() and smp_rmb(). For example on ARM the rmb barriers break down as follows: Barrier Call Explanation --------- -------- ---------------------------------- rmb() dsb() Data synchronization barrier - system dma_rmb() dmb(osh) data memory barrier - outer sharable smp_rmb() dmb(ish) data memory barrier - inner sharable These new barriers are not as safe as the standard rmb() and wmb(). Specifically they do not guarantee ordering between coherent and incoherent memories. The primary use case for these would be to enforce ordering of reads and writes when accessing coherent memory that is shared between the CPU and a device. It may also be noted that there is no dma_mb(). Most architectures don't provide a good mechanism for performing a coherent only full barrier without resorting to the same mechanism used in mb(). As such there isn't much to be gained in trying to define such a function. Cc: Frederic Weisbecker <fweisbec@gmail.com> Cc: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> Cc: Michael Ellerman <michael@ellerman.id.au> Cc: Michael Neuling <mikey@neuling.org> Cc: Russell King <linux@arm.linux.org.uk> Cc: Geert Uytterhoeven <geert@linux-m68k.org> Cc: Heiko Carstens <heiko.carstens@de.ibm.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Martin Schwidefsky <schwidefsky@de.ibm.com> Cc: Tony Luck <tony.luck@intel.com> Cc: Oleg Nesterov <oleg@redhat.com> Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Ingo Molnar <mingo@kernel.org> Cc: David Miller <davem@davemloft.net> Acked-by: Benjamin Herrenschmidt <benh@kernel.crashing.org> Acked-by: Will Deacon <will.deacon@arm.com> Signed-off-by: Alexander Duyck <alexander.h.duyck@redhat.com> Signed-off-by: David S. Miller <davem@davemloft.net>
91 lines
2.8 KiB
C
91 lines
2.8 KiB
C
#ifndef __ASM_BARRIER_H
|
|
#define __ASM_BARRIER_H
|
|
|
|
#ifndef __ASSEMBLY__
|
|
#include <asm/outercache.h>
|
|
|
|
#define nop() __asm__ __volatile__("mov\tr0,r0\t@ nop\n\t");
|
|
|
|
#if __LINUX_ARM_ARCH__ >= 7 || \
|
|
(__LINUX_ARM_ARCH__ == 6 && defined(CONFIG_CPU_32v6K))
|
|
#define sev() __asm__ __volatile__ ("sev" : : : "memory")
|
|
#define wfe() __asm__ __volatile__ ("wfe" : : : "memory")
|
|
#define wfi() __asm__ __volatile__ ("wfi" : : : "memory")
|
|
#endif
|
|
|
|
#if __LINUX_ARM_ARCH__ >= 7
|
|
#define isb(option) __asm__ __volatile__ ("isb " #option : : : "memory")
|
|
#define dsb(option) __asm__ __volatile__ ("dsb " #option : : : "memory")
|
|
#define dmb(option) __asm__ __volatile__ ("dmb " #option : : : "memory")
|
|
#elif defined(CONFIG_CPU_XSC3) || __LINUX_ARM_ARCH__ == 6
|
|
#define isb(x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c5, 4" \
|
|
: : "r" (0) : "memory")
|
|
#define dsb(x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 4" \
|
|
: : "r" (0) : "memory")
|
|
#define dmb(x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 5" \
|
|
: : "r" (0) : "memory")
|
|
#elif defined(CONFIG_CPU_FA526)
|
|
#define isb(x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c5, 4" \
|
|
: : "r" (0) : "memory")
|
|
#define dsb(x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 4" \
|
|
: : "r" (0) : "memory")
|
|
#define dmb(x) __asm__ __volatile__ ("" : : : "memory")
|
|
#else
|
|
#define isb(x) __asm__ __volatile__ ("" : : : "memory")
|
|
#define dsb(x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 4" \
|
|
: : "r" (0) : "memory")
|
|
#define dmb(x) __asm__ __volatile__ ("" : : : "memory")
|
|
#endif
|
|
|
|
#ifdef CONFIG_ARCH_HAS_BARRIERS
|
|
#include <mach/barriers.h>
|
|
#elif defined(CONFIG_ARM_DMA_MEM_BUFFERABLE) || defined(CONFIG_SMP)
|
|
#define mb() do { dsb(); outer_sync(); } while (0)
|
|
#define rmb() dsb()
|
|
#define wmb() do { dsb(st); outer_sync(); } while (0)
|
|
#define dma_rmb() dmb(osh)
|
|
#define dma_wmb() dmb(oshst)
|
|
#else
|
|
#define mb() barrier()
|
|
#define rmb() barrier()
|
|
#define wmb() barrier()
|
|
#define dma_rmb() barrier()
|
|
#define dma_wmb() barrier()
|
|
#endif
|
|
|
|
#ifndef CONFIG_SMP
|
|
#define smp_mb() barrier()
|
|
#define smp_rmb() barrier()
|
|
#define smp_wmb() barrier()
|
|
#else
|
|
#define smp_mb() dmb(ish)
|
|
#define smp_rmb() smp_mb()
|
|
#define smp_wmb() dmb(ishst)
|
|
#endif
|
|
|
|
#define smp_store_release(p, v) \
|
|
do { \
|
|
compiletime_assert_atomic_type(*p); \
|
|
smp_mb(); \
|
|
ACCESS_ONCE(*p) = (v); \
|
|
} while (0)
|
|
|
|
#define smp_load_acquire(p) \
|
|
({ \
|
|
typeof(*p) ___p1 = ACCESS_ONCE(*p); \
|
|
compiletime_assert_atomic_type(*p); \
|
|
smp_mb(); \
|
|
___p1; \
|
|
})
|
|
|
|
#define read_barrier_depends() do { } while(0)
|
|
#define smp_read_barrier_depends() do { } while(0)
|
|
|
|
#define set_mb(var, value) do { var = value; smp_mb(); } while (0)
|
|
|
|
#define smp_mb__before_atomic() smp_mb()
|
|
#define smp_mb__after_atomic() smp_mb()
|
|
|
|
#endif /* !__ASSEMBLY__ */
|
|
#endif /* __ASM_BARRIER_H */
|