mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-13 17:36:46 +07:00
2077cef4d5
Firstly, handle zero length calls properly. Believe it or not there
are a few of these happening during early boot.
Next, we can't just drop to a memcpy() call in the forward copy case
where dst <= src. The reason is that the cache initializing stores
used in the Niagara memcpy() implementations can end up clearing out
cache lines before we've sourced their original contents completely.
For example, considering NG4memcpy, the main unrolled loop begins like
this:
load src + 0x00
load src + 0x08
load src + 0x10
load src + 0x18
load src + 0x20
store dst + 0x00
Assume dst is 64 byte aligned and let's say that dst is src - 8 for
this memcpy() call. That store at the end there is the one to the
first line in the cache line, thus clearing the whole line, which thus
clobbers "src + 0x28" before it even gets loaded.
To avoid this, just fall through to a simple copy only mildly
optimized for the case where src and dst are 8 byte aligned and the
length is a multiple of 8 as well. We could get fancy and call
GENmemcpy() but this is good enough for how this thing is actually
used.
Reported-by: David Ahern <david.ahern@oracle.com>
Reported-by: Bob Picco <bpicco@meloft.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
60 lines
1.2 KiB
ArmAsm
60 lines
1.2 KiB
ArmAsm
/* memmove.S: Simple memmove implementation.
|
|
*
|
|
* Copyright (C) 1997, 2004 David S. Miller (davem@redhat.com)
|
|
* Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz)
|
|
*/
|
|
|
|
#include <linux/linkage.h>
|
|
|
|
.text
|
|
ENTRY(memmove) /* o0=dst o1=src o2=len */
|
|
brz,pn %o2, 99f
|
|
mov %o0, %g1
|
|
|
|
cmp %o0, %o1
|
|
bleu,pt %xcc, 2f
|
|
add %o1, %o2, %g7
|
|
cmp %g7, %o0
|
|
bleu,pt %xcc, memcpy
|
|
add %o0, %o2, %o5
|
|
sub %g7, 1, %o1
|
|
|
|
sub %o5, 1, %o0
|
|
1: ldub [%o1], %g7
|
|
subcc %o2, 1, %o2
|
|
sub %o1, 1, %o1
|
|
stb %g7, [%o0]
|
|
bne,pt %icc, 1b
|
|
sub %o0, 1, %o0
|
|
99:
|
|
retl
|
|
mov %g1, %o0
|
|
|
|
/* We can't just call memcpy for these memmove cases. On some
|
|
* chips the memcpy uses cache initializing stores and when dst
|
|
* and src are close enough, those can clobber the source data
|
|
* before we've loaded it in.
|
|
*/
|
|
2: or %o0, %o1, %g7
|
|
or %o2, %g7, %g7
|
|
andcc %g7, 0x7, %g0
|
|
bne,pn %xcc, 4f
|
|
nop
|
|
|
|
3: ldx [%o1], %g7
|
|
add %o1, 8, %o1
|
|
subcc %o2, 8, %o2
|
|
add %o0, 8, %o0
|
|
bne,pt %icc, 3b
|
|
stx %g7, [%o0 - 0x8]
|
|
ba,a,pt %xcc, 99b
|
|
|
|
4: ldub [%o1], %g7
|
|
add %o1, 1, %o1
|
|
subcc %o2, 1, %o2
|
|
add %o0, 1, %o0
|
|
bne,pt %icc, 4b
|
|
stb %g7, [%o0 - 0x1]
|
|
ba,a,pt %xcc, 99b
|
|
ENDPROC(memmove)
|