AuxXxilium/linux_dsm_epyc7002
1
0
Fork 0
mirror of https://github.com/AuxXxilium/linux_dsm_epyc7002.git synced 2024-11-25 11:40:53 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity
5d6ac29b9e
linux_dsm_epyc7002/arch/sparc/lib/NGpage.S

138 lines
3.5 KiB
ArmAsm
Raw Normal View History

[SPARC64]: Niagara copy/clear page. Happily we have no D-cache aliasing issues on these chips, so the implementation is very straightforward. Add a stub in bootup which will be where the patching calls will be made for niagara/sun4v/hypervisor. Signed-off-by: David S. Miller <davem@davemloft.net>
2006-02-08 07:09:12 +07:00
/* NGpage.S: Niagara optimize clear and copy page.
*
* Copyright (C) 2006 (davem@davemloft.net)
*/
#include <asm/asi.h>
#include <asm/page.h>
.text
.align 32
/* This is heavily simplified from the sun4u variants
* because Niagara does not have any D-cache aliasing issues
* and also we don't need to use the FPU in order to implement
* an optimal page copy/clear.
*/
NGcopy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */
sparc: Minor tweaks to Niagara page copy/clear. Don't use floating point on Niagara2, use the traditional plain Niagara code instead. Unroll Niagara loops to 128 bytes for copy, and 256 bytes for clear. Signed-off-by: David S. Miller <davem@davemloft.net>
2011-08-02 08:18:57 +07:00
save %sp, -192, %sp
rd %asi, %g3
wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
[SPARC64]: Niagara copy/clear page. Happily we have no D-cache aliasing issues on these chips, so the implementation is very straightforward. Add a stub in bootup which will be where the patching calls will be made for niagara/sun4v/hypervisor. Signed-off-by: David S. Miller <davem@davemloft.net>
2006-02-08 07:09:12 +07:00
set PAGE_SIZE, %g7
sparc: Minor tweaks to Niagara page copy/clear. Don't use floating point on Niagara2, use the traditional plain Niagara code instead. Unroll Niagara loops to 128 bytes for copy, and 256 bytes for clear. Signed-off-by: David S. Miller <davem@davemloft.net>
2011-08-02 08:18:57 +07:00
prefetch [%i1 + 0x00], #one_read
prefetch [%i1 + 0x40], #one_read
[SPARC64]: Niagara copy/clear page. Happily we have no D-cache aliasing issues on these chips, so the implementation is very straightforward. Add a stub in bootup which will be where the patching calls will be made for niagara/sun4v/hypervisor. Signed-off-by: David S. Miller <davem@davemloft.net>
2006-02-08 07:09:12 +07:00
sparc: Minor tweaks to Niagara page copy/clear. Don't use floating point on Niagara2, use the traditional plain Niagara code instead. Unroll Niagara loops to 128 bytes for copy, and 256 bytes for clear. Signed-off-by: David S. Miller <davem@davemloft.net>
2011-08-02 08:18:57 +07:00
1: prefetch [%i1 + 0x80], #one_read
prefetch [%i1 + 0xc0], #one_read
ldda [%i1 + 0x00] %asi, %o2
ldda [%i1 + 0x10] %asi, %o4
ldda [%i1 + 0x20] %asi, %l2
ldda [%i1 + 0x30] %asi, %l4
stxa %o2, [%i0 + 0x00] %asi
stxa %o3, [%i0 + 0x08] %asi
stxa %o4, [%i0 + 0x10] %asi
stxa %o5, [%i0 + 0x18] %asi
stxa %l2, [%i0 + 0x20] %asi
stxa %l3, [%i0 + 0x28] %asi
stxa %l4, [%i0 + 0x30] %asi
stxa %l5, [%i0 + 0x38] %asi
ldda [%i1 + 0x40] %asi, %o2
ldda [%i1 + 0x50] %asi, %o4
ldda [%i1 + 0x60] %asi, %l2
ldda [%i1 + 0x70] %asi, %l4
stxa %o2, [%i0 + 0x40] %asi
stxa %o3, [%i0 + 0x48] %asi
stxa %o4, [%i0 + 0x50] %asi
stxa %o5, [%i0 + 0x58] %asi
stxa %l2, [%i0 + 0x60] %asi
stxa %l3, [%i0 + 0x68] %asi
stxa %l4, [%i0 + 0x70] %asi
stxa %l5, [%i0 + 0x78] %asi
add %i1, 128, %i1
subcc %g7, 128, %g7
[SPARC64]: Niagara copy/clear page. Happily we have no D-cache aliasing issues on these chips, so the implementation is very straightforward. Add a stub in bootup which will be where the patching calls will be made for niagara/sun4v/hypervisor. Signed-off-by: David S. Miller <davem@davemloft.net>
2006-02-08 07:09:12 +07:00
bne,pt %xcc, 1b
sparc: Minor tweaks to Niagara page copy/clear. Don't use floating point on Niagara2, use the traditional plain Niagara code instead. Unroll Niagara loops to 128 bytes for copy, and 256 bytes for clear. Signed-off-by: David S. Miller <davem@davemloft.net>
2011-08-02 08:18:57 +07:00
add %i0, 128, %i0
wr %g3, 0x0, %asi
[SPARC64]: store-init needs trailing membar. The manual says that it is required and we actually have crash reports where loads see stale data due to not having membars here. In one case the networking does: memset(skb, 0, offsetof(struct sk_buff, truesize)); and then some code later checks skb->nohdr for zero, but it's still the value that was there before the memset(). Note that arch/sparc64/lib/xor.S already got this right. Signed-off-by: David S. Miller <davem@davemloft.net>
2007-03-20 03:27:33 +07:00
membar #Sync
sparc: Minor tweaks to Niagara page copy/clear. Don't use floating point on Niagara2, use the traditional plain Niagara code instead. Unroll Niagara loops to 128 bytes for copy, and 256 bytes for clear. Signed-off-by: David S. Miller <davem@davemloft.net>
2011-08-02 08:18:57 +07:00
ret
restore
[SPARC64]: Niagara copy/clear page. Happily we have no D-cache aliasing issues on these chips, so the implementation is very straightforward. Add a stub in bootup which will be where the patching calls will be made for niagara/sun4v/hypervisor. Signed-off-by: David S. Miller <davem@davemloft.net>
2006-02-08 07:09:12 +07:00
sparc: Minor tweaks to Niagara page copy/clear. Don't use floating point on Niagara2, use the traditional plain Niagara code instead. Unroll Niagara loops to 128 bytes for copy, and 256 bytes for clear. Signed-off-by: David S. Miller <davem@davemloft.net>
2011-08-02 08:18:57 +07:00
.align 32
sparc64: Add SPARC-T4 optimized memcpy. Before After -------------- -------------- bw_tcp: 1288.53 MB/sec 1637.77 MB/sec bw_pipe: 1517.18 MB/sec 2107.61 MB/sec bw_unix: 1838.38 MB/sec 2640.91 MB/sec make -s -j128 allmodconfig 5min 49sec 5min 31sec Signed-off-by: David S. Miller <davem@davemloft.net>
2012-09-27 11:11:01 +07:00
.globl NGclear_page
.globl NGclear_user_page
[SPARC64]: Niagara copy/clear page. Happily we have no D-cache aliasing issues on these chips, so the implementation is very straightforward. Add a stub in bootup which will be where the patching calls will be made for niagara/sun4v/hypervisor. Signed-off-by: David S. Miller <davem@davemloft.net>
2006-02-08 07:09:12 +07:00
NGclear_page: /* %o0=dest */
NGclear_user_page: /* %o0=dest, %o1=vaddr */
sparc: Minor tweaks to Niagara page copy/clear. Don't use floating point on Niagara2, use the traditional plain Niagara code instead. Unroll Niagara loops to 128 bytes for copy, and 256 bytes for clear. Signed-off-by: David S. Miller <davem@davemloft.net>
2011-08-02 08:18:57 +07:00
rd %asi, %g3
wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
[SPARC64]: Niagara copy/clear page. Happily we have no D-cache aliasing issues on these chips, so the implementation is very straightforward. Add a stub in bootup which will be where the patching calls will be made for niagara/sun4v/hypervisor. Signed-off-by: David S. Miller <davem@davemloft.net>
2006-02-08 07:09:12 +07:00
set PAGE_SIZE, %g7
sparc: Minor tweaks to Niagara page copy/clear. Don't use floating point on Niagara2, use the traditional plain Niagara code instead. Unroll Niagara loops to 128 bytes for copy, and 256 bytes for clear. Signed-off-by: David S. Miller <davem@davemloft.net>
2011-08-02 08:18:57 +07:00
1: stxa %g0, [%o0 + 0x00] %asi
stxa %g0, [%o0 + 0x08] %asi
stxa %g0, [%o0 + 0x10] %asi
stxa %g0, [%o0 + 0x18] %asi
stxa %g0, [%o0 + 0x20] %asi
stxa %g0, [%o0 + 0x28] %asi
stxa %g0, [%o0 + 0x30] %asi
stxa %g0, [%o0 + 0x38] %asi
stxa %g0, [%o0 + 0x40] %asi
stxa %g0, [%o0 + 0x48] %asi
stxa %g0, [%o0 + 0x50] %asi
stxa %g0, [%o0 + 0x58] %asi
stxa %g0, [%o0 + 0x60] %asi
stxa %g0, [%o0 + 0x68] %asi
stxa %g0, [%o0 + 0x70] %asi
stxa %g0, [%o0 + 0x78] %asi
stxa %g0, [%o0 + 0x80] %asi
stxa %g0, [%o0 + 0x88] %asi
stxa %g0, [%o0 + 0x90] %asi
stxa %g0, [%o0 + 0x98] %asi
stxa %g0, [%o0 + 0xa0] %asi
stxa %g0, [%o0 + 0xa8] %asi
stxa %g0, [%o0 + 0xb0] %asi
stxa %g0, [%o0 + 0xb8] %asi
stxa %g0, [%o0 + 0xc0] %asi
stxa %g0, [%o0 + 0xc8] %asi
stxa %g0, [%o0 + 0xd0] %asi
stxa %g0, [%o0 + 0xd8] %asi
stxa %g0, [%o0 + 0xe0] %asi
stxa %g0, [%o0 + 0xe8] %asi
stxa %g0, [%o0 + 0xf0] %asi
stxa %g0, [%o0 + 0xf8] %asi
subcc %g7, 256, %g7
[SPARC64]: Niagara copy/clear page. Happily we have no D-cache aliasing issues on these chips, so the implementation is very straightforward. Add a stub in bootup which will be where the patching calls will be made for niagara/sun4v/hypervisor. Signed-off-by: David S. Miller <davem@davemloft.net>
2006-02-08 07:09:12 +07:00
bne,pt %xcc, 1b
sparc: Minor tweaks to Niagara page copy/clear. Don't use floating point on Niagara2, use the traditional plain Niagara code instead. Unroll Niagara loops to 128 bytes for copy, and 256 bytes for clear. Signed-off-by: David S. Miller <davem@davemloft.net>
2011-08-02 08:18:57 +07:00
add %o0, 256, %o0
wr %g3, 0x0, %asi
[SPARC64]: store-init needs trailing membar. The manual says that it is required and we actually have crash reports where loads see stale data due to not having membars here. In one case the networking does: memset(skb, 0, offsetof(struct sk_buff, truesize)); and then some code later checks skb->nohdr for zero, but it's still the value that was there before the memset(). Note that arch/sparc64/lib/xor.S already got this right. Signed-off-by: David S. Miller <davem@davemloft.net>
2007-03-20 03:27:33 +07:00
membar #Sync
[SPARC64]: Niagara copy/clear page. Happily we have no D-cache aliasing issues on these chips, so the implementation is very straightforward. Add a stub in bootup which will be where the patching calls will be made for niagara/sun4v/hypervisor. Signed-off-by: David S. Miller <davem@davemloft.net>
2006-02-08 07:09:12 +07:00
retl
nop
#define BRANCH_ALWAYS 0x10680000
#define NOP 0x01000000
#define NG_DO_PATCH(OLD, NEW) \
sethi %hi(NEW), %g1; \
or %g1, %lo(NEW), %g1; \
sethi %hi(OLD), %g2; \
or %g2, %lo(OLD), %g2; \
sub %g1, %g2, %g1; \
sethi %hi(BRANCH_ALWAYS), %g3; \
[SPARC64]: Fix branch signedness bug in all code patching. The bug that hit SUN4V TLB patching exists elsewhere. Make sure we cure all such cases. Signed-off-by: David S. Miller <davem@davemloft.net>
2006-02-12 14:28:40 +07:00
sll %g1, 11, %g1; \
srl %g1, 11 + 2, %g1; \
[SPARC64]: Niagara copy/clear page. Happily we have no D-cache aliasing issues on these chips, so the implementation is very straightforward. Add a stub in bootup which will be where the patching calls will be made for niagara/sun4v/hypervisor. Signed-off-by: David S. Miller <davem@davemloft.net>
2006-02-08 07:09:12 +07:00
or %g3, %lo(BRANCH_ALWAYS), %g3; \
or %g3, %g1, %g3; \
stw %g3, [%g2]; \
sethi %hi(NOP), %g3; \
or %g3, %lo(NOP), %g3; \
stw %g3, [%g2 + 0x4]; \
flush %g2;
.globl niagara_patch_pageops
.type niagara_patch_pageops,#function
niagara_patch_pageops:
NG_DO_PATCH(copy_user_page, NGcopy_user_page)
NG_DO_PATCH(_clear_page, NGclear_page)
NG_DO_PATCH(clear_user_page, NGclear_user_page)
retl
nop
.size niagara_patch_pageops,.-niagara_patch_pageops
Reference in New Issue Copy Permalink