mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-19 03:08:34 +07:00
5777eaed56
Apparently there exist certain workloads which rely heavily on software checksumming, for which the generic do_csum() implementation becomes a significant bottleneck. Therefore let's give arm64 its own optimised version - for ease of maintenance this foregoes assembly or intrisics, and is thus not actually arm64-specific, but does rely heavily on C idioms that translate well to the A64 ISA and the typical load/store capabilities of most ARMv8 CPU cores. The resulting increase in checksum throughput scales nicely with buffer size, tending towards 4x for a small in-order core (Cortex-A53), and up to 6x or more for an aggressive big core (Ampere eMAG). Reported-by: Lingyan Huang <huanglingyan2@huawei.com> Tested-by: Lingyan Huang <huanglingyan2@huawei.com> Signed-off-by: Robin Murphy <robin.murphy@arm.com> Signed-off-by: Will Deacon <will@kernel.org>
44 lines
893 B
C
44 lines
893 B
C
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/*
|
|
* Copyright (C) 2016 ARM Ltd.
|
|
*/
|
|
#ifndef __ASM_CHECKSUM_H
|
|
#define __ASM_CHECKSUM_H
|
|
|
|
#include <linux/types.h>
|
|
|
|
static inline __sum16 csum_fold(__wsum csum)
|
|
{
|
|
u32 sum = (__force u32)csum;
|
|
sum += (sum >> 16) | (sum << 16);
|
|
return ~(__force __sum16)(sum >> 16);
|
|
}
|
|
#define csum_fold csum_fold
|
|
|
|
static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl)
|
|
{
|
|
__uint128_t tmp;
|
|
u64 sum;
|
|
|
|
tmp = *(const __uint128_t *)iph;
|
|
iph += 16;
|
|
ihl -= 4;
|
|
tmp += ((tmp >> 64) | (tmp << 64));
|
|
sum = tmp >> 64;
|
|
do {
|
|
sum += *(const u32 *)iph;
|
|
iph += 4;
|
|
} while (--ihl);
|
|
|
|
sum += ((sum >> 32) | (sum << 32));
|
|
return csum_fold((__force u32)(sum >> 32));
|
|
}
|
|
#define ip_fast_csum ip_fast_csum
|
|
|
|
extern unsigned int do_csum(const unsigned char *buff, int len);
|
|
#define do_csum do_csum
|
|
|
|
#include <asm-generic/checksum.h>
|
|
|
|
#endif /* __ASM_CHECKSUM_H */
|