mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-05 15:56:40 +07:00
0ade34c370
We've measured that we spend ~0.6% of sys cpu time in cpumask_next_and(). It's essentially a joined iteration in search for a non-zero bit, which is currently implemented as a lookup join (find a nonzero bit on the lhs, lookup the rhs to see if it's set there). Implement a direct join (find a nonzero bit on the incrementally built join). Also add generic bitmap benchmarks in the new `test_find_bit` module for new function (see `find_next_and_bit` in [2] and [3] below). For cpumask_next_and, direct benchmarking shows that it's 1.17x to 14x faster with a geometric mean of 2.1 on 32 CPUs [1]. No impact on memory usage. Note that on Arm, the new pure-C implementation still outperforms the old one that uses a mix of C and asm (`find_next_bit`) [3]. [1] Approximate benchmark code: ``` unsigned long src1p[nr_cpumask_longs] = {pattern1}; unsigned long src2p[nr_cpumask_longs] = {pattern2}; for (/*a bunch of repetitions*/) { for (int n = -1; n <= nr_cpu_ids; ++n) { asm volatile("" : "+rm"(src1p)); // prevent any optimization asm volatile("" : "+rm"(src2p)); unsigned long result = cpumask_next_and(n, src1p, src2p); asm volatile("" : "+rm"(result)); } } ``` Results: pattern1 pattern2 time_before/time_after 0x0000ffff 0x0000ffff 1.65 0x0000ffff 0x00005555 2.24 0x0000ffff 0x00001111 2.94 0x0000ffff 0x00000000 14.0 0x00005555 0x0000ffff 1.67 0x00005555 0x00005555 1.71 0x00005555 0x00001111 1.90 0x00005555 0x00000000 6.58 0x00001111 0x0000ffff 1.46 0x00001111 0x00005555 1.49 0x00001111 0x00001111 1.45 0x00001111 0x00000000 3.10 0x00000000 0x0000ffff 1.18 0x00000000 0x00005555 1.18 0x00000000 0x00001111 1.17 0x00000000 0x00000000 1.25 ----------------------------- geo.mean 2.06 [2] test_find_next_bit, X86 (skylake) [ 3913.477422] Start testing find_bit() with random-filled bitmap [ 3913.477847] find_next_bit: 160868 cycles, 16484 iterations [ 3913.477933] find_next_zero_bit: 169542 cycles, 16285 iterations [ 3913.478036] find_last_bit: 201638 cycles, 16483 iterations [ 3913.480214] find_first_bit: 4353244 cycles, 16484 iterations [ 3913.480216] Start testing find_next_and_bit() with random-filled bitmap [ 3913.481074] find_next_and_bit: 89604 cycles, 8216 iterations [ 3913.481075] Start testing find_bit() with sparse bitmap [ 3913.481078] find_next_bit: 2536 cycles, 66 iterations [ 3913.481252] find_next_zero_bit: 344404 cycles, 32703 iterations [ 3913.481255] find_last_bit: 2006 cycles, 66 iterations [ 3913.481265] find_first_bit: 17488 cycles, 66 iterations [ 3913.481266] Start testing find_next_and_bit() with sparse bitmap [ 3913.481272] find_next_and_bit: 764 cycles, 1 iterations [3] test_find_next_bit, arm (v7 odroid XU3). [ 267.206928] Start testing find_bit() with random-filled bitmap [ 267.214752] find_next_bit: 4474 cycles, 16419 iterations [ 267.221850] find_next_zero_bit: 5976 cycles, 16350 iterations [ 267.229294] find_last_bit: 4209 cycles, 16419 iterations [ 267.279131] find_first_bit: 1032991 cycles, 16420 iterations [ 267.286265] Start testing find_next_and_bit() with random-filled bitmap [ 267.302386] find_next_and_bit: 2290 cycles, 8140 iterations [ 267.309422] Start testing find_bit() with sparse bitmap [ 267.316054] find_next_bit: 191 cycles, 66 iterations [ 267.322726] find_next_zero_bit: 8758 cycles, 32703 iterations [ 267.329803] find_last_bit: 84 cycles, 66 iterations [ 267.336169] find_first_bit: 4118 cycles, 66 iterations [ 267.342627] Start testing find_next_and_bit() with sparse bitmap [ 267.356919] find_next_and_bit: 91 cycles, 1 iterations [courbet@google.com: v6] Link: http://lkml.kernel.org/r/20171129095715.23430-1-courbet@google.com [geert@linux-m68k.org: m68k/bitops: always include <asm-generic/bitops/find.h>] Link: http://lkml.kernel.org/r/1512556816-28627-1-git-send-email-geert@linux-m68k.org Link: http://lkml.kernel.org/r/20171128131334.23491-1-courbet@google.com Signed-off-by: Clement Courbet <courbet@google.com> Signed-off-by: Geert Uytterhoeven <geert@linux-m68k.org> Cc: Yury Norov <ynorov@caviumnetworks.com> Cc: Geert Uytterhoeven <geert@linux-m68k.org> Cc: Alexey Dobriyan <adobriyan@gmail.com> Cc: Rasmus Villemoes <linux@rasmusvillemoes.dk> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
231 lines
5.8 KiB
C
231 lines
5.8 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
#include <linux/slab.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/bitops.h>
|
|
#include <linux/cpumask.h>
|
|
#include <linux/export.h>
|
|
#include <linux/bootmem.h>
|
|
|
|
/**
|
|
* cpumask_next - get the next cpu in a cpumask
|
|
* @n: the cpu prior to the place to search (ie. return will be > @n)
|
|
* @srcp: the cpumask pointer
|
|
*
|
|
* Returns >= nr_cpu_ids if no further cpus set.
|
|
*/
|
|
unsigned int cpumask_next(int n, const struct cpumask *srcp)
|
|
{
|
|
/* -1 is a legal arg here. */
|
|
if (n != -1)
|
|
cpumask_check(n);
|
|
return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1);
|
|
}
|
|
EXPORT_SYMBOL(cpumask_next);
|
|
|
|
/**
|
|
* cpumask_next_and - get the next cpu in *src1p & *src2p
|
|
* @n: the cpu prior to the place to search (ie. return will be > @n)
|
|
* @src1p: the first cpumask pointer
|
|
* @src2p: the second cpumask pointer
|
|
*
|
|
* Returns >= nr_cpu_ids if no further cpus set in both.
|
|
*/
|
|
int cpumask_next_and(int n, const struct cpumask *src1p,
|
|
const struct cpumask *src2p)
|
|
{
|
|
/* -1 is a legal arg here. */
|
|
if (n != -1)
|
|
cpumask_check(n);
|
|
return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p),
|
|
nr_cpumask_bits, n + 1);
|
|
}
|
|
EXPORT_SYMBOL(cpumask_next_and);
|
|
|
|
/**
|
|
* cpumask_any_but - return a "random" in a cpumask, but not this one.
|
|
* @mask: the cpumask to search
|
|
* @cpu: the cpu to ignore.
|
|
*
|
|
* Often used to find any cpu but smp_processor_id() in a mask.
|
|
* Returns >= nr_cpu_ids if no cpus set.
|
|
*/
|
|
int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)
|
|
{
|
|
unsigned int i;
|
|
|
|
cpumask_check(cpu);
|
|
for_each_cpu(i, mask)
|
|
if (i != cpu)
|
|
break;
|
|
return i;
|
|
}
|
|
EXPORT_SYMBOL(cpumask_any_but);
|
|
|
|
/**
|
|
* cpumask_next_wrap - helper to implement for_each_cpu_wrap
|
|
* @n: the cpu prior to the place to search
|
|
* @mask: the cpumask pointer
|
|
* @start: the start point of the iteration
|
|
* @wrap: assume @n crossing @start terminates the iteration
|
|
*
|
|
* Returns >= nr_cpu_ids on completion
|
|
*
|
|
* Note: the @wrap argument is required for the start condition when
|
|
* we cannot assume @start is set in @mask.
|
|
*/
|
|
int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap)
|
|
{
|
|
int next;
|
|
|
|
again:
|
|
next = cpumask_next(n, mask);
|
|
|
|
if (wrap && n < start && next >= start) {
|
|
return nr_cpumask_bits;
|
|
|
|
} else if (next >= nr_cpumask_bits) {
|
|
wrap = true;
|
|
n = -1;
|
|
goto again;
|
|
}
|
|
|
|
return next;
|
|
}
|
|
EXPORT_SYMBOL(cpumask_next_wrap);
|
|
|
|
/* These are not inline because of header tangles. */
|
|
#ifdef CONFIG_CPUMASK_OFFSTACK
|
|
/**
|
|
* alloc_cpumask_var_node - allocate a struct cpumask on a given node
|
|
* @mask: pointer to cpumask_var_t where the cpumask is returned
|
|
* @flags: GFP_ flags
|
|
*
|
|
* Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
|
|
* a nop returning a constant 1 (in <linux/cpumask.h>)
|
|
* Returns TRUE if memory allocation succeeded, FALSE otherwise.
|
|
*
|
|
* In addition, mask will be NULL if this fails. Note that gcc is
|
|
* usually smart enough to know that mask can never be NULL if
|
|
* CONFIG_CPUMASK_OFFSTACK=n, so does code elimination in that case
|
|
* too.
|
|
*/
|
|
bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
|
|
{
|
|
*mask = kmalloc_node(cpumask_size(), flags, node);
|
|
|
|
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
|
|
if (!*mask) {
|
|
printk(KERN_ERR "=> alloc_cpumask_var: failed!\n");
|
|
dump_stack();
|
|
}
|
|
#endif
|
|
|
|
return *mask != NULL;
|
|
}
|
|
EXPORT_SYMBOL(alloc_cpumask_var_node);
|
|
|
|
bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
|
|
{
|
|
return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node);
|
|
}
|
|
EXPORT_SYMBOL(zalloc_cpumask_var_node);
|
|
|
|
/**
|
|
* alloc_cpumask_var - allocate a struct cpumask
|
|
* @mask: pointer to cpumask_var_t where the cpumask is returned
|
|
* @flags: GFP_ flags
|
|
*
|
|
* Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
|
|
* a nop returning a constant 1 (in <linux/cpumask.h>).
|
|
*
|
|
* See alloc_cpumask_var_node.
|
|
*/
|
|
bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
|
|
{
|
|
return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE);
|
|
}
|
|
EXPORT_SYMBOL(alloc_cpumask_var);
|
|
|
|
bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
|
|
{
|
|
return alloc_cpumask_var(mask, flags | __GFP_ZERO);
|
|
}
|
|
EXPORT_SYMBOL(zalloc_cpumask_var);
|
|
|
|
/**
|
|
* alloc_bootmem_cpumask_var - allocate a struct cpumask from the bootmem arena.
|
|
* @mask: pointer to cpumask_var_t where the cpumask is returned
|
|
*
|
|
* Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
|
|
* a nop (in <linux/cpumask.h>).
|
|
* Either returns an allocated (zero-filled) cpumask, or causes the
|
|
* system to panic.
|
|
*/
|
|
void __init alloc_bootmem_cpumask_var(cpumask_var_t *mask)
|
|
{
|
|
*mask = memblock_virt_alloc(cpumask_size(), 0);
|
|
}
|
|
|
|
/**
|
|
* free_cpumask_var - frees memory allocated for a struct cpumask.
|
|
* @mask: cpumask to free
|
|
*
|
|
* This is safe on a NULL mask.
|
|
*/
|
|
void free_cpumask_var(cpumask_var_t mask)
|
|
{
|
|
kfree(mask);
|
|
}
|
|
EXPORT_SYMBOL(free_cpumask_var);
|
|
|
|
/**
|
|
* free_bootmem_cpumask_var - frees result of alloc_bootmem_cpumask_var
|
|
* @mask: cpumask to free
|
|
*/
|
|
void __init free_bootmem_cpumask_var(cpumask_var_t mask)
|
|
{
|
|
memblock_free_early(__pa(mask), cpumask_size());
|
|
}
|
|
#endif
|
|
|
|
/**
|
|
* cpumask_local_spread - select the i'th cpu with local numa cpu's first
|
|
* @i: index number
|
|
* @node: local numa_node
|
|
*
|
|
* This function selects an online CPU according to a numa aware policy;
|
|
* local cpus are returned first, followed by non-local ones, then it
|
|
* wraps around.
|
|
*
|
|
* It's not very efficient, but useful for setup.
|
|
*/
|
|
unsigned int cpumask_local_spread(unsigned int i, int node)
|
|
{
|
|
int cpu;
|
|
|
|
/* Wrap: we always want a cpu. */
|
|
i %= num_online_cpus();
|
|
|
|
if (node == -1) {
|
|
for_each_cpu(cpu, cpu_online_mask)
|
|
if (i-- == 0)
|
|
return cpu;
|
|
} else {
|
|
/* NUMA first. */
|
|
for_each_cpu_and(cpu, cpumask_of_node(node), cpu_online_mask)
|
|
if (i-- == 0)
|
|
return cpu;
|
|
|
|
for_each_cpu(cpu, cpu_online_mask) {
|
|
/* Skip NUMA nodes, done above. */
|
|
if (cpumask_test_cpu(cpu, cpumask_of_node(node)))
|
|
continue;
|
|
|
|
if (i-- == 0)
|
|
return cpu;
|
|
}
|
|
}
|
|
BUG();
|
|
}
|
|
EXPORT_SYMBOL(cpumask_local_spread);
|