mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-28 02:05:20 +07:00
ba54f0c3f7
Currently, the jump label of a static key is transformed via the arch
specific function:
void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type)
The new approach (batch mode) uses two arch functions, the first has the
same arguments of the arch_jump_label_transform(), and is the function:
bool arch_jump_label_transform_queue(struct jump_entry *entry,
enum jump_label_type type)
Rather than transforming the code, it adds the jump_entry in a queue of
entries to be updated. This functions returns true in the case of a
successful enqueue of an entry. If it returns false, the caller must to
apply the queue and then try to queue again, for instance, because the
queue is full.
This function expects the caller to sort the entries by the address before
enqueueuing then. This is already done by the arch independent code, though.
After queuing all jump_entries, the function:
void arch_jump_label_transform_apply(void)
Applies the changes in the queue.
Signed-off-by: Daniel Bristot de Oliveira <bristot@redhat.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Chris von Recklinghausen <crecklin@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Jason Baron <jbaron@akamai.com>
Cc: Jiri Kosina <jkosina@suse.cz>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Scott Wood <swood@redhat.com>
Cc: Steven Rostedt (VMware) <rostedt@goodmis.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lkml.kernel.org/r/57b4caa654bad7e3b066301c9a9ae233dea065b5.1560325897.git.bristot@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
95 lines
1.9 KiB
C
95 lines
1.9 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _ASM_X86_JUMP_LABEL_H
|
|
#define _ASM_X86_JUMP_LABEL_H
|
|
|
|
#define HAVE_JUMP_LABEL_BATCH
|
|
|
|
#define JUMP_LABEL_NOP_SIZE 5
|
|
|
|
#ifdef CONFIG_X86_64
|
|
# define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC
|
|
#else
|
|
# define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC
|
|
#endif
|
|
|
|
#include <asm/asm.h>
|
|
#include <asm/nops.h>
|
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
#include <linux/stringify.h>
|
|
#include <linux/types.h>
|
|
|
|
static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
|
|
{
|
|
asm_volatile_goto("1:"
|
|
".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
|
|
".pushsection __jump_table, \"aw\" \n\t"
|
|
_ASM_ALIGN "\n\t"
|
|
".long 1b - ., %l[l_yes] - . \n\t"
|
|
_ASM_PTR "%c0 + %c1 - .\n\t"
|
|
".popsection \n\t"
|
|
: : "i" (key), "i" (branch) : : l_yes);
|
|
|
|
return false;
|
|
l_yes:
|
|
return true;
|
|
}
|
|
|
|
static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
|
|
{
|
|
asm_volatile_goto("1:"
|
|
".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t"
|
|
"2:\n\t"
|
|
".pushsection __jump_table, \"aw\" \n\t"
|
|
_ASM_ALIGN "\n\t"
|
|
".long 1b - ., %l[l_yes] - . \n\t"
|
|
_ASM_PTR "%c0 + %c1 - .\n\t"
|
|
".popsection \n\t"
|
|
: : "i" (key), "i" (branch) : : l_yes);
|
|
|
|
return false;
|
|
l_yes:
|
|
return true;
|
|
}
|
|
|
|
#else /* __ASSEMBLY__ */
|
|
|
|
.macro STATIC_JUMP_IF_TRUE target, key, def
|
|
.Lstatic_jump_\@:
|
|
.if \def
|
|
/* Equivalent to "jmp.d32 \target" */
|
|
.byte 0xe9
|
|
.long \target - .Lstatic_jump_after_\@
|
|
.Lstatic_jump_after_\@:
|
|
.else
|
|
.byte STATIC_KEY_INIT_NOP
|
|
.endif
|
|
.pushsection __jump_table, "aw"
|
|
_ASM_ALIGN
|
|
.long .Lstatic_jump_\@ - ., \target - .
|
|
_ASM_PTR \key - .
|
|
.popsection
|
|
.endm
|
|
|
|
.macro STATIC_JUMP_IF_FALSE target, key, def
|
|
.Lstatic_jump_\@:
|
|
.if \def
|
|
.byte STATIC_KEY_INIT_NOP
|
|
.else
|
|
/* Equivalent to "jmp.d32 \target" */
|
|
.byte 0xe9
|
|
.long \target - .Lstatic_jump_after_\@
|
|
.Lstatic_jump_after_\@:
|
|
.endif
|
|
.pushsection __jump_table, "aw"
|
|
_ASM_ALIGN
|
|
.long .Lstatic_jump_\@ - ., \target - .
|
|
_ASM_PTR \key + 1 - .
|
|
.popsection
|
|
.endm
|
|
|
|
#endif /* __ASSEMBLY__ */
|
|
|
|
#endif
|