AuxXxilium/linux_dsm_epyc7002
1
0
Fork 0
mirror of https://github.com/AuxXxilium/linux_dsm_epyc7002.git synced 2024-11-24 05:00:55 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity

fs/binfmt_elf.c: better codegen around current->mm

Browse Source 
"current->mm" pointer is stable in general except few cases one of which
execve(2).  Compiler can't treat is as stable but it _is_ stable most of
the time.  During ELF loading process ->mm becomes stable right after
flush_old_exec().

Help compiler by caching current->mm, otherwise it continues to refetch
it.

	add/remove: 0/0 grow/shrink: 0/2 up/down: 0/-141 (-141)
	Function                                     old     new   delta
	elf_core_dump                               5062    5039     -23
	load_elf_binary                             5426    5308    -118

Note: other cases are left as is because it is either pessimisation or
no change in binary size.

Link: http://lkml.kernel.org/r/20191215124755.GB21124@avx2
Signed-off-by: Alexey Dobriyan <adobriyan@gmail.com>
Reviewed-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Alexey Dobriyan 2020-01-30 22:16:58 -08:00 committed by Linus Torvalds
parent a62c5b1b66
commit 03c6d723ee
1 changed files with 28 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
fs/binfmt_elf.c
View File

@ -165,6 +165,7 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec,
unsigned long load_addr, unsigned long interp_load_addr, unsigned long load_addr, unsigned long interp_load_addr,
unsigned long e_entry) unsigned long e_entry)
{ {
struct mm_struct *mm = current->mm;
unsigned long p = bprm->p; unsigned long p = bprm->p;
int argc = bprm->argc; int argc = bprm->argc;
int envc = bprm->envc; int envc = bprm->envc;
@ -227,7 +228,7 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec,
return -EFAULT; return -EFAULT;
/* Create the ELF interpreter info */ /* Create the ELF interpreter info */
elf_info = (elf_addr_t *)current->mm->saved_auxv; elf_info = (elf_addr_t *)mm->saved_auxv;
/* update AT_VECTOR_SIZE_BASE if the number of NEW_AUX_ENT() changes */ /* update AT_VECTOR_SIZE_BASE if the number of NEW_AUX_ENT() changes */
#define NEW_AUX_ENT(id, val) \ #define NEW_AUX_ENT(id, val) \
do { \ do { \
@ -276,13 +277,13 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec,
} }
#undef NEW_AUX_ENT #undef NEW_AUX_ENT
/* AT_NULL is zero; clear the rest too */ /* AT_NULL is zero; clear the rest too */
memset(elf_info, 0, (char *)current->mm->saved_auxv + memset(elf_info, 0, (char *)mm->saved_auxv +
sizeof(current->mm->saved_auxv) - (char *)elf_info); sizeof(mm->saved_auxv) - (char *)elf_info);
/* And advance past the AT_NULL entry. */ /* And advance past the AT_NULL entry. */
elf_info += 2; elf_info += 2;
ei_index = elf_info - (elf_addr_t *)current->mm->saved_auxv; ei_index = elf_info - (elf_addr_t *)mm->saved_auxv;
sp = STACK_ADD(p, ei_index); sp = STACK_ADD(p, ei_index);
items = (argc + 1) + (envc + 1) + 1; items = (argc + 1) + (envc + 1) + 1;
@ -301,7 +302,7 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec,
* Grow the stack manually; some architectures have a limit on how * Grow the stack manually; some architectures have a limit on how
* far ahead a user-space access may be in order to grow the stack. * far ahead a user-space access may be in order to grow the stack.
*/ */
vma = find_extend_vma(current->mm, bprm->p); vma = find_extend_vma(mm, bprm->p);
if (!vma) if (!vma)
return -EFAULT; return -EFAULT;
@ -310,7 +311,7 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec,
return -EFAULT; return -EFAULT;
/* Populate list of argv pointers back to argv strings. */ /* Populate list of argv pointers back to argv strings. */
p = current->mm->arg_end = current->mm->arg_start; p = mm->arg_end = mm->arg_start;
while (argc-- > 0) { while (argc-- > 0) {
size_t len; size_t len;
if (__put_user((elf_addr_t)p, sp++)) if (__put_user((elf_addr_t)p, sp++))
@ -322,10 +323,10 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec,
} }
if (__put_user(0, sp++)) if (__put_user(0, sp++))
return -EFAULT; return -EFAULT;
current->mm->arg_end = p; mm->arg_end = p;
/* Populate list of envp pointers back to envp strings. */ /* Populate list of envp pointers back to envp strings. */
current->mm->env_end = current->mm->env_start = p; mm->env_end = mm->env_start = p;
while (envc-- > 0) { while (envc-- > 0) {
size_t len; size_t len;
if (__put_user((elf_addr_t)p, sp++)) if (__put_user((elf_addr_t)p, sp++))
@ -337,10 +338,10 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec,
} }
if (__put_user(0, sp++)) if (__put_user(0, sp++))
return -EFAULT; return -EFAULT;
current->mm->env_end = p; mm->env_end = p;
/* Put the elf_info on the stack in the right place. */ /* Put the elf_info on the stack in the right place. */
if (copy_to_user(sp, current->mm->saved_auxv, ei_index * sizeof(elf_addr_t))) if (copy_to_user(sp, mm->saved_auxv, ei_index * sizeof(elf_addr_t)))
return -EFAULT; return -EFAULT;
return 0; return 0;
} }
@ -701,6 +702,7 @@ static int load_elf_binary(struct linux_binprm *bprm)
struct elfhdr interp_elf_ex; struct elfhdr interp_elf_ex;
} *loc; } *loc;
struct arch_elf_state arch_state = INIT_ARCH_ELF_STATE; struct arch_elf_state arch_state = INIT_ARCH_ELF_STATE;
struct mm_struct *mm;
struct pt_regs *regs; struct pt_regs *regs;
loc = kmalloc(sizeof(*loc), GFP_KERNEL); loc = kmalloc(sizeof(*loc), GFP_KERNEL);
@ -1096,11 +1098,13 @@ static int load_elf_binary(struct linux_binprm *bprm)
load_addr, interp_load_addr, e_entry); load_addr, interp_load_addr, e_entry);
if (retval < 0) if (retval < 0)
goto out; goto out;
current->mm->end_code = end_code;
current->mm->start_code = start_code; mm = current->mm;
current->mm->start_data = start_data; mm->end_code = end_code;
current->mm->end_data = end_data; mm->start_code = start_code;
current->mm->start_stack = bprm->p; mm->start_data = start_data;
mm->end_data = end_data;
mm->start_stack = bprm->p;
if ((current->flags & PF_RANDOMIZE) && (randomize_va_space > 1)) { if ((current->flags & PF_RANDOMIZE) && (randomize_va_space > 1)) {
/* /*
@ -1111,12 +1115,11 @@ static int load_elf_binary(struct linux_binprm *bprm)
* growing down), and into the unused ELF_ET_DYN_BASE region. * growing down), and into the unused ELF_ET_DYN_BASE region.
*/ */
if (IS_ENABLED(CONFIG_ARCH_HAS_ELF_RANDOMIZE) && if (IS_ENABLED(CONFIG_ARCH_HAS_ELF_RANDOMIZE) &&
elf_ex->e_type == ET_DYN && !interpreter) elf_ex->e_type == ET_DYN && !interpreter) {
current->mm->brk = current->mm->start_brk = mm->brk = mm->start_brk = ELF_ET_DYN_BASE;
ELF_ET_DYN_BASE; }
current->mm->brk = current->mm->start_brk = mm->brk = mm->start_brk = arch_randomize_brk(mm);
arch_randomize_brk(current->mm);
#ifdef compat_brk_randomized #ifdef compat_brk_randomized
current->brk_randomized = 1; current->brk_randomized = 1;
#endif #endif
@ -1574,6 +1577,7 @@ static void fill_siginfo_note(struct memelfnote *note, user_siginfo_t *csigdata,
*/ */
static int fill_files_note(struct memelfnote *note) static int fill_files_note(struct memelfnote *note)
{ {
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma; struct vm_area_struct *vma;
unsigned count, size, names_ofs, remaining, n; unsigned count, size, names_ofs, remaining, n;
user_long_t *data; user_long_t *data;
@ -1581,7 +1585,7 @@ static int fill_files_note(struct memelfnote *note)
char *name_base, *name_curpos; char *name_base, *name_curpos;
/* *Estimated* file count and total data size needed */ /* *Estimated* file count and total data size needed */
count = current->mm->map_count; count = mm->map_count;
if (count > UINT_MAX / 64) if (count > UINT_MAX / 64)
return -EINVAL; return -EINVAL;
size = count * 64; size = count * 64;
@ -1599,7 +1603,7 @@ static int fill_files_note(struct memelfnote *note)
name_base = name_curpos = ((char *)data) + names_ofs; name_base = name_curpos = ((char *)data) + names_ofs;
remaining = size - names_ofs; remaining = size - names_ofs;
count = 0; count = 0;
for (vma = current->mm->mmap; vma != NULL; vma = vma->vm_next) { for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) {
struct file *file; struct file *file;
const char *filename; const char *filename;
@ -1633,10 +1637,10 @@ static int fill_files_note(struct memelfnote *note)
data[0] = count; data[0] = count;
data[1] = PAGE_SIZE; data[1] = PAGE_SIZE;
/* /*
* Count usually is less than current->mm->map_count, * Count usually is less than mm->map_count,
* we need to move filenames down. * we need to move filenames down.
*/ */
n = current->mm->map_count - count; n = mm->map_count - count;
if (n != 0) { if (n != 0) {
unsigned shift_bytes = n * 3 * sizeof(data[0]); unsigned shift_bytes = n * 3 * sizeof(data[0]);
memmove(name_base - shift_bytes, name_base, memmove(name_base - shift_bytes, name_base,