2005-04-17 05:20:36 +07:00
|
|
|
/*
|
|
|
|
* Compatibility mode system call entry point for x86-64.
|
|
|
|
*
|
|
|
|
* Copyright 2000-2002 Andi Kleen, SuSE Labs.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <asm/dwarf2.h>
|
|
|
|
#include <asm/calling.h>
|
2005-09-10 02:28:48 +07:00
|
|
|
#include <asm/asm-offsets.h>
|
2005-04-17 05:20:36 +07:00
|
|
|
#include <asm/current.h>
|
|
|
|
#include <asm/errno.h>
|
|
|
|
#include <asm/ia32_unistd.h>
|
|
|
|
#include <asm/thread_info.h>
|
|
|
|
#include <asm/segment.h>
|
2006-07-03 14:24:45 +07:00
|
|
|
#include <asm/irqflags.h>
|
2012-04-21 02:19:50 +07:00
|
|
|
#include <asm/asm.h>
|
2012-09-22 02:43:12 +07:00
|
|
|
#include <asm/smap.h>
|
2005-04-17 05:20:36 +07:00
|
|
|
#include <linux/linkage.h>
|
2012-01-04 02:23:06 +07:00
|
|
|
#include <linux/err.h>
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2008-06-24 15:13:31 +07:00
|
|
|
/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
|
|
|
|
#include <linux/elf-em.h>
|
|
|
|
#define AUDIT_ARCH_I386 (EM_386|__AUDIT_ARCH_LE)
|
|
|
|
#define __AUDIT_ARCH_LE 0x40000000
|
|
|
|
|
|
|
|
#ifndef CONFIG_AUDITSYSCALL
|
2009-09-30 17:22:11 +07:00
|
|
|
#define sysexit_audit ia32_ret_from_sys_call
|
|
|
|
#define sysretl_audit ia32_ret_from_sys_call
|
2008-06-24 15:13:31 +07:00
|
|
|
#endif
|
|
|
|
|
2011-03-08 01:10:39 +07:00
|
|
|
.section .entry.text, "ax"
|
|
|
|
|
2015-02-27 05:40:30 +07:00
|
|
|
/* clobbers %rax */
|
|
|
|
.macro CLEAR_RREGS _r9=rax
|
2005-04-17 05:20:36 +07:00
|
|
|
xorl %eax,%eax
|
2015-02-27 05:40:30 +07:00
|
|
|
movq %rax,R11(%rsp)
|
|
|
|
movq %rax,R10(%rsp)
|
|
|
|
movq %\_r9,R9(%rsp)
|
|
|
|
movq %rax,R8(%rsp)
|
2005-04-17 05:20:36 +07:00
|
|
|
.endm
|
|
|
|
|
2008-07-09 16:38:07 +07:00
|
|
|
/*
|
|
|
|
* Reload arg registers from stack in case ptrace changed them.
|
|
|
|
* We don't reload %eax because syscall_trace_enter() returned
|
2010-09-15 02:22:58 +07:00
|
|
|
* the %rax value we should see. Instead, we just truncate that
|
|
|
|
* value to 32 bits again as we did on entry from user mode.
|
|
|
|
* If it's a new value set by user_regset during entry tracing,
|
|
|
|
* this matches the normal truncation of the user-mode value.
|
|
|
|
* If it's -1 to make us punt the syscall, then (u32)-1 is still
|
|
|
|
* an appropriately invalid value.
|
2008-07-09 16:38:07 +07:00
|
|
|
*/
|
2015-02-27 05:40:30 +07:00
|
|
|
.macro LOAD_ARGS32 _r9=0
|
2008-08-29 19:21:11 +07:00
|
|
|
.if \_r9
|
2015-02-27 05:40:30 +07:00
|
|
|
movl R9(%rsp),%r9d
|
2008-08-29 19:21:11 +07:00
|
|
|
.endif
|
2015-02-27 05:40:30 +07:00
|
|
|
movl RCX(%rsp),%ecx
|
|
|
|
movl RDX(%rsp),%edx
|
|
|
|
movl RSI(%rsp),%esi
|
|
|
|
movl RDI(%rsp),%edi
|
2010-09-15 02:22:58 +07:00
|
|
|
movl %eax,%eax /* zero extension */
|
2007-09-21 21:16:18 +07:00
|
|
|
.endm
|
|
|
|
|
2006-01-12 04:41:59 +07:00
|
|
|
.macro CFI_STARTPROC32 simple
|
|
|
|
CFI_STARTPROC \simple
|
|
|
|
CFI_UNDEFINED r8
|
|
|
|
CFI_UNDEFINED r9
|
|
|
|
CFI_UNDEFINED r10
|
|
|
|
CFI_UNDEFINED r11
|
|
|
|
CFI_UNDEFINED r12
|
|
|
|
CFI_UNDEFINED r13
|
|
|
|
CFI_UNDEFINED r14
|
|
|
|
CFI_UNDEFINED r15
|
|
|
|
.endm
|
|
|
|
|
2008-06-25 11:19:28 +07:00
|
|
|
#ifdef CONFIG_PARAVIRT
|
|
|
|
ENTRY(native_usergs_sysret32)
|
|
|
|
swapgs
|
|
|
|
sysretl
|
|
|
|
ENDPROC(native_usergs_sysret32)
|
|
|
|
|
|
|
|
ENTRY(native_irq_enable_sysexit)
|
|
|
|
swapgs
|
|
|
|
sti
|
|
|
|
sysexit
|
|
|
|
ENDPROC(native_irq_enable_sysexit)
|
|
|
|
#endif
|
|
|
|
|
2005-04-17 05:20:36 +07:00
|
|
|
/*
|
|
|
|
* 32bit SYSENTER instruction entry.
|
|
|
|
*
|
2015-02-27 05:40:32 +07:00
|
|
|
* SYSENTER loads ss, rsp, cs, and rip from previously programmed MSRs.
|
|
|
|
* IF and VM in rflags are cleared (IOW: interrupts are off).
|
|
|
|
* SYSENTER does not save anything on the stack,
|
|
|
|
* and does not save old rip (!!!) and rflags.
|
|
|
|
*
|
2005-04-17 05:20:36 +07:00
|
|
|
* Arguments:
|
2015-02-27 05:40:32 +07:00
|
|
|
* eax system call number
|
|
|
|
* ebx arg1
|
|
|
|
* ecx arg2
|
|
|
|
* edx arg3
|
|
|
|
* esi arg4
|
|
|
|
* edi arg5
|
|
|
|
* ebp user stack
|
|
|
|
* 0(%ebp) arg6
|
|
|
|
*
|
2005-04-17 05:20:36 +07:00
|
|
|
* This is purely a fast path. For anything complicated we use the int 0x80
|
2015-02-27 05:40:32 +07:00
|
|
|
* path below. We set up a complete hardware stack frame to share code
|
2005-04-17 05:20:36 +07:00
|
|
|
* with the int 0x80 path.
|
2015-02-27 05:40:32 +07:00
|
|
|
*/
|
2005-04-17 05:20:36 +07:00
|
|
|
ENTRY(ia32_sysenter_target)
|
2006-01-12 04:41:59 +07:00
|
|
|
CFI_STARTPROC32 simple
|
2006-09-26 15:52:41 +07:00
|
|
|
CFI_SIGNAL_FRAME
|
2005-09-12 23:49:24 +07:00
|
|
|
CFI_DEF_CFA rsp,0
|
|
|
|
CFI_REGISTER rsp,rbp
|
2015-03-27 17:36:20 +07:00
|
|
|
|
2006-07-03 14:24:45 +07:00
|
|
|
/*
|
2015-03-27 17:36:20 +07:00
|
|
|
* Interrupts are off on entry.
|
|
|
|
* We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
|
|
|
|
* it is too small to ever cause noticeable irq latency.
|
2006-07-03 14:24:45 +07:00
|
|
|
*/
|
2015-03-27 17:36:20 +07:00
|
|
|
SWAPGS_UNSAFE_STACK
|
|
|
|
movq PER_CPU_VAR(cpu_tss + TSS_sp0), %rsp
|
2008-06-25 11:19:29 +07:00
|
|
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
2015-03-27 17:36:20 +07:00
|
|
|
|
2015-03-27 17:36:21 +07:00
|
|
|
/* Zero-extending 32-bit regs, do not remove */
|
|
|
|
movl %ebp, %ebp
|
|
|
|
movl %eax, %eax
|
|
|
|
|
2015-04-01 00:00:04 +07:00
|
|
|
movl ASM_THREAD_INFO(TI_sysenter_return, %rsp, 0), %r10d
|
2005-09-12 23:49:24 +07:00
|
|
|
CFI_REGISTER rip,r10
|
2015-04-01 00:00:04 +07:00
|
|
|
|
|
|
|
/* Construct struct pt_regs on stack */
|
|
|
|
pushq_cfi $__USER32_DS /* pt_regs->ss */
|
|
|
|
pushq_cfi %rbp /* pt_regs->sp */
|
|
|
|
CFI_REL_OFFSET rsp,0
|
|
|
|
pushfq_cfi /* pt_regs->flags */
|
|
|
|
pushq_cfi $__USER32_CS /* pt_regs->cs */
|
|
|
|
pushq_cfi %r10 /* pt_regs->ip = thread_info->sysenter_return */
|
|
|
|
CFI_REL_OFFSET rip,0
|
|
|
|
pushq_cfi_reg rax /* pt_regs->orig_ax */
|
|
|
|
pushq_cfi_reg rdi /* pt_regs->di */
|
|
|
|
pushq_cfi_reg rsi /* pt_regs->si */
|
|
|
|
pushq_cfi_reg rdx /* pt_regs->dx */
|
|
|
|
pushq_cfi_reg rcx /* pt_regs->cx */
|
|
|
|
pushq_cfi_reg rax /* pt_regs->ax */
|
2005-04-17 05:20:36 +07:00
|
|
|
cld
|
2015-04-01 00:00:04 +07:00
|
|
|
sub $(10*8),%rsp /* pt_regs->r8-11,bp,bx,r12-15 not saved */
|
|
|
|
CFI_ADJUST_CFA_OFFSET 10*8
|
|
|
|
|
2015-02-27 05:40:32 +07:00
|
|
|
/*
|
|
|
|
* no need to do an access_ok check here because rbp has been
|
|
|
|
* 32bit zero extended
|
|
|
|
*/
|
2012-09-22 02:43:12 +07:00
|
|
|
ASM_STAC
|
2008-08-29 19:21:11 +07:00
|
|
|
1: movl (%rbp),%ebp
|
2012-04-21 02:19:50 +07:00
|
|
|
_ASM_EXTABLE(1b,ia32_badarg)
|
2012-09-22 02:43:12 +07:00
|
|
|
ASM_CLAC
|
x86_64, entry: Filter RFLAGS.NT on entry from userspace
The NT flag doesn't do anything in long mode other than causing IRET
to #GP. Oddly, CPL3 code can still set NT using popf.
Entry via hardware or software interrupt clears NT automatically, so
the only relevant entries are fast syscalls.
If user code causes kernel code to run with NT set, then there's at
least some (small) chance that it could cause trouble. For example,
user code could cause a call to EFI code with NT set, and who knows
what would happen? Apparently some games on Wine sometimes do
this (!), and, if an IRET return happens, they will segfault. That
segfault cannot be handled, because signal delivery fails, too.
This patch programs the CPU to clear NT on entry via SYSCALL (both
32-bit and 64-bit, by my reading of the AMD APM), and it clears NT
in software on entry via SYSENTER.
To save a few cycles, this borrows a trick from Jan Beulich in Xen:
it checks whether NT is set before trying to clear it. As a result,
it seems to have very little effect on SYSENTER performance on my
machine.
There's another minor bug fix in here: it looks like the CFI
annotations were wrong if CONFIG_AUDITSYSCALL=n.
Testers beware: on Xen, SYSENTER with NT set turns into a GPF.
I haven't touched anything on 32-bit kernels.
The syscall mask change comes from a variant of this patch by Anish
Bhatt.
Note to stable maintainers: there is no known security issue here.
A misguided program can set NT and cause the kernel to try and fail
to deliver SIGSEGV, crashing the program. This patch fixes Far Cry
on Wine: https://bugs.winehq.org/show_bug.cgi?id=33275
Cc: <stable@vger.kernel.org>
Reported-by: Anish Bhatt <anish@chelsio.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Link: http://lkml.kernel.org/r/395749a5d39a29bd3e4b35899cf3a3c1340e5595.1412189265.git.luto@amacapital.net
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
2014-10-02 01:49:04 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Sysenter doesn't filter flags, so we need to clear NT
|
|
|
|
* ourselves. To save a few cycles, we can check whether
|
|
|
|
* NT was set instead of doing an unconditional popfq.
|
|
|
|
*/
|
2015-02-27 05:40:30 +07:00
|
|
|
testl $X86_EFLAGS_NT,EFLAGS(%rsp)
|
x86_64, entry: Filter RFLAGS.NT on entry from userspace
The NT flag doesn't do anything in long mode other than causing IRET
to #GP. Oddly, CPL3 code can still set NT using popf.
Entry via hardware or software interrupt clears NT automatically, so
the only relevant entries are fast syscalls.
If user code causes kernel code to run with NT set, then there's at
least some (small) chance that it could cause trouble. For example,
user code could cause a call to EFI code with NT set, and who knows
what would happen? Apparently some games on Wine sometimes do
this (!), and, if an IRET return happens, they will segfault. That
segfault cannot be handled, because signal delivery fails, too.
This patch programs the CPU to clear NT on entry via SYSCALL (both
32-bit and 64-bit, by my reading of the AMD APM), and it clears NT
in software on entry via SYSENTER.
To save a few cycles, this borrows a trick from Jan Beulich in Xen:
it checks whether NT is set before trying to clear it. As a result,
it seems to have very little effect on SYSENTER performance on my
machine.
There's another minor bug fix in here: it looks like the CFI
annotations were wrong if CONFIG_AUDITSYSCALL=n.
Testers beware: on Xen, SYSENTER with NT set turns into a GPF.
I haven't touched anything on 32-bit kernels.
The syscall mask change comes from a variant of this patch by Anish
Bhatt.
Note to stable maintainers: there is no known security issue here.
A misguided program can set NT and cause the kernel to try and fail
to deliver SIGSEGV, crashing the program. This patch fixes Far Cry
on Wine: https://bugs.winehq.org/show_bug.cgi?id=33275
Cc: <stable@vger.kernel.org>
Reported-by: Anish Bhatt <anish@chelsio.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Link: http://lkml.kernel.org/r/395749a5d39a29bd3e4b35899cf3a3c1340e5595.1412189265.git.luto@amacapital.net
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
2014-10-02 01:49:04 +07:00
|
|
|
jnz sysenter_fix_flags
|
|
|
|
sysenter_flags_fixed:
|
|
|
|
|
2015-03-25 01:44:42 +07:00
|
|
|
orl $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
|
|
|
|
testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
|
2005-09-12 23:49:24 +07:00
|
|
|
CFI_REMEMBER_STATE
|
2005-04-17 05:20:36 +07:00
|
|
|
jnz sysenter_tracesys
|
x86-64, compat: Test %rax for the syscall number, not %eax
On 64 bits, we always, by necessity, jump through the system call
table via %rax. For 32-bit system calls, in theory the system call
number is stored in %eax, and the code was testing %eax for a valid
system call number. At one point we loaded the stored value back from
the stack to enforce zero-extension, but that was removed in checkin
d4d67150165df8bf1cc05e532f6efca96f907cab. An actual 32-bit process
will not be able to introduce a non-zero-extended number, but it can
happen via ptrace.
Instead of re-introducing the zero-extension, test what we are
actually going to use, i.e. %rax. This only adds a handful of REX
prefixes to the code.
Reported-by: Ben Hawkes <hawkes@sota.gen.nz>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
Cc: <stable@kernel.org>
Cc: Roland McGrath <roland@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
2010-09-15 02:42:41 +07:00
|
|
|
cmpq $(IA32_NR_syscalls-1),%rax
|
2006-04-08 00:50:31 +07:00
|
|
|
ja ia32_badsys
|
2008-07-09 16:38:07 +07:00
|
|
|
sysenter_do_call:
|
2015-02-27 05:40:35 +07:00
|
|
|
/* 32bit syscall -> 64bit C ABI argument conversion */
|
|
|
|
movl %edi,%r8d /* arg5 */
|
|
|
|
movl %ebp,%r9d /* arg6 */
|
|
|
|
xchg %ecx,%esi /* rsi:arg2, rcx:arg4 */
|
|
|
|
movl %ebx,%edi /* arg1 */
|
|
|
|
movl %edx,%edx /* arg3 (zero extension) */
|
2008-06-24 15:13:31 +07:00
|
|
|
sysenter_dispatch:
|
2005-04-17 05:20:36 +07:00
|
|
|
call *ia32_sys_call_table(,%rax,8)
|
2015-02-27 05:40:30 +07:00
|
|
|
movq %rax,RAX(%rsp)
|
2008-06-25 11:19:29 +07:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 14:24:45 +07:00
|
|
|
TRACE_IRQS_OFF
|
2015-03-25 01:44:42 +07:00
|
|
|
testl $_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
|
2008-06-24 15:13:31 +07:00
|
|
|
jnz sysexit_audit
|
|
|
|
sysexit_from_sys_call:
|
2015-03-25 01:44:42 +07:00
|
|
|
andl $~TS_COMPAT,ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
|
2005-04-17 05:20:36 +07:00
|
|
|
/* clear IF, that popfq doesn't enable interrupts early */
|
2015-02-27 05:40:30 +07:00
|
|
|
andl $~0x200,EFLAGS(%rsp)
|
|
|
|
movl RIP(%rsp),%edx /* User %eip */
|
2008-01-30 19:30:43 +07:00
|
|
|
CFI_REGISTER rip,rdx
|
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 05:40:27 +07:00
|
|
|
RESTORE_RSI_RDI
|
2015-02-27 05:40:32 +07:00
|
|
|
/* pop everything except ss,rsp,rflags slots */
|
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 05:40:27 +07:00
|
|
|
REMOVE_PT_GPREGS_FROM_STACK 3*8
|
2009-09-30 17:22:11 +07:00
|
|
|
xorq %r8,%r8
|
|
|
|
xorq %r9,%r9
|
|
|
|
xorq %r10,%r10
|
|
|
|
xorq %r11,%r11
|
2011-02-28 22:54:40 +07:00
|
|
|
popfq_cfi
|
2005-09-12 23:49:24 +07:00
|
|
|
/*CFI_RESTORE rflags*/
|
2011-02-28 22:54:40 +07:00
|
|
|
popq_cfi %rcx /* User %esp */
|
2005-09-12 23:49:24 +07:00
|
|
|
CFI_REGISTER rsp,rcx
|
2006-07-03 14:24:45 +07:00
|
|
|
TRACE_IRQS_ON
|
2015-02-27 05:40:32 +07:00
|
|
|
/*
|
|
|
|
* 32bit SYSEXIT restores eip from edx, esp from ecx.
|
|
|
|
* cs and ss are loaded from MSRs.
|
|
|
|
*/
|
2008-06-25 11:19:28 +07:00
|
|
|
ENABLE_INTERRUPTS_SYSEXIT32
|
2005-04-17 05:20:36 +07:00
|
|
|
|
x86_64, entry: Filter RFLAGS.NT on entry from userspace
The NT flag doesn't do anything in long mode other than causing IRET
to #GP. Oddly, CPL3 code can still set NT using popf.
Entry via hardware or software interrupt clears NT automatically, so
the only relevant entries are fast syscalls.
If user code causes kernel code to run with NT set, then there's at
least some (small) chance that it could cause trouble. For example,
user code could cause a call to EFI code with NT set, and who knows
what would happen? Apparently some games on Wine sometimes do
this (!), and, if an IRET return happens, they will segfault. That
segfault cannot be handled, because signal delivery fails, too.
This patch programs the CPU to clear NT on entry via SYSCALL (both
32-bit and 64-bit, by my reading of the AMD APM), and it clears NT
in software on entry via SYSENTER.
To save a few cycles, this borrows a trick from Jan Beulich in Xen:
it checks whether NT is set before trying to clear it. As a result,
it seems to have very little effect on SYSENTER performance on my
machine.
There's another minor bug fix in here: it looks like the CFI
annotations were wrong if CONFIG_AUDITSYSCALL=n.
Testers beware: on Xen, SYSENTER with NT set turns into a GPF.
I haven't touched anything on 32-bit kernels.
The syscall mask change comes from a variant of this patch by Anish
Bhatt.
Note to stable maintainers: there is no known security issue here.
A misguided program can set NT and cause the kernel to try and fail
to deliver SIGSEGV, crashing the program. This patch fixes Far Cry
on Wine: https://bugs.winehq.org/show_bug.cgi?id=33275
Cc: <stable@vger.kernel.org>
Reported-by: Anish Bhatt <anish@chelsio.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Link: http://lkml.kernel.org/r/395749a5d39a29bd3e4b35899cf3a3c1340e5595.1412189265.git.luto@amacapital.net
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
2014-10-02 01:49:04 +07:00
|
|
|
CFI_RESTORE_STATE
|
|
|
|
|
2008-06-24 15:13:31 +07:00
|
|
|
#ifdef CONFIG_AUDITSYSCALL
|
|
|
|
.macro auditsys_entry_common
|
2014-03-04 22:38:06 +07:00
|
|
|
movl %esi,%r8d /* 5th arg: 4th syscall arg */
|
|
|
|
movl %ecx,%r9d /*swap with edx*/
|
|
|
|
movl %edx,%ecx /* 4th arg: 3rd syscall arg */
|
|
|
|
movl %r9d,%edx /* 3rd arg: 2nd syscall arg */
|
|
|
|
movl %ebx,%esi /* 2nd arg: 1st syscall arg */
|
|
|
|
movl %eax,%edi /* 1st arg: syscall number */
|
2012-01-04 02:23:06 +07:00
|
|
|
call __audit_syscall_entry
|
2015-02-27 05:40:30 +07:00
|
|
|
movl RAX(%rsp),%eax /* reload syscall number */
|
x86-64, compat: Test %rax for the syscall number, not %eax
On 64 bits, we always, by necessity, jump through the system call
table via %rax. For 32-bit system calls, in theory the system call
number is stored in %eax, and the code was testing %eax for a valid
system call number. At one point we loaded the stored value back from
the stack to enforce zero-extension, but that was removed in checkin
d4d67150165df8bf1cc05e532f6efca96f907cab. An actual 32-bit process
will not be able to introduce a non-zero-extended number, but it can
happen via ptrace.
Instead of re-introducing the zero-extension, test what we are
actually going to use, i.e. %rax. This only adds a handful of REX
prefixes to the code.
Reported-by: Ben Hawkes <hawkes@sota.gen.nz>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
Cc: <stable@kernel.org>
Cc: Roland McGrath <roland@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
2010-09-15 02:42:41 +07:00
|
|
|
cmpq $(IA32_NR_syscalls-1),%rax
|
2008-06-24 15:13:31 +07:00
|
|
|
ja ia32_badsys
|
|
|
|
movl %ebx,%edi /* reload 1st syscall arg */
|
2015-02-27 05:40:30 +07:00
|
|
|
movl RCX(%rsp),%esi /* reload 2nd syscall arg */
|
|
|
|
movl RDX(%rsp),%edx /* reload 3rd syscall arg */
|
|
|
|
movl RSI(%rsp),%ecx /* reload 4th syscall arg */
|
|
|
|
movl RDI(%rsp),%r8d /* reload 5th syscall arg */
|
2008-06-24 15:13:31 +07:00
|
|
|
.endm
|
|
|
|
|
2009-10-26 22:20:29 +07:00
|
|
|
.macro auditsys_exit exit
|
2015-03-25 01:44:42 +07:00
|
|
|
testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
|
2009-09-30 17:22:11 +07:00
|
|
|
jnz ia32_ret_from_sys_call
|
2008-06-24 15:13:31 +07:00
|
|
|
TRACE_IRQS_ON
|
2013-01-30 14:55:53 +07:00
|
|
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
2008-06-24 15:13:31 +07:00
|
|
|
movl %eax,%esi /* second arg, syscall return value */
|
2012-01-04 02:23:06 +07:00
|
|
|
cmpl $-MAX_ERRNO,%eax /* is it an error ? */
|
2012-01-04 02:23:06 +07:00
|
|
|
jbe 1f
|
|
|
|
movslq %eax, %rsi /* if error sign extend to 64 bits */
|
|
|
|
1: setbe %al /* 1 if error, 0 if not */
|
2008-06-24 15:13:31 +07:00
|
|
|
movzbl %al,%edi /* zero-extend that into %edi */
|
2012-01-04 02:23:06 +07:00
|
|
|
call __audit_syscall_exit
|
2015-02-27 05:40:30 +07:00
|
|
|
movq RAX(%rsp),%rax /* reload syscall return value */
|
2008-06-24 15:13:31 +07:00
|
|
|
movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
|
2013-01-30 14:55:53 +07:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2008-06-24 15:13:31 +07:00
|
|
|
TRACE_IRQS_OFF
|
2015-03-25 01:44:42 +07:00
|
|
|
testl %edi, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
|
2009-09-30 17:22:11 +07:00
|
|
|
jz \exit
|
2015-02-27 05:40:30 +07:00
|
|
|
CLEAR_RREGS
|
2009-09-30 17:22:11 +07:00
|
|
|
jmp int_with_check
|
2008-06-24 15:13:31 +07:00
|
|
|
.endm
|
|
|
|
|
|
|
|
sysenter_auditsys:
|
|
|
|
auditsys_entry_common
|
|
|
|
movl %ebp,%r9d /* reload 6th syscall arg */
|
|
|
|
jmp sysenter_dispatch
|
|
|
|
|
|
|
|
sysexit_audit:
|
|
|
|
auditsys_exit sysexit_from_sys_call
|
|
|
|
#endif
|
|
|
|
|
x86_64, entry: Filter RFLAGS.NT on entry from userspace
The NT flag doesn't do anything in long mode other than causing IRET
to #GP. Oddly, CPL3 code can still set NT using popf.
Entry via hardware or software interrupt clears NT automatically, so
the only relevant entries are fast syscalls.
If user code causes kernel code to run with NT set, then there's at
least some (small) chance that it could cause trouble. For example,
user code could cause a call to EFI code with NT set, and who knows
what would happen? Apparently some games on Wine sometimes do
this (!), and, if an IRET return happens, they will segfault. That
segfault cannot be handled, because signal delivery fails, too.
This patch programs the CPU to clear NT on entry via SYSCALL (both
32-bit and 64-bit, by my reading of the AMD APM), and it clears NT
in software on entry via SYSENTER.
To save a few cycles, this borrows a trick from Jan Beulich in Xen:
it checks whether NT is set before trying to clear it. As a result,
it seems to have very little effect on SYSENTER performance on my
machine.
There's another minor bug fix in here: it looks like the CFI
annotations were wrong if CONFIG_AUDITSYSCALL=n.
Testers beware: on Xen, SYSENTER with NT set turns into a GPF.
I haven't touched anything on 32-bit kernels.
The syscall mask change comes from a variant of this patch by Anish
Bhatt.
Note to stable maintainers: there is no known security issue here.
A misguided program can set NT and cause the kernel to try and fail
to deliver SIGSEGV, crashing the program. This patch fixes Far Cry
on Wine: https://bugs.winehq.org/show_bug.cgi?id=33275
Cc: <stable@vger.kernel.org>
Reported-by: Anish Bhatt <anish@chelsio.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Link: http://lkml.kernel.org/r/395749a5d39a29bd3e4b35899cf3a3c1340e5595.1412189265.git.luto@amacapital.net
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
2014-10-02 01:49:04 +07:00
|
|
|
sysenter_fix_flags:
|
|
|
|
pushq_cfi $(X86_EFLAGS_IF|X86_EFLAGS_FIXED)
|
|
|
|
popfq_cfi
|
|
|
|
jmp sysenter_flags_fixed
|
|
|
|
|
2008-06-24 15:13:31 +07:00
|
|
|
sysenter_tracesys:
|
|
|
|
#ifdef CONFIG_AUDITSYSCALL
|
2015-03-25 01:44:42 +07:00
|
|
|
testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
|
2008-06-24 15:13:31 +07:00
|
|
|
jz sysenter_auditsys
|
|
|
|
#endif
|
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 05:40:27 +07:00
|
|
|
SAVE_EXTRA_REGS
|
2005-04-17 05:20:36 +07:00
|
|
|
CLEAR_RREGS
|
2008-03-19 08:23:50 +07:00
|
|
|
movq $-ENOSYS,RAX(%rsp)/* ptrace can change this for a bad syscall */
|
2005-04-17 05:20:36 +07:00
|
|
|
movq %rsp,%rdi /* &pt_regs -> arg1 */
|
|
|
|
call syscall_trace_enter
|
2015-02-27 05:40:30 +07:00
|
|
|
LOAD_ARGS32 /* reload args from stack in case ptrace changed it */
|
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 05:40:27 +07:00
|
|
|
RESTORE_EXTRA_REGS
|
x86-64, compat: Test %rax for the syscall number, not %eax
On 64 bits, we always, by necessity, jump through the system call
table via %rax. For 32-bit system calls, in theory the system call
number is stored in %eax, and the code was testing %eax for a valid
system call number. At one point we loaded the stored value back from
the stack to enforce zero-extension, but that was removed in checkin
d4d67150165df8bf1cc05e532f6efca96f907cab. An actual 32-bit process
will not be able to introduce a non-zero-extended number, but it can
happen via ptrace.
Instead of re-introducing the zero-extension, test what we are
actually going to use, i.e. %rax. This only adds a handful of REX
prefixes to the code.
Reported-by: Ben Hawkes <hawkes@sota.gen.nz>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
Cc: <stable@kernel.org>
Cc: Roland McGrath <roland@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
2010-09-15 02:42:41 +07:00
|
|
|
cmpq $(IA32_NR_syscalls-1),%rax
|
2008-03-19 08:23:50 +07:00
|
|
|
ja int_ret_from_sys_call /* sysenter_tracesys has set RAX(%rsp) */
|
2005-04-17 05:20:36 +07:00
|
|
|
jmp sysenter_do_call
|
|
|
|
CFI_ENDPROC
|
2006-06-26 18:56:55 +07:00
|
|
|
ENDPROC(ia32_sysenter_target)
|
2005-04-17 05:20:36 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* 32bit SYSCALL instruction entry.
|
|
|
|
*
|
2015-02-27 05:40:32 +07:00
|
|
|
* 32bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
|
|
|
|
* then loads new ss, cs, and rip from previously programmed MSRs.
|
|
|
|
* rflags gets masked by a value from another MSR (so CLD and CLAC
|
|
|
|
* are not needed). SYSCALL does not save anything on the stack
|
|
|
|
* and does not change rsp.
|
|
|
|
*
|
|
|
|
* Note: rflags saving+masking-with-MSR happens only in Long mode
|
|
|
|
* (in legacy 32bit mode, IF, RF and VM bits are cleared and that's it).
|
|
|
|
* Don't get confused: rflags saving+masking depends on Long Mode Active bit
|
|
|
|
* (EFER.LMA=1), NOT on bitness of userspace where SYSCALL executes
|
|
|
|
* or target CS descriptor's L bit (SYSCALL does not read segment descriptors).
|
|
|
|
*
|
2005-04-17 05:20:36 +07:00
|
|
|
* Arguments:
|
2015-02-27 05:40:32 +07:00
|
|
|
* eax system call number
|
|
|
|
* ecx return address
|
|
|
|
* ebx arg1
|
|
|
|
* ebp arg2 (note: not saved in the stack frame, should not be touched)
|
|
|
|
* edx arg3
|
|
|
|
* esi arg4
|
|
|
|
* edi arg5
|
|
|
|
* esp user stack
|
|
|
|
* 0(%esp) arg6
|
|
|
|
*
|
2005-04-17 05:20:36 +07:00
|
|
|
* This is purely a fast path. For anything complicated we use the int 0x80
|
2015-02-27 05:40:32 +07:00
|
|
|
* path below. We set up a complete hardware stack frame to share code
|
|
|
|
* with the int 0x80 path.
|
|
|
|
*/
|
2005-04-17 05:20:36 +07:00
|
|
|
ENTRY(ia32_cstar_target)
|
2006-01-12 04:41:59 +07:00
|
|
|
CFI_STARTPROC32 simple
|
2006-09-26 15:52:41 +07:00
|
|
|
CFI_SIGNAL_FRAME
|
2015-03-20 00:17:46 +07:00
|
|
|
CFI_DEF_CFA rsp,0
|
2005-09-12 23:49:24 +07:00
|
|
|
CFI_REGISTER rip,rcx
|
|
|
|
/*CFI_REGISTER rflags,r11*/
|
2015-03-27 17:36:20 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Interrupts are off on entry.
|
|
|
|
* We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
|
|
|
|
* it is too small to ever cause noticeable irq latency.
|
|
|
|
*/
|
2008-06-26 21:28:51 +07:00
|
|
|
SWAPGS_UNSAFE_STACK
|
2005-04-17 05:20:36 +07:00
|
|
|
movl %esp,%r8d
|
2005-09-12 23:49:24 +07:00
|
|
|
CFI_REGISTER rsp,r8
|
2009-01-18 22:38:58 +07:00
|
|
|
movq PER_CPU_VAR(kernel_stack),%rsp
|
2008-06-25 11:19:29 +07:00
|
|
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
2015-03-27 17:36:20 +07:00
|
|
|
|
2015-03-27 17:36:21 +07:00
|
|
|
/* Zero-extending 32-bit regs, do not remove */
|
|
|
|
movl %eax,%eax
|
|
|
|
|
2015-04-01 00:00:04 +07:00
|
|
|
/* Construct struct pt_regs on stack */
|
|
|
|
pushq_cfi $__USER32_DS /* pt_regs->ss */
|
|
|
|
pushq_cfi %r8 /* pt_regs->sp */
|
|
|
|
CFI_REL_OFFSET rsp,0
|
|
|
|
pushq_cfi %r11 /* pt_regs->flags */
|
|
|
|
pushq_cfi $__USER32_CS /* pt_regs->cs */
|
|
|
|
pushq_cfi %rcx /* pt_regs->ip */
|
|
|
|
CFI_REL_OFFSET rip,0
|
|
|
|
pushq_cfi_reg rax /* pt_regs->orig_ax */
|
|
|
|
pushq_cfi_reg rdi /* pt_regs->di */
|
|
|
|
pushq_cfi_reg rsi /* pt_regs->si */
|
|
|
|
pushq_cfi_reg rdx /* pt_regs->dx */
|
|
|
|
pushq_cfi_reg rbp /* pt_regs->cx */
|
2005-04-17 05:20:36 +07:00
|
|
|
movl %ebp,%ecx
|
2015-04-01 00:00:04 +07:00
|
|
|
pushq_cfi_reg rax /* pt_regs->ax */
|
|
|
|
sub $(10*8),%rsp /* pt_regs->r8-11,bp,bx,r12-15 not saved */
|
|
|
|
CFI_ADJUST_CFA_OFFSET 10*8
|
|
|
|
|
2015-02-27 05:40:32 +07:00
|
|
|
/*
|
|
|
|
* no need to do an access_ok check here because r8 has been
|
|
|
|
* 32bit zero extended
|
|
|
|
*/
|
2012-09-22 02:43:12 +07:00
|
|
|
ASM_STAC
|
2005-04-17 05:20:36 +07:00
|
|
|
1: movl (%r8),%r9d
|
2012-04-21 06:51:50 +07:00
|
|
|
_ASM_EXTABLE(1b,ia32_badarg)
|
2012-09-22 02:43:12 +07:00
|
|
|
ASM_CLAC
|
2015-03-25 01:44:42 +07:00
|
|
|
orl $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
|
|
|
|
testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
|
2005-09-12 23:49:24 +07:00
|
|
|
CFI_REMEMBER_STATE
|
2005-04-17 05:20:36 +07:00
|
|
|
jnz cstar_tracesys
|
x86-64, compat: Test %rax for the syscall number, not %eax
On 64 bits, we always, by necessity, jump through the system call
table via %rax. For 32-bit system calls, in theory the system call
number is stored in %eax, and the code was testing %eax for a valid
system call number. At one point we loaded the stored value back from
the stack to enforce zero-extension, but that was removed in checkin
d4d67150165df8bf1cc05e532f6efca96f907cab. An actual 32-bit process
will not be able to introduce a non-zero-extended number, but it can
happen via ptrace.
Instead of re-introducing the zero-extension, test what we are
actually going to use, i.e. %rax. This only adds a handful of REX
prefixes to the code.
Reported-by: Ben Hawkes <hawkes@sota.gen.nz>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
Cc: <stable@kernel.org>
Cc: Roland McGrath <roland@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
2010-09-15 02:42:41 +07:00
|
|
|
cmpq $IA32_NR_syscalls-1,%rax
|
2006-04-08 00:50:31 +07:00
|
|
|
ja ia32_badsys
|
2008-08-29 19:21:11 +07:00
|
|
|
cstar_do_call:
|
2015-02-27 05:40:35 +07:00
|
|
|
/* 32bit syscall -> 64bit C ABI argument conversion */
|
|
|
|
movl %edi,%r8d /* arg5 */
|
|
|
|
/* r9 already loaded */ /* arg6 */
|
|
|
|
xchg %ecx,%esi /* rsi:arg2, rcx:arg4 */
|
|
|
|
movl %ebx,%edi /* arg1 */
|
|
|
|
movl %edx,%edx /* arg3 (zero extension) */
|
2008-06-24 15:13:31 +07:00
|
|
|
cstar_dispatch:
|
2005-04-17 05:20:36 +07:00
|
|
|
call *ia32_sys_call_table(,%rax,8)
|
2015-02-27 05:40:30 +07:00
|
|
|
movq %rax,RAX(%rsp)
|
2008-06-25 11:19:29 +07:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 14:24:45 +07:00
|
|
|
TRACE_IRQS_OFF
|
2015-03-25 01:44:42 +07:00
|
|
|
testl $_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
|
2008-06-24 15:13:31 +07:00
|
|
|
jnz sysretl_audit
|
|
|
|
sysretl_from_sys_call:
|
2015-03-25 01:44:42 +07:00
|
|
|
andl $~TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
|
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 05:40:27 +07:00
|
|
|
RESTORE_RSI_RDI_RDX
|
2015-02-27 05:40:30 +07:00
|
|
|
movl RIP(%rsp),%ecx
|
2005-09-12 23:49:24 +07:00
|
|
|
CFI_REGISTER rip,rcx
|
2015-02-27 05:40:30 +07:00
|
|
|
movl EFLAGS(%rsp),%r11d
|
2005-09-12 23:49:24 +07:00
|
|
|
/*CFI_REGISTER rflags,r11*/
|
2009-09-30 17:22:11 +07:00
|
|
|
xorq %r10,%r10
|
|
|
|
xorq %r9,%r9
|
|
|
|
xorq %r8,%r8
|
2006-07-03 14:24:45 +07:00
|
|
|
TRACE_IRQS_ON
|
2015-02-27 05:40:30 +07:00
|
|
|
movl RSP(%rsp),%esp
|
2005-09-12 23:49:24 +07:00
|
|
|
CFI_RESTORE rsp
|
2015-02-27 05:40:32 +07:00
|
|
|
/*
|
|
|
|
* 64bit->32bit SYSRET restores eip from ecx,
|
|
|
|
* eflags from r11 (but RF and VM bits are forced to 0),
|
|
|
|
* cs and ss are loaded from MSRs.
|
|
|
|
* (Note: 32bit->32bit SYSRET is different: since r11
|
|
|
|
* does not exist, it merely sets eflags.IF=1).
|
|
|
|
*/
|
2008-06-25 11:19:28 +07:00
|
|
|
USERGS_SYSRET32
|
2015-02-27 05:40:32 +07:00
|
|
|
|
2008-06-24 15:13:31 +07:00
|
|
|
#ifdef CONFIG_AUDITSYSCALL
|
|
|
|
cstar_auditsys:
|
2005-09-12 23:49:24 +07:00
|
|
|
CFI_RESTORE_STATE
|
2015-02-27 05:40:30 +07:00
|
|
|
movl %r9d,R9(%rsp) /* register to be clobbered by call */
|
2008-06-24 15:13:31 +07:00
|
|
|
auditsys_entry_common
|
2015-02-27 05:40:30 +07:00
|
|
|
movl R9(%rsp),%r9d /* reload 6th syscall arg */
|
2008-06-24 15:13:31 +07:00
|
|
|
jmp cstar_dispatch
|
|
|
|
|
|
|
|
sysretl_audit:
|
2009-10-26 22:20:29 +07:00
|
|
|
auditsys_exit sysretl_from_sys_call
|
2008-06-24 15:13:31 +07:00
|
|
|
#endif
|
|
|
|
|
|
|
|
cstar_tracesys:
|
|
|
|
#ifdef CONFIG_AUDITSYSCALL
|
2015-03-25 01:44:42 +07:00
|
|
|
testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
|
2008-06-24 15:13:31 +07:00
|
|
|
jz cstar_auditsys
|
|
|
|
#endif
|
2007-11-07 22:48:39 +07:00
|
|
|
xchgl %r9d,%ebp
|
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 05:40:27 +07:00
|
|
|
SAVE_EXTRA_REGS
|
2015-02-27 05:40:30 +07:00
|
|
|
CLEAR_RREGS r9
|
2008-03-19 08:23:50 +07:00
|
|
|
movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
|
2005-04-17 05:20:36 +07:00
|
|
|
movq %rsp,%rdi /* &pt_regs -> arg1 */
|
|
|
|
call syscall_trace_enter
|
2015-02-27 05:40:30 +07:00
|
|
|
LOAD_ARGS32 1 /* reload args from stack in case ptrace changed it */
|
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 05:40:27 +07:00
|
|
|
RESTORE_EXTRA_REGS
|
2007-11-07 22:48:39 +07:00
|
|
|
xchgl %ebp,%r9d
|
x86-64, compat: Test %rax for the syscall number, not %eax
On 64 bits, we always, by necessity, jump through the system call
table via %rax. For 32-bit system calls, in theory the system call
number is stored in %eax, and the code was testing %eax for a valid
system call number. At one point we loaded the stored value back from
the stack to enforce zero-extension, but that was removed in checkin
d4d67150165df8bf1cc05e532f6efca96f907cab. An actual 32-bit process
will not be able to introduce a non-zero-extended number, but it can
happen via ptrace.
Instead of re-introducing the zero-extension, test what we are
actually going to use, i.e. %rax. This only adds a handful of REX
prefixes to the code.
Reported-by: Ben Hawkes <hawkes@sota.gen.nz>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
Cc: <stable@kernel.org>
Cc: Roland McGrath <roland@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
2010-09-15 02:42:41 +07:00
|
|
|
cmpq $(IA32_NR_syscalls-1),%rax
|
2008-03-19 08:23:50 +07:00
|
|
|
ja int_ret_from_sys_call /* cstar_tracesys has set RAX(%rsp) */
|
2005-04-17 05:20:36 +07:00
|
|
|
jmp cstar_do_call
|
2006-06-26 18:56:55 +07:00
|
|
|
END(ia32_cstar_target)
|
2005-04-17 05:20:36 +07:00
|
|
|
|
|
|
|
ia32_badarg:
|
2012-09-22 02:43:12 +07:00
|
|
|
ASM_CLAC
|
2005-04-17 05:20:36 +07:00
|
|
|
movq $-EFAULT,%rax
|
|
|
|
jmp ia32_sysret
|
|
|
|
CFI_ENDPROC
|
|
|
|
|
2015-02-27 05:40:32 +07:00
|
|
|
/*
|
|
|
|
* Emulated IA32 system calls via int 0x80.
|
2005-04-17 05:20:36 +07:00
|
|
|
*
|
2015-02-27 05:40:32 +07:00
|
|
|
* Arguments:
|
|
|
|
* eax system call number
|
|
|
|
* ebx arg1
|
|
|
|
* ecx arg2
|
|
|
|
* edx arg3
|
|
|
|
* esi arg4
|
|
|
|
* edi arg5
|
|
|
|
* ebp arg6 (note: not saved in the stack frame, should not be touched)
|
2005-04-17 05:20:36 +07:00
|
|
|
*
|
|
|
|
* Notes:
|
2015-02-27 05:40:32 +07:00
|
|
|
* Uses the same stack frame as the x86-64 version.
|
|
|
|
* All registers except eax must be saved (but ptrace may violate that).
|
2005-04-17 05:20:36 +07:00
|
|
|
* Arguments are zero extended. For system calls that want sign extension and
|
|
|
|
* take long arguments a wrapper is needed. Most calls can just be called
|
|
|
|
* directly.
|
2015-02-27 05:40:32 +07:00
|
|
|
* Assumes it is only called from user space and entered with interrupts off.
|
|
|
|
*/
|
2005-04-17 05:20:36 +07:00
|
|
|
|
|
|
|
ENTRY(ia32_syscall)
|
2007-07-21 22:10:20 +07:00
|
|
|
CFI_STARTPROC32 simple
|
2006-09-26 15:52:41 +07:00
|
|
|
CFI_SIGNAL_FRAME
|
2015-03-27 17:36:20 +07:00
|
|
|
CFI_DEF_CFA rsp,5*8
|
|
|
|
/*CFI_REL_OFFSET ss,4*8 */
|
|
|
|
CFI_REL_OFFSET rsp,3*8
|
|
|
|
/*CFI_REL_OFFSET rflags,2*8 */
|
|
|
|
/*CFI_REL_OFFSET cs,1*8 */
|
|
|
|
CFI_REL_OFFSET rip,0*8
|
|
|
|
|
2006-07-03 14:24:45 +07:00
|
|
|
/*
|
2015-03-27 17:36:20 +07:00
|
|
|
* Interrupts are off on entry.
|
|
|
|
* We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
|
|
|
|
* it is too small to ever cause noticeable irq latency.
|
2006-07-03 14:24:45 +07:00
|
|
|
*/
|
2015-03-27 17:36:20 +07:00
|
|
|
PARAVIRT_ADJUST_EXCEPTION_FRAME
|
|
|
|
SWAPGS
|
2008-06-25 11:19:29 +07:00
|
|
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
2015-03-27 17:36:20 +07:00
|
|
|
|
2015-03-27 17:36:21 +07:00
|
|
|
/* Zero-extending 32-bit regs, do not remove */
|
|
|
|
movl %eax,%eax
|
|
|
|
|
2015-04-01 00:00:04 +07:00
|
|
|
/* Construct struct pt_regs on stack (iret frame is already on stack) */
|
|
|
|
pushq_cfi_reg rax /* pt_regs->orig_ax */
|
|
|
|
pushq_cfi_reg rdi /* pt_regs->di */
|
|
|
|
pushq_cfi_reg rsi /* pt_regs->si */
|
|
|
|
pushq_cfi_reg rdx /* pt_regs->dx */
|
|
|
|
pushq_cfi_reg rcx /* pt_regs->cx */
|
|
|
|
pushq_cfi_reg rax /* pt_regs->ax */
|
2005-04-17 05:20:36 +07:00
|
|
|
cld
|
2015-04-01 00:00:04 +07:00
|
|
|
sub $(10*8),%rsp /* pt_regs->r8-11,bp,bx,r12-15 not saved */
|
|
|
|
CFI_ADJUST_CFA_OFFSET 10*8
|
|
|
|
|
2015-03-25 01:44:42 +07:00
|
|
|
orl $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
|
|
|
|
testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
|
2005-04-17 05:20:36 +07:00
|
|
|
jnz ia32_tracesys
|
x86-64, compat: Test %rax for the syscall number, not %eax
On 64 bits, we always, by necessity, jump through the system call
table via %rax. For 32-bit system calls, in theory the system call
number is stored in %eax, and the code was testing %eax for a valid
system call number. At one point we loaded the stored value back from
the stack to enforce zero-extension, but that was removed in checkin
d4d67150165df8bf1cc05e532f6efca96f907cab. An actual 32-bit process
will not be able to introduce a non-zero-extended number, but it can
happen via ptrace.
Instead of re-introducing the zero-extension, test what we are
actually going to use, i.e. %rax. This only adds a handful of REX
prefixes to the code.
Reported-by: Ben Hawkes <hawkes@sota.gen.nz>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
Cc: <stable@kernel.org>
Cc: Roland McGrath <roland@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
2010-09-15 02:42:41 +07:00
|
|
|
cmpq $(IA32_NR_syscalls-1),%rax
|
2009-02-07 09:15:18 +07:00
|
|
|
ja ia32_badsys
|
|
|
|
ia32_do_call:
|
2015-02-27 05:40:35 +07:00
|
|
|
/* 32bit syscall -> 64bit C ABI argument conversion */
|
|
|
|
movl %edi,%r8d /* arg5 */
|
|
|
|
movl %ebp,%r9d /* arg6 */
|
|
|
|
xchg %ecx,%esi /* rsi:arg2, rcx:arg4 */
|
|
|
|
movl %ebx,%edi /* arg1 */
|
|
|
|
movl %edx,%edx /* arg3 (zero extension) */
|
2005-04-17 05:20:36 +07:00
|
|
|
call *ia32_sys_call_table(,%rax,8) # xxx: rip relative
|
|
|
|
ia32_sysret:
|
2015-02-27 05:40:30 +07:00
|
|
|
movq %rax,RAX(%rsp)
|
2009-09-30 17:22:11 +07:00
|
|
|
ia32_ret_from_sys_call:
|
2015-02-27 05:40:30 +07:00
|
|
|
CLEAR_RREGS
|
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 05:40:27 +07:00
|
|
|
jmp int_ret_from_sys_call
|
2005-04-17 05:20:36 +07:00
|
|
|
|
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 05:40:27 +07:00
|
|
|
ia32_tracesys:
|
|
|
|
SAVE_EXTRA_REGS
|
2007-07-21 22:10:20 +07:00
|
|
|
CLEAR_RREGS
|
2008-03-17 11:57:41 +07:00
|
|
|
movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
|
2005-04-17 05:20:36 +07:00
|
|
|
movq %rsp,%rdi /* &pt_regs -> arg1 */
|
|
|
|
call syscall_trace_enter
|
2015-02-27 05:40:30 +07:00
|
|
|
LOAD_ARGS32 /* reload args from stack in case ptrace changed it */
|
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 05:40:27 +07:00
|
|
|
RESTORE_EXTRA_REGS
|
x86-64, compat: Test %rax for the syscall number, not %eax
On 64 bits, we always, by necessity, jump through the system call
table via %rax. For 32-bit system calls, in theory the system call
number is stored in %eax, and the code was testing %eax for a valid
system call number. At one point we loaded the stored value back from
the stack to enforce zero-extension, but that was removed in checkin
d4d67150165df8bf1cc05e532f6efca96f907cab. An actual 32-bit process
will not be able to introduce a non-zero-extended number, but it can
happen via ptrace.
Instead of re-introducing the zero-extension, test what we are
actually going to use, i.e. %rax. This only adds a handful of REX
prefixes to the code.
Reported-by: Ben Hawkes <hawkes@sota.gen.nz>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
Cc: <stable@kernel.org>
Cc: Roland McGrath <roland@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
2010-09-15 02:42:41 +07:00
|
|
|
cmpq $(IA32_NR_syscalls-1),%rax
|
2009-02-07 09:15:18 +07:00
|
|
|
ja int_ret_from_sys_call /* ia32_tracesys has set RAX(%rsp) */
|
|
|
|
jmp ia32_do_call
|
2006-06-26 18:56:55 +07:00
|
|
|
END(ia32_syscall)
|
2005-04-17 05:20:36 +07:00
|
|
|
|
|
|
|
ia32_badsys:
|
2015-02-27 05:40:30 +07:00
|
|
|
movq $0,ORIG_RAX(%rsp)
|
2009-09-30 17:22:11 +07:00
|
|
|
movq $-ENOSYS,%rax
|
|
|
|
jmp ia32_sysret
|
2005-04-17 05:20:36 +07:00
|
|
|
|
|
|
|
CFI_ENDPROC
|
|
|
|
|
2013-07-11 01:04:28 +07:00
|
|
|
.macro PTREGSCALL label, func
|
2011-11-29 18:24:10 +07:00
|
|
|
ALIGN
|
|
|
|
GLOBAL(\label)
|
2005-04-17 05:20:36 +07:00
|
|
|
leaq \func(%rip),%rax
|
|
|
|
jmp ia32_ptregs_common
|
|
|
|
.endm
|
|
|
|
|
2006-01-12 04:41:59 +07:00
|
|
|
CFI_STARTPROC32
|
2005-09-12 23:49:24 +07:00
|
|
|
|
2012-11-13 02:32:42 +07:00
|
|
|
PTREGSCALL stub32_rt_sigreturn, sys32_rt_sigreturn
|
|
|
|
PTREGSCALL stub32_sigreturn, sys32_sigreturn
|
|
|
|
PTREGSCALL stub32_execve, compat_sys_execve
|
2014-12-13 07:57:33 +07:00
|
|
|
PTREGSCALL stub32_execveat, compat_sys_execveat
|
2012-11-13 02:32:42 +07:00
|
|
|
PTREGSCALL stub32_fork, sys_fork
|
|
|
|
PTREGSCALL stub32_vfork, sys_vfork
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2012-10-23 09:34:11 +07:00
|
|
|
ALIGN
|
|
|
|
GLOBAL(stub32_clone)
|
|
|
|
leaq sys_clone(%rip),%rax
|
|
|
|
mov %r8, %rcx
|
|
|
|
jmp ia32_ptregs_common
|
|
|
|
|
2011-11-29 18:24:10 +07:00
|
|
|
ALIGN
|
|
|
|
ia32_ptregs_common:
|
2006-01-12 04:41:59 +07:00
|
|
|
CFI_ENDPROC
|
|
|
|
CFI_STARTPROC32 simple
|
2006-09-26 15:52:41 +07:00
|
|
|
CFI_SIGNAL_FRAME
|
2015-02-27 05:40:37 +07:00
|
|
|
CFI_DEF_CFA rsp,SIZEOF_PTREGS
|
2015-02-27 05:40:30 +07:00
|
|
|
CFI_REL_OFFSET rax,RAX
|
|
|
|
CFI_REL_OFFSET rcx,RCX
|
|
|
|
CFI_REL_OFFSET rdx,RDX
|
|
|
|
CFI_REL_OFFSET rsi,RSI
|
|
|
|
CFI_REL_OFFSET rdi,RDI
|
|
|
|
CFI_REL_OFFSET rip,RIP
|
|
|
|
/* CFI_REL_OFFSET cs,CS*/
|
|
|
|
/* CFI_REL_OFFSET rflags,EFLAGS*/
|
|
|
|
CFI_REL_OFFSET rsp,RSP
|
|
|
|
/* CFI_REL_OFFSET ss,SS*/
|
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 05:40:27 +07:00
|
|
|
SAVE_EXTRA_REGS 8
|
2005-04-17 05:20:36 +07:00
|
|
|
call *%rax
|
x86/asm/entry/64: Always allocate a complete "struct pt_regs" on the kernel stack
The 64-bit entry code was using six stack slots less by not
saving/restoring registers which are callee-preserved according
to the C ABI, and was not allocating space for them.
Only when syscalls needed a complete "struct pt_regs" was
the complete area allocated and filled in.
As an additional twist, on interrupt entry a "slightly less
truncated pt_regs" trick is used, to make nested interrupt
stacks easier to unwind.
This proved to be a source of significant obfuscation and subtle
bugs. For example, 'stub_fork' had to pop the return address,
extend the struct, save registers, and push return address back.
Ugly. 'ia32_ptregs_common' pops return address and "returns" via
jmp insn, throwing a wrench into CPU return stack cache.
This patch changes the code to always allocate a complete
"struct pt_regs" on the kernel stack. The saving of registers
is still done lazily.
"Partial pt_regs" trick on interrupt stack is retained.
Macros which manipulate "struct pt_regs" on stack are reworked:
- ALLOC_PT_GPREGS_ON_STACK allocates the structure.
- SAVE_C_REGS saves to it those registers which are clobbered
by C code.
- SAVE_EXTRA_REGS saves to it all other registers.
- Corresponding RESTORE_* and REMOVE_PT_GPREGS_FROM_STACK macros
reverse it.
'ia32_ptregs_common', 'stub_fork' and friends lost their ugly dance
with the return pointer.
LOAD_ARGS32 in ia32entry.S now uses symbolic stack offsets
instead of magic numbers.
'error_entry' and 'save_paranoid' now use SAVE_C_REGS +
SAVE_EXTRA_REGS instead of having it open-coded yet again.
Patch was run-tested: 64-bit executables, 32-bit executables,
strace works.
Timing tests did not show measurable difference in 32-bit
and 64-bit syscalls.
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1423778052-21038-2-git-send-email-dvlasenk@redhat.com
Link: http://lkml.kernel.org/r/b89763d354aa23e670b9bdf3a40ae320320a7c2e.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-02-27 05:40:27 +07:00
|
|
|
RESTORE_EXTRA_REGS 8
|
|
|
|
ret
|
2005-04-17 05:20:36 +07:00
|
|
|
CFI_ENDPROC
|
2006-06-26 18:56:55 +07:00
|
|
|
END(ia32_ptregs_common)
|