2005-04-17 05:20:36 +07:00
|
|
|
/*
|
|
|
|
* Copyright (C) 1992, 1998 Linus Torvalds, Ingo Molnar
|
|
|
|
*
|
|
|
|
* This file contains the lowest level x86-specific interrupt
|
|
|
|
* entry, irq-stacks and irq statistics code. All the remaining
|
|
|
|
* irq logic is done by the generic kernel/irq/ code and
|
|
|
|
* by the x86-specific irq controller code. (e.g. i8259.c and
|
|
|
|
* io_apic.c.)
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/seq_file.h>
|
|
|
|
#include <linux/interrupt.h>
|
|
|
|
#include <linux/kernel_stat.h>
|
2005-06-26 04:54:50 +07:00
|
|
|
#include <linux/notifier.h>
|
|
|
|
#include <linux/cpu.h>
|
|
|
|
#include <linux/delay.h>
|
2009-01-04 18:02:36 +07:00
|
|
|
#include <linux/uaccess.h>
|
2009-02-17 10:46:42 +07:00
|
|
|
#include <linux/percpu.h>
|
2010-10-28 21:40:54 +07:00
|
|
|
#include <linux/mm.h>
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2007-02-16 16:27:58 +07:00
|
|
|
#include <asm/apic.h>
|
|
|
|
|
2008-05-05 20:58:15 +07:00
|
|
|
#ifdef CONFIG_DEBUG_STACKOVERFLOW
|
2011-12-05 18:25:44 +07:00
|
|
|
|
|
|
|
int sysctl_panic_on_stackoverflow __read_mostly;
|
|
|
|
|
2008-05-05 20:58:15 +07:00
|
|
|
/* Debugging check for stack overflow: is there less than 1KB free? */
|
|
|
|
static int check_stack_overflow(void)
|
|
|
|
{
|
|
|
|
long sp;
|
|
|
|
|
|
|
|
__asm__ __volatile__("andl %%esp,%0" :
|
|
|
|
"=r" (sp) : "0" (THREAD_SIZE - 1));
|
|
|
|
|
|
|
|
return sp < (sizeof(struct thread_info) + STACK_WARN);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void print_stack_overflow(void)
|
|
|
|
{
|
|
|
|
printk(KERN_WARNING "low stack detected by irq handler\n");
|
|
|
|
dump_stack();
|
2011-11-29 13:08:36 +07:00
|
|
|
if (sysctl_panic_on_stackoverflow)
|
|
|
|
panic("low stack detected by irq handler - check messages\n");
|
2008-05-05 20:58:15 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
|
|
|
static inline int check_stack_overflow(void) { return 0; }
|
|
|
|
static inline void print_stack_overflow(void) { }
|
|
|
|
#endif
|
|
|
|
|
2014-02-06 21:41:31 +07:00
|
|
|
DEFINE_PER_CPU(struct irq_stack *, hardirq_stack);
|
|
|
|
DEFINE_PER_CPU(struct irq_stack *, softirq_stack);
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2008-05-05 23:13:50 +07:00
|
|
|
static void call_on_stack(void *func, void *stack)
|
2008-05-05 17:36:38 +07:00
|
|
|
{
|
2008-05-05 23:13:50 +07:00
|
|
|
asm volatile("xchgl %%ebx,%%esp \n"
|
|
|
|
"call *%%edi \n"
|
|
|
|
"movl %%ebx,%%esp \n"
|
|
|
|
: "=b" (stack)
|
|
|
|
: "0" (stack),
|
|
|
|
"D"(func)
|
|
|
|
: "memory", "cc", "edx", "ecx", "eax");
|
2008-05-05 17:36:38 +07:00
|
|
|
}
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2014-02-06 21:41:31 +07:00
|
|
|
static inline void *current_stack(void)
|
|
|
|
{
|
2014-11-14 06:57:07 +07:00
|
|
|
return (void *)(current_stack_pointer() & ~(THREAD_SIZE - 1));
|
2014-02-06 21:41:31 +07:00
|
|
|
}
|
|
|
|
|
2015-09-14 15:42:37 +07:00
|
|
|
static inline int execute_on_irq_stack(int overflow, struct irq_desc *desc)
|
2008-05-05 20:58:15 +07:00
|
|
|
{
|
2014-02-06 21:41:31 +07:00
|
|
|
struct irq_stack *curstk, *irqstk;
|
2015-09-14 15:42:37 +07:00
|
|
|
u32 *isp, *prev_esp, arg1;
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2014-02-06 21:41:31 +07:00
|
|
|
curstk = (struct irq_stack *) current_stack();
|
|
|
|
irqstk = __this_cpu_read(hardirq_stack);
|
2005-04-17 05:20:36 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* this is where we switch to the IRQ stack. However, if we are
|
|
|
|
* already using the IRQ stack (because we interrupted a hardirq
|
|
|
|
* handler) we can't do that and just have to keep using the
|
|
|
|
* current stack (which is the irq stack already after all)
|
|
|
|
*/
|
2014-02-06 21:41:31 +07:00
|
|
|
if (unlikely(curstk == irqstk))
|
2008-05-05 20:58:15 +07:00
|
|
|
return 0;
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2014-02-06 21:41:31 +07:00
|
|
|
isp = (u32 *) ((char *)irqstk + sizeof(*irqstk));
|
|
|
|
|
|
|
|
/* Save the next esp at the bottom of the stack */
|
|
|
|
prev_esp = (u32 *)irqstk;
|
2014-11-14 06:57:07 +07:00
|
|
|
*prev_esp = current_stack_pointer();
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2008-05-05 20:58:15 +07:00
|
|
|
if (unlikely(overflow))
|
2008-05-05 23:13:50 +07:00
|
|
|
call_on_stack(print_stack_overflow, isp);
|
|
|
|
|
|
|
|
asm volatile("xchgl %%ebx,%%esp \n"
|
|
|
|
"call *%%edi \n"
|
|
|
|
"movl %%ebx,%%esp \n"
|
2015-09-14 15:42:37 +07:00
|
|
|
: "=a" (arg1), "=b" (isp)
|
|
|
|
: "0" (desc), "1" (isp),
|
2008-05-05 23:13:50 +07:00
|
|
|
"D" (desc->handle_irq)
|
|
|
|
: "memory", "cc", "ecx");
|
2005-04-17 05:20:36 +07:00
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* allocate per-cpu stacks for hardirq and for softirq processing
|
|
|
|
*/
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-19 05:23:59 +07:00
|
|
|
void irq_ctx_init(int cpu)
|
2005-04-17 05:20:36 +07:00
|
|
|
{
|
2014-02-06 21:41:31 +07:00
|
|
|
struct irq_stack *irqstk;
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2014-02-06 21:41:31 +07:00
|
|
|
if (per_cpu(hardirq_stack, cpu))
|
2005-04-17 05:20:36 +07:00
|
|
|
return;
|
|
|
|
|
2014-02-06 21:41:31 +07:00
|
|
|
irqstk = page_address(alloc_pages_node(cpu_to_node(cpu),
|
2012-05-05 22:05:42 +07:00
|
|
|
THREADINFO_GFP,
|
|
|
|
THREAD_SIZE_ORDER));
|
2014-02-06 21:41:31 +07:00
|
|
|
per_cpu(hardirq_stack, cpu) = irqstk;
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2014-02-06 21:41:31 +07:00
|
|
|
irqstk = page_address(alloc_pages_node(cpu_to_node(cpu),
|
2012-05-05 22:05:42 +07:00
|
|
|
THREADINFO_GFP,
|
|
|
|
THREAD_SIZE_ORDER));
|
2014-02-06 21:41:31 +07:00
|
|
|
per_cpu(softirq_stack, cpu) = irqstk;
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2008-05-05 23:13:50 +07:00
|
|
|
printk(KERN_DEBUG "CPU %u irqstacks, hard=%p soft=%p\n",
|
2014-02-06 21:41:31 +07:00
|
|
|
cpu, per_cpu(hardirq_stack, cpu), per_cpu(softirq_stack, cpu));
|
2005-04-17 05:20:36 +07:00
|
|
|
}
|
|
|
|
|
2013-09-05 20:49:45 +07:00
|
|
|
void do_softirq_own_stack(void)
|
2005-04-17 05:20:36 +07:00
|
|
|
{
|
2014-02-06 21:41:31 +07:00
|
|
|
struct irq_stack *irqstk;
|
2014-02-06 21:41:30 +07:00
|
|
|
u32 *isp, *prev_esp;
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2014-02-06 21:41:31 +07:00
|
|
|
irqstk = __this_cpu_read(softirq_stack);
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2013-09-05 20:49:45 +07:00
|
|
|
/* build the stack frame on the softirq stack */
|
2014-02-06 21:41:31 +07:00
|
|
|
isp = (u32 *) ((char *)irqstk + sizeof(*irqstk));
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2014-02-06 21:41:30 +07:00
|
|
|
/* Push the previous esp onto the stack */
|
2014-02-06 21:41:31 +07:00
|
|
|
prev_esp = (u32 *)irqstk;
|
2014-11-14 06:57:07 +07:00
|
|
|
*prev_esp = current_stack_pointer();
|
2014-02-06 21:41:30 +07:00
|
|
|
|
2013-09-05 20:49:45 +07:00
|
|
|
call_on_stack(__do_softirq, isp);
|
2005-04-17 05:20:36 +07:00
|
|
|
}
|
2008-05-05 23:13:50 +07:00
|
|
|
|
2015-08-03 03:38:27 +07:00
|
|
|
bool handle_irq(struct irq_desc *desc, struct pt_regs *regs)
|
2009-02-07 05:09:40 +07:00
|
|
|
{
|
2015-09-14 15:42:37 +07:00
|
|
|
int overflow = check_stack_overflow();
|
2009-02-07 05:09:40 +07:00
|
|
|
|
2015-08-03 03:38:27 +07:00
|
|
|
if (IS_ERR_OR_NULL(desc))
|
2009-02-07 05:09:40 +07:00
|
|
|
return false;
|
|
|
|
|
2015-09-14 15:42:37 +07:00
|
|
|
if (user_mode(regs) || !execute_on_irq_stack(overflow, desc)) {
|
2009-02-07 05:09:40 +07:00
|
|
|
if (unlikely(overflow))
|
|
|
|
print_stack_overflow();
|
2015-09-14 15:42:37 +07:00
|
|
|
generic_handle_irq_desc(desc);
|
2009-02-07 05:09:40 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|