mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-11-25 20:10:50 +07:00
ef53d9c5e4
Currently list of kretprobe instances are stored in kretprobe object (as used_instances,free_instances) and in kretprobe hash table. We have one global kretprobe lock to serialise the access to these lists. This causes only one kretprobe handler to execute at a time. Hence affects system performance, particularly on SMP systems and when return probe is set on lot of functions (like on all systemcalls). Solution proposed here gives fine-grain locks that performs better on SMP system compared to present kretprobe implementation. Solution: 1) Instead of having one global lock to protect kretprobe instances present in kretprobe object and kretprobe hash table. We will have two locks, one lock for protecting kretprobe hash table and another lock for kretporbe object. 2) We hold lock present in kretprobe object while we modify kretprobe instance in kretprobe object and we hold per-hash-list lock while modifying kretprobe instances present in that hash list. To prevent deadlock, we never grab a per-hash-list lock while holding a kretprobe lock. 3) We can remove used_instances from struct kretprobe, as we can track used instances of kretprobe instances using kretprobe hash table. Time duration for kernel compilation ("make -j 8") on a 8-way ppc64 system with return probes set on all systemcalls looks like this. cacheline non-cacheline Un-patched kernel aligned patch aligned patch =============================================================================== real 9m46.784s 9m54.412s 10m2.450s user 40m5.715s 40m7.142s 40m4.273s sys 2m57.754s 2m58.583s 3m17.430s =========================================================== Time duration for kernel compilation ("make -j 8) on the same system, when kernel is not probed. ========================= real 9m26.389s user 40m8.775s sys 2m7.283s ========================= Signed-off-by: Srinivasa DS <srinivasa@in.ibm.com> Signed-off-by: Jim Keniston <jkenisto@us.ibm.com> Acked-by: Ananth N Mavinakayanahalli <ananth@in.ibm.com> Cc: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com> Cc: David S. Miller <davem@davemloft.net> Cc: Masami Hiramatsu <mhiramat@redhat.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
321 lines
9.3 KiB
C
321 lines
9.3 KiB
C
#ifndef _LINUX_KPROBES_H
|
|
#define _LINUX_KPROBES_H
|
|
/*
|
|
* Kernel Probes (KProbes)
|
|
* include/linux/kprobes.h
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program; if not, write to the Free Software
|
|
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
|
*
|
|
* Copyright (C) IBM Corporation, 2002, 2004
|
|
*
|
|
* 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
|
|
* Probes initial implementation ( includes suggestions from
|
|
* Rusty Russell).
|
|
* 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
|
|
* interface to access function arguments.
|
|
* 2005-May Hien Nguyen <hien@us.ibm.com> and Jim Keniston
|
|
* <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
|
|
* <prasanna@in.ibm.com> added function-return probes.
|
|
*/
|
|
#include <linux/list.h>
|
|
#include <linux/notifier.h>
|
|
#include <linux/smp.h>
|
|
#include <linux/percpu.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/rcupdate.h>
|
|
#include <linux/mutex.h>
|
|
|
|
#ifdef CONFIG_KPROBES
|
|
#include <asm/kprobes.h>
|
|
|
|
/* kprobe_status settings */
|
|
#define KPROBE_HIT_ACTIVE 0x00000001
|
|
#define KPROBE_HIT_SS 0x00000002
|
|
#define KPROBE_REENTER 0x00000004
|
|
#define KPROBE_HIT_SSDONE 0x00000008
|
|
|
|
/* Attach to insert probes on any functions which should be ignored*/
|
|
#define __kprobes __attribute__((__section__(".kprobes.text")))
|
|
|
|
struct kprobe;
|
|
struct pt_regs;
|
|
struct kretprobe;
|
|
struct kretprobe_instance;
|
|
typedef int (*kprobe_pre_handler_t) (struct kprobe *, struct pt_regs *);
|
|
typedef int (*kprobe_break_handler_t) (struct kprobe *, struct pt_regs *);
|
|
typedef void (*kprobe_post_handler_t) (struct kprobe *, struct pt_regs *,
|
|
unsigned long flags);
|
|
typedef int (*kprobe_fault_handler_t) (struct kprobe *, struct pt_regs *,
|
|
int trapnr);
|
|
typedef int (*kretprobe_handler_t) (struct kretprobe_instance *,
|
|
struct pt_regs *);
|
|
|
|
struct kprobe {
|
|
struct hlist_node hlist;
|
|
|
|
/* list of kprobes for multi-handler support */
|
|
struct list_head list;
|
|
|
|
/* Indicates that the corresponding module has been ref counted */
|
|
unsigned int mod_refcounted;
|
|
|
|
/*count the number of times this probe was temporarily disarmed */
|
|
unsigned long nmissed;
|
|
|
|
/* location of the probe point */
|
|
kprobe_opcode_t *addr;
|
|
|
|
/* Allow user to indicate symbol name of the probe point */
|
|
const char *symbol_name;
|
|
|
|
/* Offset into the symbol */
|
|
unsigned int offset;
|
|
|
|
/* Called before addr is executed. */
|
|
kprobe_pre_handler_t pre_handler;
|
|
|
|
/* Called after addr is executed, unless... */
|
|
kprobe_post_handler_t post_handler;
|
|
|
|
/* ... called if executing addr causes a fault (eg. page fault).
|
|
* Return 1 if it handled fault, otherwise kernel will see it. */
|
|
kprobe_fault_handler_t fault_handler;
|
|
|
|
/* ... called if breakpoint trap occurs in probe handler.
|
|
* Return 1 if it handled break, otherwise kernel will see it. */
|
|
kprobe_break_handler_t break_handler;
|
|
|
|
/* Saved opcode (which has been replaced with breakpoint) */
|
|
kprobe_opcode_t opcode;
|
|
|
|
/* copy of the original instruction */
|
|
struct arch_specific_insn ainsn;
|
|
};
|
|
|
|
/*
|
|
* Special probe type that uses setjmp-longjmp type tricks to resume
|
|
* execution at a specified entry with a matching prototype corresponding
|
|
* to the probed function - a trick to enable arguments to become
|
|
* accessible seamlessly by probe handling logic.
|
|
* Note:
|
|
* Because of the way compilers allocate stack space for local variables
|
|
* etc upfront, regardless of sub-scopes within a function, this mirroring
|
|
* principle currently works only for probes placed on function entry points.
|
|
*/
|
|
struct jprobe {
|
|
struct kprobe kp;
|
|
void *entry; /* probe handling code to jump to */
|
|
};
|
|
|
|
/* For backward compatibility with old code using JPROBE_ENTRY() */
|
|
#define JPROBE_ENTRY(handler) (handler)
|
|
|
|
DECLARE_PER_CPU(struct kprobe *, current_kprobe);
|
|
DECLARE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
|
|
|
|
#ifdef CONFIG_KRETPROBES
|
|
extern void arch_prepare_kretprobe(struct kretprobe_instance *ri,
|
|
struct pt_regs *regs);
|
|
extern int arch_trampoline_kprobe(struct kprobe *p);
|
|
#else /* CONFIG_KRETPROBES */
|
|
static inline void arch_prepare_kretprobe(struct kretprobe *rp,
|
|
struct pt_regs *regs)
|
|
{
|
|
}
|
|
static inline int arch_trampoline_kprobe(struct kprobe *p)
|
|
{
|
|
return 0;
|
|
}
|
|
#endif /* CONFIG_KRETPROBES */
|
|
/*
|
|
* Function-return probe -
|
|
* Note:
|
|
* User needs to provide a handler function, and initialize maxactive.
|
|
* maxactive - The maximum number of instances of the probed function that
|
|
* can be active concurrently.
|
|
* nmissed - tracks the number of times the probed function's return was
|
|
* ignored, due to maxactive being too low.
|
|
*
|
|
*/
|
|
struct kretprobe {
|
|
struct kprobe kp;
|
|
kretprobe_handler_t handler;
|
|
kretprobe_handler_t entry_handler;
|
|
int maxactive;
|
|
int nmissed;
|
|
size_t data_size;
|
|
struct hlist_head free_instances;
|
|
spinlock_t lock;
|
|
};
|
|
|
|
struct kretprobe_instance {
|
|
struct hlist_node hlist;
|
|
struct kretprobe *rp;
|
|
kprobe_opcode_t *ret_addr;
|
|
struct task_struct *task;
|
|
char data[0];
|
|
};
|
|
|
|
struct kretprobe_blackpoint {
|
|
const char *name;
|
|
void *addr;
|
|
};
|
|
|
|
struct kprobe_blackpoint {
|
|
const char *name;
|
|
unsigned long start_addr;
|
|
unsigned long range;
|
|
};
|
|
|
|
extern struct kretprobe_blackpoint kretprobe_blacklist[];
|
|
|
|
static inline void kretprobe_assert(struct kretprobe_instance *ri,
|
|
unsigned long orig_ret_address, unsigned long trampoline_address)
|
|
{
|
|
if (!orig_ret_address || (orig_ret_address == trampoline_address)) {
|
|
printk("kretprobe BUG!: Processing kretprobe %p @ %p\n",
|
|
ri->rp, ri->rp->kp.addr);
|
|
BUG();
|
|
}
|
|
}
|
|
|
|
#ifdef CONFIG_KPROBES_SANITY_TEST
|
|
extern int init_test_probes(void);
|
|
#else
|
|
static inline int init_test_probes(void)
|
|
{
|
|
return 0;
|
|
}
|
|
#endif /* CONFIG_KPROBES_SANITY_TEST */
|
|
|
|
extern struct mutex kprobe_mutex;
|
|
extern int arch_prepare_kprobe(struct kprobe *p);
|
|
extern void arch_arm_kprobe(struct kprobe *p);
|
|
extern void arch_disarm_kprobe(struct kprobe *p);
|
|
extern int arch_init_kprobes(void);
|
|
extern void show_registers(struct pt_regs *regs);
|
|
extern kprobe_opcode_t *get_insn_slot(void);
|
|
extern void free_insn_slot(kprobe_opcode_t *slot, int dirty);
|
|
extern void kprobes_inc_nmissed_count(struct kprobe *p);
|
|
|
|
/* Get the kprobe at this addr (if any) - called with preemption disabled */
|
|
struct kprobe *get_kprobe(void *addr);
|
|
void kretprobe_hash_lock(struct task_struct *tsk,
|
|
struct hlist_head **head, unsigned long *flags);
|
|
void kretprobe_hash_unlock(struct task_struct *tsk, unsigned long *flags);
|
|
struct hlist_head * kretprobe_inst_table_head(struct task_struct *tsk);
|
|
|
|
/* kprobe_running() will just return the current_kprobe on this CPU */
|
|
static inline struct kprobe *kprobe_running(void)
|
|
{
|
|
return (__get_cpu_var(current_kprobe));
|
|
}
|
|
|
|
static inline void reset_current_kprobe(void)
|
|
{
|
|
__get_cpu_var(current_kprobe) = NULL;
|
|
}
|
|
|
|
static inline struct kprobe_ctlblk *get_kprobe_ctlblk(void)
|
|
{
|
|
return (&__get_cpu_var(kprobe_ctlblk));
|
|
}
|
|
|
|
int register_kprobe(struct kprobe *p);
|
|
void unregister_kprobe(struct kprobe *p);
|
|
int register_kprobes(struct kprobe **kps, int num);
|
|
void unregister_kprobes(struct kprobe **kps, int num);
|
|
int setjmp_pre_handler(struct kprobe *, struct pt_regs *);
|
|
int longjmp_break_handler(struct kprobe *, struct pt_regs *);
|
|
int register_jprobe(struct jprobe *p);
|
|
void unregister_jprobe(struct jprobe *p);
|
|
int register_jprobes(struct jprobe **jps, int num);
|
|
void unregister_jprobes(struct jprobe **jps, int num);
|
|
void jprobe_return(void);
|
|
unsigned long arch_deref_entry_point(void *);
|
|
|
|
int register_kretprobe(struct kretprobe *rp);
|
|
void unregister_kretprobe(struct kretprobe *rp);
|
|
int register_kretprobes(struct kretprobe **rps, int num);
|
|
void unregister_kretprobes(struct kretprobe **rps, int num);
|
|
|
|
void kprobe_flush_task(struct task_struct *tk);
|
|
void recycle_rp_inst(struct kretprobe_instance *ri, struct hlist_head *head);
|
|
|
|
#else /* CONFIG_KPROBES */
|
|
|
|
#define __kprobes /**/
|
|
struct jprobe;
|
|
struct kretprobe;
|
|
|
|
static inline struct kprobe *get_kprobe(void *addr)
|
|
{
|
|
return NULL;
|
|
}
|
|
static inline struct kprobe *kprobe_running(void)
|
|
{
|
|
return NULL;
|
|
}
|
|
static inline int register_kprobe(struct kprobe *p)
|
|
{
|
|
return -ENOSYS;
|
|
}
|
|
static inline int register_kprobes(struct kprobe **kps, int num)
|
|
{
|
|
return -ENOSYS;
|
|
}
|
|
static inline void unregister_kprobe(struct kprobe *p)
|
|
{
|
|
}
|
|
static inline void unregister_kprobes(struct kprobe **kps, int num)
|
|
{
|
|
}
|
|
static inline int register_jprobe(struct jprobe *p)
|
|
{
|
|
return -ENOSYS;
|
|
}
|
|
static inline int register_jprobes(struct jprobe **jps, int num)
|
|
{
|
|
return -ENOSYS;
|
|
}
|
|
static inline void unregister_jprobe(struct jprobe *p)
|
|
{
|
|
}
|
|
static inline void unregister_jprobes(struct jprobe **jps, int num)
|
|
{
|
|
}
|
|
static inline void jprobe_return(void)
|
|
{
|
|
}
|
|
static inline int register_kretprobe(struct kretprobe *rp)
|
|
{
|
|
return -ENOSYS;
|
|
}
|
|
static inline int register_kretprobes(struct kretprobe **rps, int num)
|
|
{
|
|
return -ENOSYS;
|
|
}
|
|
static inline void unregister_kretprobe(struct kretprobe *rp)
|
|
{
|
|
}
|
|
static inline void unregister_kretprobes(struct kretprobe **rps, int num)
|
|
{
|
|
}
|
|
static inline void kprobe_flush_task(struct task_struct *tk)
|
|
{
|
|
}
|
|
#endif /* CONFIG_KPROBES */
|
|
#endif /* _LINUX_KPROBES_H */
|