AuxXxilium/linux_dsm_epyc7002
1
0
Fork 0
mirror of https://github.com/AuxXxilium/linux_dsm_epyc7002.git synced 2025-01-19 02:27:51 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity

rps: Various optimizations

Browse Source 
Introduce ____napi_schedule() helper for callers in irq disabled
contexts. rps_trigger_softirq() becomes a leaf function.

Use container_of() in process_backlog() instead of accessing per_cpu
address.

Use a custom inlined version of __napi_complete() in process_backlog()
to avoid one locked instruction :

 only current cpu owns and manipulates this napi,
 and NAPI_STATE_SCHED is the only possible flag set on backlog.
 we can use a plain write instead of clear_bit(),
 and we dont need an smp_mb() memory barrier, since RPS is on,
 backlog is protected by a spinlock.

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Eric Dumazet 2010-05-06 22:07:48 -07:00 committed by David S. Miller
parent 39e0786d3c
commit eecfd7c4e3
1 changed files with 22 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
net/core/dev.c
View File

@ -2205,6 +2205,14 @@ int netdev_max_backlog __read_mostly = 1000;
int netdev_budget __read_mostly = 300;
int weight_p __read_mostly = 64; /* old backlog weight */
/* Called with irq disabled */
static inline void ____napi_schedule(struct softnet_data *sd,
struct napi_struct *napi)
{
list_add_tail(&napi->poll_list, &sd->poll_list);
__raise_softirq_irqoff(NET_RX_SOFTIRQ);
}
#ifdef CONFIG_RPS
/* One global table that all flow-based protocols share. */
@ -2363,7 +2371,7 @@ static void rps_trigger_softirq(void *data)
{
struct softnet_data *sd = data;
__napi_schedule(&sd->backlog);
____napi_schedule(sd, &sd->backlog);
sd->received_rps++;
}
@ -2421,7 +2429,7 @@ static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
/* Schedule NAPI for backlog device */
if (napi_schedule_prep(&sd->backlog)) {
if (!rps_ipi_queued(sd))
__napi_schedule(&sd->backlog);
____napi_schedule(sd, &sd->backlog);
}
goto enqueue;
}
@ -3280,7 +3288,7 @@ static void net_rps_action_and_irq_enable(struct softnet_data *sd)
static int process_backlog(struct napi_struct *napi, int quota)
{
int work = 0;
struct softnet_data *sd = &__get_cpu_var(softnet_data);
struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
#ifdef CONFIG_RPS
/* Check if we have pending ipi, its better to send them now,
@ -3313,7 +3321,16 @@ static int process_backlog(struct napi_struct *napi, int quota)
&sd->process_queue);
}
if (qlen < quota - work) {
__napi_complete(napi);
/*
* Inline a custom version of __napi_complete().
* only current cpu owns and manipulates this napi,
* and NAPI_STATE_SCHED is the only possible flag set on backlog.
* we can use a plain write instead of clear_bit(),
* and we dont need an smp_mb() memory barrier.
*/
list_del(&napi->poll_list);
napi->state = 0;
quota = work + qlen;
}
rps_unlock(sd);
@ -3334,8 +3351,7 @@ void __napi_schedule(struct napi_struct *n)
unsigned long flags;
local_irq_save(flags);
list_add_tail(&n->poll_list, &__get_cpu_var(softnet_data).poll_list);
__raise_softirq_irqoff(NET_RX_SOFTIRQ);
____napi_schedule(&__get_cpu_var(softnet_data), n);
local_irq_restore(flags);
}
EXPORT_SYMBOL(__napi_schedule);