AuxXxilium/linux_dsm_epyc7002
1
0
Fork 0
mirror of https://github.com/AuxXxilium/linux_dsm_epyc7002.git synced 2024-11-24 14:30:58 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity

net: use jump_label to shortcut RPS if not setup

Browse Source 
Most machines dont use RPS/RFS, and pay a fair amount of instructions in
netif_receive_skb() / netif_rx() / get_rps_cpu() just to discover
RPS/RFS is not setup.

Add a jump_label named rps_needed.

If no device rps_map or global rps_sock_flow_table is setup,
netif_receive_skb() / netif_rx() do a single instruction instead of many
ones, including conditional jumps.

jmp +0    (if CONFIG_JUMP_LABEL=y)

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
CC: Tom Herbert <therbert@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Eric Dumazet 2011-11-17 03:13:26 +00:00 committed by David S. Miller
parent d6f144830b
commit adc9300e78
4 changed files with 26 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
include/linux/netdevice.h
View File

@ -214,6 +214,11 @@ enum {
#include <linux/cache.h> #include <linux/cache.h>
#include <linux/skbuff.h> #include <linux/skbuff.h>
#ifdef CONFIG_RPS
#include <linux/jump_label.h>
extern struct jump_label_key rps_needed;
#endif
struct neighbour; struct neighbour;
struct neigh_parms; struct neigh_parms;
struct sk_buff; struct sk_buff;

21
net/core/dev.c
View File

@ -2711,6 +2711,8 @@ EXPORT_SYMBOL(__skb_get_rxhash);
struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly; struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
EXPORT_SYMBOL(rps_sock_flow_table); EXPORT_SYMBOL(rps_sock_flow_table);
struct jump_label_key rps_needed __read_mostly;
static struct rps_dev_flow * static struct rps_dev_flow *
set_rps_cpu(struct net_device *dev, struct sk_buff *skb, set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
struct rps_dev_flow *rflow, u16 next_cpu) struct rps_dev_flow *rflow, u16 next_cpu)
@ -2994,7 +2996,7 @@ int netif_rx(struct sk_buff *skb)
trace_netif_rx(skb); trace_netif_rx(skb);
#ifdef CONFIG_RPS #ifdef CONFIG_RPS
{ if (static_branch(&rps_needed)) {
struct rps_dev_flow voidflow, *rflow = &voidflow; struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu; int cpu;
@ -3009,14 +3011,13 @@ int netif_rx(struct sk_buff *skb)
rcu_read_unlock(); rcu_read_unlock();
preempt_enable(); preempt_enable();
} } else
#else #endif
{ {
unsigned int qtail; unsigned int qtail;
ret = enqueue_to_backlog(skb, get_cpu(), &qtail); ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
put_cpu(); put_cpu();
} }
#endif
return ret; return ret;
} }
EXPORT_SYMBOL(netif_rx); EXPORT_SYMBOL(netif_rx);
@ -3359,7 +3360,7 @@ int netif_receive_skb(struct sk_buff *skb)
return NET_RX_SUCCESS; return NET_RX_SUCCESS;
#ifdef CONFIG_RPS #ifdef CONFIG_RPS
{ if (static_branch(&rps_needed)) {
struct rps_dev_flow voidflow, *rflow = &voidflow; struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu, ret; int cpu, ret;
@ -3370,16 +3371,12 @@ int netif_receive_skb(struct sk_buff *skb)
if (cpu >= 0) { if (cpu >= 0) {
ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
rcu_read_unlock(); rcu_read_unlock();
} else { return ret;
rcu_read_unlock();
ret = __netif_receive_skb(skb);
} }
rcu_read_unlock();
return ret;
} }
#else
return __netif_receive_skb(skb);
#endif #endif
return __netif_receive_skb(skb);
} }
EXPORT_SYMBOL(netif_receive_skb); EXPORT_SYMBOL(netif_receive_skb);

7
net/core/net-sysfs.c
View File

@ -606,9 +606,12 @@ static ssize_t store_rps_map(struct netdev_rx_queue *queue,
rcu_assign_pointer(queue->rps_map, map); rcu_assign_pointer(queue->rps_map, map);
spin_unlock(&rps_map_lock); spin_unlock(&rps_map_lock);
if (old_map) if (map)
jump_label_inc(&rps_needed);
if (old_map) {
kfree_rcu(old_map, rcu); kfree_rcu(old_map, rcu);
jump_label_dec(&rps_needed);
}
free_cpumask_var(mask); free_cpumask_var(mask);
return len; return len;
} }

9
net/core/sysctl_net_core.c
View File

@ -68,8 +68,13 @@ static int rps_sock_flow_sysctl(ctl_table *table, int write,
if (sock_table != orig_sock_table) { if (sock_table != orig_sock_table) {
rcu_assign_pointer(rps_sock_flow_table, sock_table); rcu_assign_pointer(rps_sock_flow_table, sock_table);
synchronize_rcu(); if (sock_table)
vfree(orig_sock_table); jump_label_inc(&rps_needed);
if (orig_sock_table) {
jump_label_dec(&rps_needed);
synchronize_rcu();
vfree(orig_sock_table);
}
} }
} }