mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-03 12:46:47 +07:00
ebc0ffae5d
fib_lookup() converted to be called in RCU protected context, no reference taken and released on a contended cache line (fib_clntref) fib_table_lookup() and fib_semantic_match() get an additional parameter. struct fib_info gets an rcu_head field, and is freed after an rcu grace period. Stress test : (Sending 160.000.000 UDP frames on same neighbour, IP route cache disabled, dual E5540 @2.53GHz, 32bit kernel, FIB_HASH) (about same results for FIB_TRIE) Before patch : real 1m31.199s user 0m13.761s sys 23m24.780s After patch: real 1m5.375s user 0m14.997s sys 15m50.115s Before patch Profile : 13044.00 15.4% __ip_route_output_key vmlinux 8438.00 10.0% dst_destroy vmlinux 5983.00 7.1% fib_semantic_match vmlinux 5410.00 6.4% fib_rules_lookup vmlinux 4803.00 5.7% neigh_lookup vmlinux 4420.00 5.2% _raw_spin_lock vmlinux 3883.00 4.6% rt_set_nexthop vmlinux 3261.00 3.9% _raw_read_lock vmlinux 2794.00 3.3% fib_table_lookup vmlinux 2374.00 2.8% neigh_resolve_output vmlinux 2153.00 2.5% dst_alloc vmlinux 1502.00 1.8% _raw_read_lock_bh vmlinux 1484.00 1.8% kmem_cache_alloc vmlinux 1407.00 1.7% eth_header vmlinux 1406.00 1.7% ipv4_dst_destroy vmlinux 1298.00 1.5% __copy_from_user_ll vmlinux 1174.00 1.4% dev_queue_xmit vmlinux 1000.00 1.2% ip_output vmlinux After patch Profile : 13712.00 15.8% dst_destroy vmlinux 8548.00 9.9% __ip_route_output_key vmlinux 7017.00 8.1% neigh_lookup vmlinux 4554.00 5.3% fib_semantic_match vmlinux 4067.00 4.7% _raw_read_lock vmlinux 3491.00 4.0% dst_alloc vmlinux 3186.00 3.7% neigh_resolve_output vmlinux 3103.00 3.6% fib_table_lookup vmlinux 2098.00 2.4% _raw_read_lock_bh vmlinux 2081.00 2.4% kmem_cache_alloc vmlinux 2013.00 2.3% _raw_spin_lock vmlinux 1763.00 2.0% __copy_from_user_ll vmlinux 1763.00 2.0% ip_output vmlinux 1761.00 2.0% ipv4_dst_destroy vmlinux 1631.00 1.9% eth_header vmlinux 1440.00 1.7% _raw_read_unlock_bh vmlinux Reference results, if IP route cache is enabled : real 0m29.718s user 0m10.845s sys 7m37.341s 25213.00 29.5% __ip_route_output_key vmlinux 9011.00 10.5% dst_release vmlinux 4817.00 5.6% ip_push_pending_frames vmlinux 4232.00 5.0% ip_finish_output vmlinux 3940.00 4.6% udp_sendmsg vmlinux 3730.00 4.4% __copy_from_user_ll vmlinux 3716.00 4.4% ip_route_output_flow vmlinux 2451.00 2.9% __xfrm_lookup vmlinux 2221.00 2.6% ip_append_data vmlinux 1718.00 2.0% _raw_spin_lock_bh vmlinux 1655.00 1.9% __alloc_skb vmlinux 1572.00 1.8% sock_wfree vmlinux 1345.00 1.6% kfree vmlinux Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com> Signed-off-by: David S. Miller <davem@davemloft.net>
307 lines
6.6 KiB
C
307 lines
6.6 KiB
C
/*
|
|
* INET An implementation of the TCP/IP protocol suite for the LINUX
|
|
* operating system. INET is implemented using the BSD Socket
|
|
* interface as the means of communication with the user level.
|
|
*
|
|
* IPv4 Forwarding Information Base: policy rules.
|
|
*
|
|
* Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
|
|
* Thomas Graf <tgraf@suug.ch>
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU General Public License
|
|
* as published by the Free Software Foundation; either version
|
|
* 2 of the License, or (at your option) any later version.
|
|
*
|
|
* Fixes:
|
|
* Rani Assaf : local_rule cannot be deleted
|
|
* Marc Boucher : routing by fwmark
|
|
*/
|
|
|
|
#include <linux/types.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/netdevice.h>
|
|
#include <linux/netlink.h>
|
|
#include <linux/inetdevice.h>
|
|
#include <linux/init.h>
|
|
#include <linux/list.h>
|
|
#include <linux/rcupdate.h>
|
|
#include <net/ip.h>
|
|
#include <net/route.h>
|
|
#include <net/tcp.h>
|
|
#include <net/ip_fib.h>
|
|
#include <net/fib_rules.h>
|
|
|
|
struct fib4_rule {
|
|
struct fib_rule common;
|
|
u8 dst_len;
|
|
u8 src_len;
|
|
u8 tos;
|
|
__be32 src;
|
|
__be32 srcmask;
|
|
__be32 dst;
|
|
__be32 dstmask;
|
|
#ifdef CONFIG_NET_CLS_ROUTE
|
|
u32 tclassid;
|
|
#endif
|
|
};
|
|
|
|
#ifdef CONFIG_NET_CLS_ROUTE
|
|
u32 fib_rules_tclass(struct fib_result *res)
|
|
{
|
|
return res->r ? ((struct fib4_rule *) res->r)->tclassid : 0;
|
|
}
|
|
#endif
|
|
|
|
int fib_lookup(struct net *net, struct flowi *flp, struct fib_result *res)
|
|
{
|
|
struct fib_lookup_arg arg = {
|
|
.result = res,
|
|
.flags = FIB_LOOKUP_NOREF,
|
|
};
|
|
int err;
|
|
|
|
err = fib_rules_lookup(net->ipv4.rules_ops, flp, 0, &arg);
|
|
res->r = arg.rule;
|
|
|
|
return err;
|
|
}
|
|
|
|
static int fib4_rule_action(struct fib_rule *rule, struct flowi *flp,
|
|
int flags, struct fib_lookup_arg *arg)
|
|
{
|
|
int err = -EAGAIN;
|
|
struct fib_table *tbl;
|
|
|
|
switch (rule->action) {
|
|
case FR_ACT_TO_TBL:
|
|
break;
|
|
|
|
case FR_ACT_UNREACHABLE:
|
|
err = -ENETUNREACH;
|
|
goto errout;
|
|
|
|
case FR_ACT_PROHIBIT:
|
|
err = -EACCES;
|
|
goto errout;
|
|
|
|
case FR_ACT_BLACKHOLE:
|
|
default:
|
|
err = -EINVAL;
|
|
goto errout;
|
|
}
|
|
|
|
tbl = fib_get_table(rule->fr_net, rule->table);
|
|
if (!tbl)
|
|
goto errout;
|
|
|
|
err = fib_table_lookup(tbl, flp, (struct fib_result *) arg->result, arg->flags);
|
|
if (err > 0)
|
|
err = -EAGAIN;
|
|
errout:
|
|
return err;
|
|
}
|
|
|
|
|
|
static int fib4_rule_match(struct fib_rule *rule, struct flowi *fl, int flags)
|
|
{
|
|
struct fib4_rule *r = (struct fib4_rule *) rule;
|
|
__be32 daddr = fl->fl4_dst;
|
|
__be32 saddr = fl->fl4_src;
|
|
|
|
if (((saddr ^ r->src) & r->srcmask) ||
|
|
((daddr ^ r->dst) & r->dstmask))
|
|
return 0;
|
|
|
|
if (r->tos && (r->tos != fl->fl4_tos))
|
|
return 0;
|
|
|
|
return 1;
|
|
}
|
|
|
|
static struct fib_table *fib_empty_table(struct net *net)
|
|
{
|
|
u32 id;
|
|
|
|
for (id = 1; id <= RT_TABLE_MAX; id++)
|
|
if (fib_get_table(net, id) == NULL)
|
|
return fib_new_table(net, id);
|
|
return NULL;
|
|
}
|
|
|
|
static const struct nla_policy fib4_rule_policy[FRA_MAX+1] = {
|
|
FRA_GENERIC_POLICY,
|
|
[FRA_FLOW] = { .type = NLA_U32 },
|
|
};
|
|
|
|
static int fib4_rule_configure(struct fib_rule *rule, struct sk_buff *skb,
|
|
struct fib_rule_hdr *frh,
|
|
struct nlattr **tb)
|
|
{
|
|
struct net *net = sock_net(skb->sk);
|
|
int err = -EINVAL;
|
|
struct fib4_rule *rule4 = (struct fib4_rule *) rule;
|
|
|
|
if (frh->tos & ~IPTOS_TOS_MASK)
|
|
goto errout;
|
|
|
|
if (rule->table == RT_TABLE_UNSPEC) {
|
|
if (rule->action == FR_ACT_TO_TBL) {
|
|
struct fib_table *table;
|
|
|
|
table = fib_empty_table(net);
|
|
if (table == NULL) {
|
|
err = -ENOBUFS;
|
|
goto errout;
|
|
}
|
|
|
|
rule->table = table->tb_id;
|
|
}
|
|
}
|
|
|
|
if (frh->src_len)
|
|
rule4->src = nla_get_be32(tb[FRA_SRC]);
|
|
|
|
if (frh->dst_len)
|
|
rule4->dst = nla_get_be32(tb[FRA_DST]);
|
|
|
|
#ifdef CONFIG_NET_CLS_ROUTE
|
|
if (tb[FRA_FLOW])
|
|
rule4->tclassid = nla_get_u32(tb[FRA_FLOW]);
|
|
#endif
|
|
|
|
rule4->src_len = frh->src_len;
|
|
rule4->srcmask = inet_make_mask(rule4->src_len);
|
|
rule4->dst_len = frh->dst_len;
|
|
rule4->dstmask = inet_make_mask(rule4->dst_len);
|
|
rule4->tos = frh->tos;
|
|
|
|
err = 0;
|
|
errout:
|
|
return err;
|
|
}
|
|
|
|
static int fib4_rule_compare(struct fib_rule *rule, struct fib_rule_hdr *frh,
|
|
struct nlattr **tb)
|
|
{
|
|
struct fib4_rule *rule4 = (struct fib4_rule *) rule;
|
|
|
|
if (frh->src_len && (rule4->src_len != frh->src_len))
|
|
return 0;
|
|
|
|
if (frh->dst_len && (rule4->dst_len != frh->dst_len))
|
|
return 0;
|
|
|
|
if (frh->tos && (rule4->tos != frh->tos))
|
|
return 0;
|
|
|
|
#ifdef CONFIG_NET_CLS_ROUTE
|
|
if (tb[FRA_FLOW] && (rule4->tclassid != nla_get_u32(tb[FRA_FLOW])))
|
|
return 0;
|
|
#endif
|
|
|
|
if (frh->src_len && (rule4->src != nla_get_be32(tb[FRA_SRC])))
|
|
return 0;
|
|
|
|
if (frh->dst_len && (rule4->dst != nla_get_be32(tb[FRA_DST])))
|
|
return 0;
|
|
|
|
return 1;
|
|
}
|
|
|
|
static int fib4_rule_fill(struct fib_rule *rule, struct sk_buff *skb,
|
|
struct fib_rule_hdr *frh)
|
|
{
|
|
struct fib4_rule *rule4 = (struct fib4_rule *) rule;
|
|
|
|
frh->dst_len = rule4->dst_len;
|
|
frh->src_len = rule4->src_len;
|
|
frh->tos = rule4->tos;
|
|
|
|
if (rule4->dst_len)
|
|
NLA_PUT_BE32(skb, FRA_DST, rule4->dst);
|
|
|
|
if (rule4->src_len)
|
|
NLA_PUT_BE32(skb, FRA_SRC, rule4->src);
|
|
|
|
#ifdef CONFIG_NET_CLS_ROUTE
|
|
if (rule4->tclassid)
|
|
NLA_PUT_U32(skb, FRA_FLOW, rule4->tclassid);
|
|
#endif
|
|
return 0;
|
|
|
|
nla_put_failure:
|
|
return -ENOBUFS;
|
|
}
|
|
|
|
static size_t fib4_rule_nlmsg_payload(struct fib_rule *rule)
|
|
{
|
|
return nla_total_size(4) /* dst */
|
|
+ nla_total_size(4) /* src */
|
|
+ nla_total_size(4); /* flow */
|
|
}
|
|
|
|
static void fib4_rule_flush_cache(struct fib_rules_ops *ops)
|
|
{
|
|
rt_cache_flush(ops->fro_net, -1);
|
|
}
|
|
|
|
static const struct fib_rules_ops __net_initdata fib4_rules_ops_template = {
|
|
.family = AF_INET,
|
|
.rule_size = sizeof(struct fib4_rule),
|
|
.addr_size = sizeof(u32),
|
|
.action = fib4_rule_action,
|
|
.match = fib4_rule_match,
|
|
.configure = fib4_rule_configure,
|
|
.compare = fib4_rule_compare,
|
|
.fill = fib4_rule_fill,
|
|
.default_pref = fib_default_rule_pref,
|
|
.nlmsg_payload = fib4_rule_nlmsg_payload,
|
|
.flush_cache = fib4_rule_flush_cache,
|
|
.nlgroup = RTNLGRP_IPV4_RULE,
|
|
.policy = fib4_rule_policy,
|
|
.owner = THIS_MODULE,
|
|
};
|
|
|
|
static int fib_default_rules_init(struct fib_rules_ops *ops)
|
|
{
|
|
int err;
|
|
|
|
err = fib_default_rule_add(ops, 0, RT_TABLE_LOCAL, 0);
|
|
if (err < 0)
|
|
return err;
|
|
err = fib_default_rule_add(ops, 0x7FFE, RT_TABLE_MAIN, 0);
|
|
if (err < 0)
|
|
return err;
|
|
err = fib_default_rule_add(ops, 0x7FFF, RT_TABLE_DEFAULT, 0);
|
|
if (err < 0)
|
|
return err;
|
|
return 0;
|
|
}
|
|
|
|
int __net_init fib4_rules_init(struct net *net)
|
|
{
|
|
int err;
|
|
struct fib_rules_ops *ops;
|
|
|
|
ops = fib_rules_register(&fib4_rules_ops_template, net);
|
|
if (IS_ERR(ops))
|
|
return PTR_ERR(ops);
|
|
|
|
err = fib_default_rules_init(ops);
|
|
if (err < 0)
|
|
goto fail;
|
|
net->ipv4.rules_ops = ops;
|
|
return 0;
|
|
|
|
fail:
|
|
/* also cleans all rules already added */
|
|
fib_rules_unregister(ops);
|
|
return err;
|
|
}
|
|
|
|
void __net_exit fib4_rules_exit(struct net *net)
|
|
{
|
|
fib_rules_unregister(net->ipv4.rules_ops);
|
|
}
|