2005-04-17 05:20:36 +07:00
|
|
|
/*
|
|
|
|
* INET An implementation of the TCP/IP protocol suite for the LINUX
|
|
|
|
* operating system. INET is implemented using the BSD Socket
|
|
|
|
* interface as the means of communication with the user level.
|
|
|
|
*
|
|
|
|
* The User Datagram Protocol (UDP).
|
|
|
|
*
|
2005-05-06 06:16:16 +07:00
|
|
|
* Authors: Ross Biro
|
2005-04-17 05:20:36 +07:00
|
|
|
* Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
|
|
|
|
* Arnt Gulbrandsen, <agulbra@nvg.unit.no>
|
2008-10-14 09:01:08 +07:00
|
|
|
* Alan Cox, <alan@lxorguk.ukuu.org.uk>
|
2005-04-17 05:20:36 +07:00
|
|
|
* Hirokazu Takahashi, <taka@valinux.co.jp>
|
|
|
|
*
|
|
|
|
* Fixes:
|
|
|
|
* Alan Cox : verify_area() calls
|
|
|
|
* Alan Cox : stopped close while in use off icmp
|
|
|
|
* messages. Not a fix but a botch that
|
|
|
|
* for udp at least is 'valid'.
|
|
|
|
* Alan Cox : Fixed icmp handling properly
|
|
|
|
* Alan Cox : Correct error for oversized datagrams
|
2007-02-09 21:24:47 +07:00
|
|
|
* Alan Cox : Tidied select() semantics.
|
|
|
|
* Alan Cox : udp_err() fixed properly, also now
|
2005-04-17 05:20:36 +07:00
|
|
|
* select and read wake correctly on errors
|
|
|
|
* Alan Cox : udp_send verify_area moved to avoid mem leak
|
|
|
|
* Alan Cox : UDP can count its memory
|
|
|
|
* Alan Cox : send to an unknown connection causes
|
|
|
|
* an ECONNREFUSED off the icmp, but
|
|
|
|
* does NOT close.
|
|
|
|
* Alan Cox : Switched to new sk_buff handlers. No more backlog!
|
|
|
|
* Alan Cox : Using generic datagram code. Even smaller and the PEEK
|
|
|
|
* bug no longer crashes it.
|
|
|
|
* Fred Van Kempen : Net2e support for sk->broadcast.
|
|
|
|
* Alan Cox : Uses skb_free_datagram
|
|
|
|
* Alan Cox : Added get/set sockopt support.
|
|
|
|
* Alan Cox : Broadcasting without option set returns EACCES.
|
|
|
|
* Alan Cox : No wakeup calls. Instead we now use the callbacks.
|
|
|
|
* Alan Cox : Use ip_tos and ip_ttl
|
|
|
|
* Alan Cox : SNMP Mibs
|
|
|
|
* Alan Cox : MSG_DONTROUTE, and 0.0.0.0 support.
|
|
|
|
* Matt Dillon : UDP length checks.
|
|
|
|
* Alan Cox : Smarter af_inet used properly.
|
|
|
|
* Alan Cox : Use new kernel side addressing.
|
|
|
|
* Alan Cox : Incorrect return on truncated datagram receive.
|
|
|
|
* Arnt Gulbrandsen : New udp_send and stuff
|
|
|
|
* Alan Cox : Cache last socket
|
|
|
|
* Alan Cox : Route cache
|
|
|
|
* Jon Peatfield : Minor efficiency fix to sendto().
|
|
|
|
* Mike Shaver : RFC1122 checks.
|
|
|
|
* Alan Cox : Nonblocking error fix.
|
|
|
|
* Willy Konynenberg : Transparent proxying support.
|
|
|
|
* Mike McLagan : Routing by source
|
|
|
|
* David S. Miller : New socket lookup architecture.
|
|
|
|
* Last socket cache retained as it
|
|
|
|
* does have a high hit rate.
|
|
|
|
* Olaf Kirch : Don't linearise iovec on sendmsg.
|
|
|
|
* Andi Kleen : Some cleanups, cache destination entry
|
2007-02-09 21:24:47 +07:00
|
|
|
* for connect.
|
2005-04-17 05:20:36 +07:00
|
|
|
* Vitaly E. Lavrov : Transparent proxy revived after year coma.
|
|
|
|
* Melvin Smith : Check msg_name not msg_namelen in sendto(),
|
|
|
|
* return ENOTCONN for unconnected sockets (POSIX)
|
|
|
|
* Janos Farkas : don't deliver multi/broadcasts to a different
|
|
|
|
* bound-to-device socket
|
|
|
|
* Hirokazu Takahashi : HW checksumming for outgoing UDP
|
|
|
|
* datagrams.
|
|
|
|
* Hirokazu Takahashi : sendfile() on UDP works now.
|
|
|
|
* Arnaldo C. Melo : convert /proc/net/udp to seq_file
|
|
|
|
* YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
|
|
|
|
* Alexey Kuznetsov: allow both IPv4 and IPv6 sockets to bind
|
|
|
|
* a single port at the same time.
|
|
|
|
* Derek Atkins <derek@ihtfp.com>: Add Encapulation Support
|
2007-06-28 05:37:46 +07:00
|
|
|
* James Chapman : Add L2TP encapsulation type.
|
2005-04-17 05:20:36 +07:00
|
|
|
*
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU General Public License
|
|
|
|
* as published by the Free Software Foundation; either version
|
|
|
|
* 2 of the License, or (at your option) any later version.
|
|
|
|
*/
|
2007-02-09 21:24:47 +07:00
|
|
|
|
2012-03-12 14:03:32 +07:00
|
|
|
#define pr_fmt(fmt) "UDP: " fmt
|
|
|
|
|
2016-12-25 02:46:01 +07:00
|
|
|
#include <linux/uaccess.h>
|
2005-04-17 05:20:36 +07:00
|
|
|
#include <asm/ioctls.h>
|
2007-12-31 15:29:24 +07:00
|
|
|
#include <linux/bootmem.h>
|
2008-10-29 16:32:32 +07:00
|
|
|
#include <linux/highmem.h>
|
|
|
|
#include <linux/swap.h>
|
2005-04-17 05:20:36 +07:00
|
|
|
#include <linux/types.h>
|
|
|
|
#include <linux/fcntl.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/socket.h>
|
|
|
|
#include <linux/sockios.h>
|
2005-12-27 11:43:12 +07:00
|
|
|
#include <linux/igmp.h>
|
2015-06-04 04:27:38 +07:00
|
|
|
#include <linux/inetdevice.h>
|
2005-04-17 05:20:36 +07:00
|
|
|
#include <linux/in.h>
|
|
|
|
#include <linux/errno.h>
|
|
|
|
#include <linux/timer.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/inet.h>
|
|
|
|
#include <linux/netdevice.h>
|
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 15:04:11 +07:00
|
|
|
#include <linux/slab.h>
|
2005-08-10 10:08:28 +07:00
|
|
|
#include <net/tcp_states.h>
|
2005-04-17 05:20:36 +07:00
|
|
|
#include <linux/skbuff.h>
|
|
|
|
#include <linux/proc_fs.h>
|
|
|
|
#include <linux/seq_file.h>
|
2007-09-12 17:01:34 +07:00
|
|
|
#include <net/net_namespace.h>
|
2005-04-17 05:20:36 +07:00
|
|
|
#include <net/icmp.h>
|
2013-10-07 23:01:39 +07:00
|
|
|
#include <net/inet_hashtables.h>
|
2005-04-17 05:20:36 +07:00
|
|
|
#include <net/route.h>
|
|
|
|
#include <net/checksum.h>
|
|
|
|
#include <net/xfrm.h>
|
2011-06-17 18:58:39 +07:00
|
|
|
#include <trace/events/udp.h>
|
2012-04-12 06:05:28 +07:00
|
|
|
#include <linux/static_key.h>
|
2012-06-27 07:23:44 +07:00
|
|
|
#include <trace/events/skb.h>
|
2013-07-10 21:13:17 +07:00
|
|
|
#include <net/busy_poll.h>
|
2006-11-28 02:10:57 +07:00
|
|
|
#include "udp_impl.h"
|
2016-01-05 05:41:46 +07:00
|
|
|
#include <net/sock_reuseport.h>
|
2016-08-18 23:59:12 +07:00
|
|
|
#include <net/addrconf.h>
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2009-10-07 07:37:59 +07:00
|
|
|
struct udp_table udp_table __read_mostly;
|
2008-10-29 15:41:45 +07:00
|
|
|
EXPORT_SYMBOL(udp_table);
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2010-11-10 06:24:26 +07:00
|
|
|
long sysctl_udp_mem[3] __read_mostly;
|
2007-12-31 15:29:24 +07:00
|
|
|
EXPORT_SYMBOL(sysctl_udp_mem);
|
2009-07-17 07:26:32 +07:00
|
|
|
|
2010-11-10 06:24:26 +07:00
|
|
|
atomic_long_t udp_memory_allocated;
|
2007-12-31 15:29:24 +07:00
|
|
|
EXPORT_SYMBOL(udp_memory_allocated);
|
|
|
|
|
2009-10-07 07:37:59 +07:00
|
|
|
#define MAX_UDP_PORTS 65536
|
|
|
|
#define PORTS_PER_CHAIN (MAX_UDP_PORTS / UDP_HTABLE_SIZE_MIN)
|
2009-01-27 12:35:35 +07:00
|
|
|
|
2017-01-27 01:02:24 +07:00
|
|
|
/* IPCB reference means this can not be used from early demux */
|
|
|
|
static bool udp_lib_exact_dif_match(struct net *net, struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
|
|
|
|
if (!net->ipv4.sysctl_udp_l3mdev_accept &&
|
|
|
|
skb && ipv4_l3mdev_skb(IPCB(skb)->flags))
|
|
|
|
return true;
|
|
|
|
#endif
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2008-10-10 04:51:27 +07:00
|
|
|
static int udp_lib_lport_inuse(struct net *net, __u16 num,
|
2008-10-29 15:41:45 +07:00
|
|
|
const struct udp_hslot *hslot,
|
2009-01-27 12:35:35 +07:00
|
|
|
unsigned long *bitmap,
|
2017-01-17 22:51:01 +07:00
|
|
|
struct sock *sk, unsigned int log)
|
2005-04-17 05:20:36 +07:00
|
|
|
{
|
2008-10-10 04:51:27 +07:00
|
|
|
struct sock *sk2;
|
2013-01-22 16:50:32 +07:00
|
|
|
kuid_t uid = sock_i_uid(sk);
|
2006-08-27 10:06:05 +07:00
|
|
|
|
2016-04-01 22:52:13 +07:00
|
|
|
sk_for_each(sk2, &hslot->head) {
|
2009-11-24 01:41:23 +07:00
|
|
|
if (net_eq(sock_net(sk2), net) &&
|
|
|
|
sk2 != sk &&
|
2009-11-08 17:17:30 +07:00
|
|
|
(bitmap || udp_sk(sk2)->udp_port_hash == num) &&
|
2009-11-24 01:41:23 +07:00
|
|
|
(!sk2->sk_reuse || !sk->sk_reuse) &&
|
|
|
|
(!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
|
|
|
|
sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
|
2017-01-17 22:51:01 +07:00
|
|
|
inet_rcv_saddr_equal(sk, sk2, true)) {
|
2017-01-06 08:22:36 +07:00
|
|
|
if (sk2->sk_reuseport && sk->sk_reuseport &&
|
|
|
|
!rcu_access_pointer(sk->sk_reuseport_cb) &&
|
|
|
|
uid_eq(uid, sock_i_uid(sk2))) {
|
|
|
|
if (!bitmap)
|
|
|
|
return 0;
|
|
|
|
} else {
|
|
|
|
if (!bitmap)
|
|
|
|
return 1;
|
|
|
|
__set_bit(udp_sk(sk2)->udp_port_hash >> log,
|
|
|
|
bitmap);
|
|
|
|
}
|
2009-01-27 12:35:35 +07:00
|
|
|
}
|
2014-11-12 12:59:20 +07:00
|
|
|
}
|
2006-08-27 10:06:05 +07:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2009-11-09 12:26:33 +07:00
|
|
|
/*
|
|
|
|
* Note: we still hold spinlock of primary hash chain, so no other writer
|
|
|
|
* can insert/delete a socket with local_port == num
|
|
|
|
*/
|
|
|
|
static int udp_lib_lport_inuse2(struct net *net, __u16 num,
|
2014-11-12 12:59:20 +07:00
|
|
|
struct udp_hslot *hslot2,
|
2017-01-17 22:51:01 +07:00
|
|
|
struct sock *sk)
|
2009-11-09 12:26:33 +07:00
|
|
|
{
|
|
|
|
struct sock *sk2;
|
2013-01-22 16:50:32 +07:00
|
|
|
kuid_t uid = sock_i_uid(sk);
|
2009-11-09 12:26:33 +07:00
|
|
|
int res = 0;
|
|
|
|
|
|
|
|
spin_lock(&hslot2->lock);
|
2016-04-01 22:52:13 +07:00
|
|
|
udp_portaddr_for_each_entry(sk2, &hslot2->head) {
|
2009-11-24 01:41:23 +07:00
|
|
|
if (net_eq(sock_net(sk2), net) &&
|
|
|
|
sk2 != sk &&
|
|
|
|
(udp_sk(sk2)->udp_port_hash == num) &&
|
|
|
|
(!sk2->sk_reuse || !sk->sk_reuse) &&
|
|
|
|
(!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
|
|
|
|
sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
|
2017-01-17 22:51:01 +07:00
|
|
|
inet_rcv_saddr_equal(sk, sk2, true)) {
|
2017-01-06 08:22:36 +07:00
|
|
|
if (sk2->sk_reuseport && sk->sk_reuseport &&
|
|
|
|
!rcu_access_pointer(sk->sk_reuseport_cb) &&
|
|
|
|
uid_eq(uid, sock_i_uid(sk2))) {
|
|
|
|
res = 0;
|
|
|
|
} else {
|
|
|
|
res = 1;
|
|
|
|
}
|
2009-11-09 12:26:33 +07:00
|
|
|
break;
|
|
|
|
}
|
2014-11-12 12:59:20 +07:00
|
|
|
}
|
2009-11-09 12:26:33 +07:00
|
|
|
spin_unlock(&hslot2->lock);
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
2017-01-17 22:51:01 +07:00
|
|
|
static int udp_reuseport_add_sock(struct sock *sk, struct udp_hslot *hslot)
|
2016-01-05 05:41:46 +07:00
|
|
|
{
|
|
|
|
struct net *net = sock_net(sk);
|
|
|
|
kuid_t uid = sock_i_uid(sk);
|
|
|
|
struct sock *sk2;
|
|
|
|
|
2016-04-01 22:52:13 +07:00
|
|
|
sk_for_each(sk2, &hslot->head) {
|
2016-01-05 05:41:46 +07:00
|
|
|
if (net_eq(sock_net(sk2), net) &&
|
|
|
|
sk2 != sk &&
|
|
|
|
sk2->sk_family == sk->sk_family &&
|
|
|
|
ipv6_only_sock(sk2) == ipv6_only_sock(sk) &&
|
|
|
|
(udp_sk(sk2)->udp_port_hash == udp_sk(sk)->udp_port_hash) &&
|
|
|
|
(sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
|
|
|
|
sk2->sk_reuseport && uid_eq(uid, sock_i_uid(sk2)) &&
|
2017-01-17 22:51:01 +07:00
|
|
|
inet_rcv_saddr_equal(sk, sk2, false)) {
|
2016-01-05 05:41:46 +07:00
|
|
|
return reuseport_add_sock(sk, sk2);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-10-20 02:00:29 +07:00
|
|
|
return reuseport_alloc(sk);
|
2016-01-05 05:41:46 +07:00
|
|
|
}
|
|
|
|
|
2006-08-27 10:06:05 +07:00
|
|
|
/**
|
2008-03-23 06:51:21 +07:00
|
|
|
* udp_lib_get_port - UDP/-Lite port lookup for IPv4 and IPv6
|
2006-08-27 10:06:05 +07:00
|
|
|
*
|
|
|
|
* @sk: socket struct in question
|
|
|
|
* @snum: port number to look up
|
2011-03-31 08:57:33 +07:00
|
|
|
* @hash2_nulladdr: AF-dependent hash value in secondary hash chains,
|
2009-11-09 12:26:33 +07:00
|
|
|
* with NULL address
|
2006-08-27 10:06:05 +07:00
|
|
|
*/
|
2008-03-23 06:51:21 +07:00
|
|
|
int udp_lib_get_port(struct sock *sk, unsigned short snum,
|
2009-11-09 12:26:33 +07:00
|
|
|
unsigned int hash2_nulladdr)
|
2006-08-27 10:06:05 +07:00
|
|
|
{
|
2009-11-08 17:17:58 +07:00
|
|
|
struct udp_hslot *hslot, *hslot2;
|
2008-10-29 15:41:45 +07:00
|
|
|
struct udp_table *udptable = sk->sk_prot->h.udp_table;
|
2006-08-27 10:06:05 +07:00
|
|
|
int error = 1;
|
2008-03-26 00:26:21 +07:00
|
|
|
struct net *net = sock_net(sk);
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2007-08-25 13:09:41 +07:00
|
|
|
if (!snum) {
|
2008-10-09 01:44:17 +07:00
|
|
|
int low, high, remaining;
|
2012-04-15 12:58:06 +07:00
|
|
|
unsigned int rand;
|
2009-01-27 12:35:35 +07:00
|
|
|
unsigned short first, last;
|
|
|
|
DECLARE_BITMAP(bitmap, PORTS_PER_CHAIN);
|
2007-08-25 13:09:41 +07:00
|
|
|
|
2013-09-29 04:10:59 +07:00
|
|
|
inet_get_local_port_range(net, &low, &high);
|
2007-10-19 12:00:17 +07:00
|
|
|
remaining = (high - low) + 1;
|
2007-10-11 07:30:46 +07:00
|
|
|
|
2014-01-11 19:15:59 +07:00
|
|
|
rand = prandom_u32();
|
2014-08-24 01:58:54 +07:00
|
|
|
first = reciprocal_scale(rand, remaining) + low;
|
2009-01-27 12:35:35 +07:00
|
|
|
/*
|
|
|
|
* force rand to be an odd multiple of UDP_HTABLE_SIZE
|
|
|
|
*/
|
2009-10-07 07:37:59 +07:00
|
|
|
rand = (rand | 1) * (udptable->mask + 1);
|
2009-12-14 10:32:39 +07:00
|
|
|
last = first + udptable->mask + 1;
|
|
|
|
do {
|
2009-10-07 07:37:59 +07:00
|
|
|
hslot = udp_hashslot(udptable, net, first);
|
2009-01-27 12:35:35 +07:00
|
|
|
bitmap_zero(bitmap, PORTS_PER_CHAIN);
|
2008-10-29 15:41:45 +07:00
|
|
|
spin_lock_bh(&hslot->lock);
|
2009-01-27 12:35:35 +07:00
|
|
|
udp_lib_lport_inuse(net, snum, hslot, bitmap, sk,
|
2017-01-17 22:51:01 +07:00
|
|
|
udptable->log);
|
2009-01-27 12:35:35 +07:00
|
|
|
|
|
|
|
snum = first;
|
|
|
|
/*
|
|
|
|
* Iterate on all possible values of snum for this hash.
|
|
|
|
* Using steps of an odd multiple of UDP_HTABLE_SIZE
|
|
|
|
* give us randomization and full range coverage.
|
|
|
|
*/
|
2008-10-09 01:44:17 +07:00
|
|
|
do {
|
2009-01-27 12:35:35 +07:00
|
|
|
if (low <= snum && snum <= high &&
|
2010-05-05 07:27:06 +07:00
|
|
|
!test_bit(snum >> udptable->log, bitmap) &&
|
2014-05-13 06:04:53 +07:00
|
|
|
!inet_is_local_reserved_port(net, snum))
|
2009-01-27 12:35:35 +07:00
|
|
|
goto found;
|
|
|
|
snum += rand;
|
|
|
|
} while (snum != first);
|
|
|
|
spin_unlock_bh(&hslot->lock);
|
2017-01-06 08:22:36 +07:00
|
|
|
cond_resched();
|
2009-12-14 10:32:39 +07:00
|
|
|
} while (++first != last);
|
2009-01-27 12:35:35 +07:00
|
|
|
goto fail;
|
2008-10-29 15:41:45 +07:00
|
|
|
} else {
|
2009-10-07 07:37:59 +07:00
|
|
|
hslot = udp_hashslot(udptable, net, snum);
|
2008-10-29 15:41:45 +07:00
|
|
|
spin_lock_bh(&hslot->lock);
|
2009-11-09 12:26:33 +07:00
|
|
|
if (hslot->count > 10) {
|
|
|
|
int exist;
|
|
|
|
unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum;
|
|
|
|
|
|
|
|
slot2 &= udptable->mask;
|
|
|
|
hash2_nulladdr &= udptable->mask;
|
|
|
|
|
|
|
|
hslot2 = udp_hashslot2(udptable, slot2);
|
|
|
|
if (hslot->count < hslot2->count)
|
|
|
|
goto scan_primary_hash;
|
|
|
|
|
2017-01-17 22:51:01 +07:00
|
|
|
exist = udp_lib_lport_inuse2(net, snum, hslot2, sk);
|
2009-11-09 12:26:33 +07:00
|
|
|
if (!exist && (hash2_nulladdr != slot2)) {
|
|
|
|
hslot2 = udp_hashslot2(udptable, hash2_nulladdr);
|
|
|
|
exist = udp_lib_lport_inuse2(net, snum, hslot2,
|
2017-01-17 22:51:01 +07:00
|
|
|
sk);
|
2009-11-09 12:26:33 +07:00
|
|
|
}
|
|
|
|
if (exist)
|
|
|
|
goto fail_unlock;
|
|
|
|
else
|
|
|
|
goto found;
|
|
|
|
}
|
|
|
|
scan_primary_hash:
|
2017-01-17 22:51:01 +07:00
|
|
|
if (udp_lib_lport_inuse(net, snum, hslot, NULL, sk, 0))
|
2008-10-29 15:41:45 +07:00
|
|
|
goto fail_unlock;
|
|
|
|
}
|
2009-01-27 12:35:35 +07:00
|
|
|
found:
|
2009-10-15 13:30:45 +07:00
|
|
|
inet_sk(sk)->inet_num = snum;
|
2009-11-08 17:17:30 +07:00
|
|
|
udp_sk(sk)->udp_port_hash = snum;
|
|
|
|
udp_sk(sk)->udp_portaddr_hash ^= snum;
|
2005-04-17 05:20:36 +07:00
|
|
|
if (sk_unhashed(sk)) {
|
2016-01-05 05:41:46 +07:00
|
|
|
if (sk->sk_reuseport &&
|
2017-01-17 22:51:01 +07:00
|
|
|
udp_reuseport_add_sock(sk, hslot)) {
|
2016-01-05 05:41:46 +07:00
|
|
|
inet_sk(sk)->inet_num = 0;
|
|
|
|
udp_sk(sk)->udp_port_hash = 0;
|
|
|
|
udp_sk(sk)->udp_portaddr_hash ^= snum;
|
|
|
|
goto fail_unlock;
|
|
|
|
}
|
|
|
|
|
2016-04-01 22:52:13 +07:00
|
|
|
sk_add_node_rcu(sk, &hslot->head);
|
2009-11-08 17:17:05 +07:00
|
|
|
hslot->count++;
|
2008-04-01 09:41:46 +07:00
|
|
|
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
|
2009-11-08 17:17:58 +07:00
|
|
|
|
|
|
|
hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
|
|
|
|
spin_lock(&hslot2->lock);
|
2016-04-13 00:11:25 +07:00
|
|
|
if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
|
2016-04-24 05:26:24 +07:00
|
|
|
sk->sk_family == AF_INET6)
|
|
|
|
hlist_add_tail_rcu(&udp_sk(sk)->udp_portaddr_node,
|
|
|
|
&hslot2->head);
|
2016-04-13 00:11:25 +07:00
|
|
|
else
|
2016-04-24 05:26:24 +07:00
|
|
|
hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
|
|
|
|
&hslot2->head);
|
2009-11-08 17:17:58 +07:00
|
|
|
hslot2->count++;
|
|
|
|
spin_unlock(&hslot2->lock);
|
2005-04-17 05:20:36 +07:00
|
|
|
}
|
2016-04-01 22:52:13 +07:00
|
|
|
sock_set_flag(sk, SOCK_RCU_FREE);
|
2006-08-27 10:06:05 +07:00
|
|
|
error = 0;
|
2008-10-29 15:41:45 +07:00
|
|
|
fail_unlock:
|
|
|
|
spin_unlock_bh(&hslot->lock);
|
2005-04-17 05:20:36 +07:00
|
|
|
fail:
|
2006-08-27 10:06:05 +07:00
|
|
|
return error;
|
|
|
|
}
|
2009-07-17 07:26:32 +07:00
|
|
|
EXPORT_SYMBOL(udp_lib_get_port);
|
2006-08-27 10:06:05 +07:00
|
|
|
|
2008-03-23 06:51:21 +07:00
|
|
|
int udp_v4_get_port(struct sock *sk, unsigned short snum)
|
2008-03-07 07:22:02 +07:00
|
|
|
{
|
2009-11-09 12:26:33 +07:00
|
|
|
unsigned int hash2_nulladdr =
|
2017-12-02 03:52:30 +07:00
|
|
|
ipv4_portaddr_hash(sock_net(sk), htonl(INADDR_ANY), snum);
|
2009-11-09 12:26:33 +07:00
|
|
|
unsigned int hash2_partial =
|
2017-12-02 03:52:30 +07:00
|
|
|
ipv4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, 0);
|
2009-11-09 12:26:33 +07:00
|
|
|
|
2009-11-08 17:17:30 +07:00
|
|
|
/* precompute partial secondary hash */
|
2009-11-09 12:26:33 +07:00
|
|
|
udp_sk(sk)->udp_portaddr_hash = hash2_partial;
|
2017-01-17 22:51:01 +07:00
|
|
|
return udp_lib_get_port(sk, snum, hash2_nulladdr);
|
2008-03-07 07:22:02 +07:00
|
|
|
}
|
|
|
|
|
udp reuseport: fix packet of same flow hashed to different socket
There is a corner case in which udp packets belonging to a same
flow are hashed to different socket when hslot->count changes from 10
to 11:
1) When hslot->count <= 10, __udp_lib_lookup() searches udp_table->hash,
and always passes 'daddr' to udp_ehashfn().
2) When hslot->count > 10, __udp_lib_lookup() searches udp_table->hash2,
but may pass 'INADDR_ANY' to udp_ehashfn() if the sockets are bound to
INADDR_ANY instead of some specific addr.
That means when hslot->count changes from 10 to 11, the hash calculated by
udp_ehashfn() is also changed, and the udp packets belonging to a same
flow will be hashed to different socket.
This is easily reproduced:
1) Create 10 udp sockets and bind all of them to 0.0.0.0:40000.
2) From the same host send udp packets to 127.0.0.1:40000, record the
socket index which receives the packets.
3) Create 1 more udp socket and bind it to 0.0.0.0:44096. The number 44096
is 40000 + UDP_HASH_SIZE(4096), this makes the new socket put into the
same hslot as the aformentioned 10 sockets, and makes the hslot->count
change from 10 to 11.
4) From the same host send udp packets to 127.0.0.1:40000, and the socket
index which receives the packets will be different from the one received
in step 2.
This should not happen as the socket bound to 0.0.0.0:44096 should not
change the behavior of the sockets bound to 0.0.0.0:40000.
It's the same case for IPv6, and this patch also fixes that.
Signed-off-by: Su, Xuemin <suxm@chinanetcenter.com>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-13 10:02:50 +07:00
|
|
|
static int compute_score(struct sock *sk, struct net *net,
|
|
|
|
__be32 saddr, __be16 sport,
|
2017-08-07 22:44:16 +07:00
|
|
|
__be32 daddr, unsigned short hnum,
|
|
|
|
int dif, int sdif, bool exact_dif)
|
2008-10-29 15:41:45 +07:00
|
|
|
{
|
2014-12-02 11:29:06 +07:00
|
|
|
int score;
|
|
|
|
struct inet_sock *inet;
|
2008-10-29 15:41:45 +07:00
|
|
|
|
2014-12-02 11:29:06 +07:00
|
|
|
if (!net_eq(sock_net(sk), net) ||
|
|
|
|
udp_sk(sk)->udp_port_hash != hnum ||
|
|
|
|
ipv6_only_sock(sk))
|
|
|
|
return -1;
|
2008-10-29 15:41:45 +07:00
|
|
|
|
2014-12-02 11:29:06 +07:00
|
|
|
score = (sk->sk_family == PF_INET) ? 2 : 1;
|
|
|
|
inet = inet_sk(sk);
|
|
|
|
|
|
|
|
if (inet->inet_rcv_saddr) {
|
|
|
|
if (inet->inet_rcv_saddr != daddr)
|
|
|
|
return -1;
|
|
|
|
score += 4;
|
2008-10-29 15:41:45 +07:00
|
|
|
}
|
2014-12-02 11:29:06 +07:00
|
|
|
|
|
|
|
if (inet->inet_daddr) {
|
|
|
|
if (inet->inet_daddr != saddr)
|
|
|
|
return -1;
|
|
|
|
score += 4;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (inet->inet_dport) {
|
|
|
|
if (inet->inet_dport != sport)
|
|
|
|
return -1;
|
|
|
|
score += 4;
|
|
|
|
}
|
|
|
|
|
2017-01-27 01:02:24 +07:00
|
|
|
if (sk->sk_bound_dev_if || exact_dif) {
|
2017-08-07 22:44:16 +07:00
|
|
|
bool dev_match = (sk->sk_bound_dev_if == dif ||
|
|
|
|
sk->sk_bound_dev_if == sdif);
|
|
|
|
|
2018-05-09 17:42:34 +07:00
|
|
|
if (!dev_match)
|
2014-12-02 11:29:06 +07:00
|
|
|
return -1;
|
2018-05-09 17:42:34 +07:00
|
|
|
if (sk->sk_bound_dev_if)
|
2017-08-07 22:44:16 +07:00
|
|
|
score += 4;
|
2014-12-02 11:29:06 +07:00
|
|
|
}
|
2017-08-07 22:44:16 +07:00
|
|
|
|
2015-10-09 09:33:21 +07:00
|
|
|
if (sk->sk_incoming_cpu == raw_smp_processor_id())
|
|
|
|
score++;
|
2008-10-29 15:41:45 +07:00
|
|
|
return score;
|
|
|
|
}
|
|
|
|
|
2015-03-19 04:05:33 +07:00
|
|
|
static u32 udp_ehashfn(const struct net *net, const __be32 laddr,
|
|
|
|
const __u16 lport, const __be32 faddr,
|
|
|
|
const __be16 fport)
|
2013-10-20 02:48:51 +07:00
|
|
|
{
|
2013-10-20 02:48:57 +07:00
|
|
|
static u32 udp_ehash_secret __read_mostly;
|
|
|
|
|
|
|
|
net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret));
|
|
|
|
|
2013-10-20 02:48:51 +07:00
|
|
|
return __inet_ehashfn(laddr, lport, faddr, fport,
|
2013-10-20 02:48:57 +07:00
|
|
|
udp_ehash_secret + net_hash_mix(net));
|
2013-10-20 02:48:51 +07:00
|
|
|
}
|
|
|
|
|
udp reuseport: fix packet of same flow hashed to different socket
There is a corner case in which udp packets belonging to a same
flow are hashed to different socket when hslot->count changes from 10
to 11:
1) When hslot->count <= 10, __udp_lib_lookup() searches udp_table->hash,
and always passes 'daddr' to udp_ehashfn().
2) When hslot->count > 10, __udp_lib_lookup() searches udp_table->hash2,
but may pass 'INADDR_ANY' to udp_ehashfn() if the sockets are bound to
INADDR_ANY instead of some specific addr.
That means when hslot->count changes from 10 to 11, the hash calculated by
udp_ehashfn() is also changed, and the udp packets belonging to a same
flow will be hashed to different socket.
This is easily reproduced:
1) Create 10 udp sockets and bind all of them to 0.0.0.0:40000.
2) From the same host send udp packets to 127.0.0.1:40000, record the
socket index which receives the packets.
3) Create 1 more udp socket and bind it to 0.0.0.0:44096. The number 44096
is 40000 + UDP_HASH_SIZE(4096), this makes the new socket put into the
same hslot as the aformentioned 10 sockets, and makes the hslot->count
change from 10 to 11.
4) From the same host send udp packets to 127.0.0.1:40000, and the socket
index which receives the packets will be different from the one received
in step 2.
This should not happen as the socket bound to 0.0.0.0:44096 should not
change the behavior of the sockets bound to 0.0.0.0:40000.
It's the same case for IPv6, and this patch also fixes that.
Signed-off-by: Su, Xuemin <suxm@chinanetcenter.com>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-13 10:02:50 +07:00
|
|
|
/* called with rcu_read_lock() */
|
ipv4: udp: optimize unicast RX path
We first locate the (local port) hash chain head
If few sockets are in this chain, we proceed with previous lookup algo.
If too many sockets are listed, we take a look at the secondary
(port, address) hash chain we added in previous patch.
We choose the shortest chain and proceed with a RCU lookup on the elected chain.
But, if we chose (port, address) chain, and fail to find a socket on given address,
we must try another lookup on (port, INADDR_ANY) chain to find socket not bound
to a particular IP.
-> No extra cost for typical setups, where the first lookup will probabbly
be performed.
RCU lookups everywhere, we dont acquire spinlock.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-11-08 17:18:11 +07:00
|
|
|
static struct sock *udp4_lib_lookup2(struct net *net,
|
2017-08-07 22:44:16 +07:00
|
|
|
__be32 saddr, __be16 sport,
|
|
|
|
__be32 daddr, unsigned int hnum,
|
|
|
|
int dif, int sdif, bool exact_dif,
|
|
|
|
struct udp_hslot *hslot2,
|
|
|
|
struct sk_buff *skb)
|
ipv4: udp: optimize unicast RX path
We first locate the (local port) hash chain head
If few sockets are in this chain, we proceed with previous lookup algo.
If too many sockets are listed, we take a look at the secondary
(port, address) hash chain we added in previous patch.
We choose the shortest chain and proceed with a RCU lookup on the elected chain.
But, if we chose (port, address) chain, and fail to find a socket on given address,
we must try another lookup on (port, INADDR_ANY) chain to find socket not bound
to a particular IP.
-> No extra cost for typical setups, where the first lookup will probabbly
be performed.
RCU lookups everywhere, we dont acquire spinlock.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-11-08 17:18:11 +07:00
|
|
|
{
|
|
|
|
struct sock *sk, *result;
|
2017-11-30 21:39:34 +07:00
|
|
|
int score, badness;
|
2013-01-22 16:50:32 +07:00
|
|
|
u32 hash = 0;
|
ipv4: udp: optimize unicast RX path
We first locate the (local port) hash chain head
If few sockets are in this chain, we proceed with previous lookup algo.
If too many sockets are listed, we take a look at the secondary
(port, address) hash chain we added in previous patch.
We choose the shortest chain and proceed with a RCU lookup on the elected chain.
But, if we chose (port, address) chain, and fail to find a socket on given address,
we must try another lookup on (port, INADDR_ANY) chain to find socket not bound
to a particular IP.
-> No extra cost for typical setups, where the first lookup will probabbly
be performed.
RCU lookups everywhere, we dont acquire spinlock.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-11-08 17:18:11 +07:00
|
|
|
|
|
|
|
result = NULL;
|
2013-01-22 16:50:32 +07:00
|
|
|
badness = 0;
|
2016-04-01 22:52:13 +07:00
|
|
|
udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
|
udp reuseport: fix packet of same flow hashed to different socket
There is a corner case in which udp packets belonging to a same
flow are hashed to different socket when hslot->count changes from 10
to 11:
1) When hslot->count <= 10, __udp_lib_lookup() searches udp_table->hash,
and always passes 'daddr' to udp_ehashfn().
2) When hslot->count > 10, __udp_lib_lookup() searches udp_table->hash2,
but may pass 'INADDR_ANY' to udp_ehashfn() if the sockets are bound to
INADDR_ANY instead of some specific addr.
That means when hslot->count changes from 10 to 11, the hash calculated by
udp_ehashfn() is also changed, and the udp packets belonging to a same
flow will be hashed to different socket.
This is easily reproduced:
1) Create 10 udp sockets and bind all of them to 0.0.0.0:40000.
2) From the same host send udp packets to 127.0.0.1:40000, record the
socket index which receives the packets.
3) Create 1 more udp socket and bind it to 0.0.0.0:44096. The number 44096
is 40000 + UDP_HASH_SIZE(4096), this makes the new socket put into the
same hslot as the aformentioned 10 sockets, and makes the hslot->count
change from 10 to 11.
4) From the same host send udp packets to 127.0.0.1:40000, and the socket
index which receives the packets will be different from the one received
in step 2.
This should not happen as the socket bound to 0.0.0.0:44096 should not
change the behavior of the sockets bound to 0.0.0.0:40000.
It's the same case for IPv6, and this patch also fixes that.
Signed-off-by: Su, Xuemin <suxm@chinanetcenter.com>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-13 10:02:50 +07:00
|
|
|
score = compute_score(sk, net, saddr, sport,
|
2017-08-07 22:44:16 +07:00
|
|
|
daddr, hnum, dif, sdif, exact_dif);
|
ipv4: udp: optimize unicast RX path
We first locate the (local port) hash chain head
If few sockets are in this chain, we proceed with previous lookup algo.
If too many sockets are listed, we take a look at the secondary
(port, address) hash chain we added in previous patch.
We choose the shortest chain and proceed with a RCU lookup on the elected chain.
But, if we chose (port, address) chain, and fail to find a socket on given address,
we must try another lookup on (port, INADDR_ANY) chain to find socket not bound
to a particular IP.
-> No extra cost for typical setups, where the first lookup will probabbly
be performed.
RCU lookups everywhere, we dont acquire spinlock.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-11-08 17:18:11 +07:00
|
|
|
if (score > badness) {
|
2017-11-30 21:39:34 +07:00
|
|
|
if (sk->sk_reuseport) {
|
2013-10-20 02:48:51 +07:00
|
|
|
hash = udp_ehashfn(net, daddr, hnum,
|
|
|
|
saddr, sport);
|
2016-04-01 22:52:13 +07:00
|
|
|
result = reuseport_select_sock(sk, hash, skb,
|
udp: fix potential infinite loop in SO_REUSEPORT logic
Using a combination of connected and un-connected sockets, Dmitry
was able to trigger soft lockups with his fuzzer.
The problem is that sockets in the SO_REUSEPORT array might have
different scores.
Right after sk2=socket(), setsockopt(sk2,...,SO_REUSEPORT, on) and
bind(sk2, ...), but _before_ the connect(sk2) is done, sk2 is added into
the soreuseport array, with a score which is smaller than the score of
first socket sk1 found in hash table (I am speaking of the regular UDP
hash table), if sk1 had the connect() done, giving a +8 to its score.
hash bucket [X] -> sk1 -> sk2 -> NULL
sk1 score = 14 (because it did a connect())
sk2 score = 6
SO_REUSEPORT fast selection is an optimization. If it turns out the
score of the selected socket does not match score of first socket, just
fallback to old SO_REUSEPORT logic instead of trying to be too smart.
Normal SO_REUSEPORT users do not mix different kind of sockets, as this
mechanism is used for load balance traffic.
Fixes: e32ea7e74727 ("soreuseport: fast reuseport UDP socket selection")
Reported-by: Dmitry Vyukov <dvyukov@google.com>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Craig Gallek <kraigatgoog@gmail.com>
Acked-by: Craig Gallek <kraig@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-01-19 23:36:43 +07:00
|
|
|
sizeof(struct udphdr));
|
2016-04-01 22:52:13 +07:00
|
|
|
if (result)
|
|
|
|
return result;
|
2013-01-22 16:50:32 +07:00
|
|
|
}
|
2016-04-01 22:52:13 +07:00
|
|
|
badness = score;
|
|
|
|
result = sk;
|
ipv4: udp: optimize unicast RX path
We first locate the (local port) hash chain head
If few sockets are in this chain, we proceed with previous lookup algo.
If too many sockets are listed, we take a look at the secondary
(port, address) hash chain we added in previous patch.
We choose the shortest chain and proceed with a RCU lookup on the elected chain.
But, if we chose (port, address) chain, and fail to find a socket on given address,
we must try another lookup on (port, INADDR_ANY) chain to find socket not bound
to a particular IP.
-> No extra cost for typical setups, where the first lookup will probabbly
be performed.
RCU lookups everywhere, we dont acquire spinlock.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-11-08 17:18:11 +07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
/* UDP is nearly always wildcards out the wazoo, it makes no sense to try
|
|
|
|
* harder than this. -DaveM
|
|
|
|
*/
|
2011-12-09 13:23:34 +07:00
|
|
|
struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
|
2017-08-07 22:44:16 +07:00
|
|
|
__be16 sport, __be32 daddr, __be16 dport, int dif,
|
|
|
|
int sdif, struct udp_table *udptable, struct sk_buff *skb)
|
2008-03-07 07:22:02 +07:00
|
|
|
{
|
udp: RCU handling for Unicast packets.
Goals are :
1) Optimizing handling of incoming Unicast UDP frames, so that no memory
writes should happen in the fast path.
Note: Multicasts and broadcasts still will need to take a lock,
because doing a full lockless lookup in this case is difficult.
2) No expensive operations in the socket bind/unhash phases :
- No expensive synchronize_rcu() calls.
- No added rcu_head in socket structure, increasing memory needs,
but more important, forcing us to use call_rcu() calls,
that have the bad property of making sockets structure cold.
(rcu grace period between socket freeing and its potential reuse
make this socket being cold in CPU cache).
David did a previous patch using call_rcu() and noticed a 20%
impact on TCP connection rates.
Quoting Cristopher Lameter :
"Right. That results in cacheline cooldown. You'd want to recycle
the object as they are cache hot on a per cpu basis. That is screwed
up by the delayed regular rcu processing. We have seen multiple
regressions due to cacheline cooldown.
The only choice in cacheline hot sensitive areas is to deal with the
complexity that comes with SLAB_DESTROY_BY_RCU or give up on RCU."
- Because udp sockets are allocated from dedicated kmem_cache,
use of SLAB_DESTROY_BY_RCU can help here.
Theory of operation :
---------------------
As the lookup is lockfree (using rcu_read_lock()/rcu_read_unlock()),
special attention must be taken by readers and writers.
Use of SLAB_DESTROY_BY_RCU is tricky too, because a socket can be freed,
reused, inserted in a different chain or in worst case in the same chain
while readers could do lookups in the same time.
In order to avoid loops, a reader must check each socket found in a chain
really belongs to the chain the reader was traversing. If it finds a
mismatch, lookup must start again at the begining. This *restart* loop
is the reason we had to use rdlock for the multicast case, because
we dont want to send same message several times to the same socket.
We use RCU only for fast path.
Thus, /proc/net/udp still takes spinlocks.
Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2008-10-29 16:11:14 +07:00
|
|
|
struct sock *sk, *result;
|
2008-03-07 07:22:02 +07:00
|
|
|
unsigned short hnum = ntohs(dport);
|
ipv4: udp: optimize unicast RX path
We first locate the (local port) hash chain head
If few sockets are in this chain, we proceed with previous lookup algo.
If too many sockets are listed, we take a look at the secondary
(port, address) hash chain we added in previous patch.
We choose the shortest chain and proceed with a RCU lookup on the elected chain.
But, if we chose (port, address) chain, and fail to find a socket on given address,
we must try another lookup on (port, INADDR_ANY) chain to find socket not bound
to a particular IP.
-> No extra cost for typical setups, where the first lookup will probabbly
be performed.
RCU lookups everywhere, we dont acquire spinlock.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-11-08 17:18:11 +07:00
|
|
|
unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask);
|
|
|
|
struct udp_hslot *hslot2, *hslot = &udptable->hash[slot];
|
2017-01-27 01:02:24 +07:00
|
|
|
bool exact_dif = udp_lib_exact_dif_match(net, skb);
|
2017-11-30 21:39:34 +07:00
|
|
|
int score, badness;
|
2013-01-22 16:50:32 +07:00
|
|
|
u32 hash = 0;
|
2008-10-29 15:41:45 +07:00
|
|
|
|
ipv4: udp: optimize unicast RX path
We first locate the (local port) hash chain head
If few sockets are in this chain, we proceed with previous lookup algo.
If too many sockets are listed, we take a look at the secondary
(port, address) hash chain we added in previous patch.
We choose the shortest chain and proceed with a RCU lookup on the elected chain.
But, if we chose (port, address) chain, and fail to find a socket on given address,
we must try another lookup on (port, INADDR_ANY) chain to find socket not bound
to a particular IP.
-> No extra cost for typical setups, where the first lookup will probabbly
be performed.
RCU lookups everywhere, we dont acquire spinlock.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-11-08 17:18:11 +07:00
|
|
|
if (hslot->count > 10) {
|
2017-12-02 03:52:30 +07:00
|
|
|
hash2 = ipv4_portaddr_hash(net, daddr, hnum);
|
ipv4: udp: optimize unicast RX path
We first locate the (local port) hash chain head
If few sockets are in this chain, we proceed with previous lookup algo.
If too many sockets are listed, we take a look at the secondary
(port, address) hash chain we added in previous patch.
We choose the shortest chain and proceed with a RCU lookup on the elected chain.
But, if we chose (port, address) chain, and fail to find a socket on given address,
we must try another lookup on (port, INADDR_ANY) chain to find socket not bound
to a particular IP.
-> No extra cost for typical setups, where the first lookup will probabbly
be performed.
RCU lookups everywhere, we dont acquire spinlock.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-11-08 17:18:11 +07:00
|
|
|
slot2 = hash2 & udptable->mask;
|
|
|
|
hslot2 = &udptable->hash2[slot2];
|
|
|
|
if (hslot->count < hslot2->count)
|
|
|
|
goto begin;
|
|
|
|
|
|
|
|
result = udp4_lib_lookup2(net, saddr, sport,
|
2017-08-07 22:44:16 +07:00
|
|
|
daddr, hnum, dif, sdif,
|
2017-01-27 01:02:24 +07:00
|
|
|
exact_dif, hslot2, skb);
|
ipv4: udp: optimize unicast RX path
We first locate the (local port) hash chain head
If few sockets are in this chain, we proceed with previous lookup algo.
If too many sockets are listed, we take a look at the secondary
(port, address) hash chain we added in previous patch.
We choose the shortest chain and proceed with a RCU lookup on the elected chain.
But, if we chose (port, address) chain, and fail to find a socket on given address,
we must try another lookup on (port, INADDR_ANY) chain to find socket not bound
to a particular IP.
-> No extra cost for typical setups, where the first lookup will probabbly
be performed.
RCU lookups everywhere, we dont acquire spinlock.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-11-08 17:18:11 +07:00
|
|
|
if (!result) {
|
udp reuseport: fix packet of same flow hashed to different socket
There is a corner case in which udp packets belonging to a same
flow are hashed to different socket when hslot->count changes from 10
to 11:
1) When hslot->count <= 10, __udp_lib_lookup() searches udp_table->hash,
and always passes 'daddr' to udp_ehashfn().
2) When hslot->count > 10, __udp_lib_lookup() searches udp_table->hash2,
but may pass 'INADDR_ANY' to udp_ehashfn() if the sockets are bound to
INADDR_ANY instead of some specific addr.
That means when hslot->count changes from 10 to 11, the hash calculated by
udp_ehashfn() is also changed, and the udp packets belonging to a same
flow will be hashed to different socket.
This is easily reproduced:
1) Create 10 udp sockets and bind all of them to 0.0.0.0:40000.
2) From the same host send udp packets to 127.0.0.1:40000, record the
socket index which receives the packets.
3) Create 1 more udp socket and bind it to 0.0.0.0:44096. The number 44096
is 40000 + UDP_HASH_SIZE(4096), this makes the new socket put into the
same hslot as the aformentioned 10 sockets, and makes the hslot->count
change from 10 to 11.
4) From the same host send udp packets to 127.0.0.1:40000, and the socket
index which receives the packets will be different from the one received
in step 2.
This should not happen as the socket bound to 0.0.0.0:44096 should not
change the behavior of the sockets bound to 0.0.0.0:40000.
It's the same case for IPv6, and this patch also fixes that.
Signed-off-by: Su, Xuemin <suxm@chinanetcenter.com>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-13 10:02:50 +07:00
|
|
|
unsigned int old_slot2 = slot2;
|
2017-12-02 03:52:30 +07:00
|
|
|
hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum);
|
ipv4: udp: optimize unicast RX path
We first locate the (local port) hash chain head
If few sockets are in this chain, we proceed with previous lookup algo.
If too many sockets are listed, we take a look at the secondary
(port, address) hash chain we added in previous patch.
We choose the shortest chain and proceed with a RCU lookup on the elected chain.
But, if we chose (port, address) chain, and fail to find a socket on given address,
we must try another lookup on (port, INADDR_ANY) chain to find socket not bound
to a particular IP.
-> No extra cost for typical setups, where the first lookup will probabbly
be performed.
RCU lookups everywhere, we dont acquire spinlock.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-11-08 17:18:11 +07:00
|
|
|
slot2 = hash2 & udptable->mask;
|
udp reuseport: fix packet of same flow hashed to different socket
There is a corner case in which udp packets belonging to a same
flow are hashed to different socket when hslot->count changes from 10
to 11:
1) When hslot->count <= 10, __udp_lib_lookup() searches udp_table->hash,
and always passes 'daddr' to udp_ehashfn().
2) When hslot->count > 10, __udp_lib_lookup() searches udp_table->hash2,
but may pass 'INADDR_ANY' to udp_ehashfn() if the sockets are bound to
INADDR_ANY instead of some specific addr.
That means when hslot->count changes from 10 to 11, the hash calculated by
udp_ehashfn() is also changed, and the udp packets belonging to a same
flow will be hashed to different socket.
This is easily reproduced:
1) Create 10 udp sockets and bind all of them to 0.0.0.0:40000.
2) From the same host send udp packets to 127.0.0.1:40000, record the
socket index which receives the packets.
3) Create 1 more udp socket and bind it to 0.0.0.0:44096. The number 44096
is 40000 + UDP_HASH_SIZE(4096), this makes the new socket put into the
same hslot as the aformentioned 10 sockets, and makes the hslot->count
change from 10 to 11.
4) From the same host send udp packets to 127.0.0.1:40000, and the socket
index which receives the packets will be different from the one received
in step 2.
This should not happen as the socket bound to 0.0.0.0:44096 should not
change the behavior of the sockets bound to 0.0.0.0:40000.
It's the same case for IPv6, and this patch also fixes that.
Signed-off-by: Su, Xuemin <suxm@chinanetcenter.com>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-13 10:02:50 +07:00
|
|
|
/* avoid searching the same slot again. */
|
|
|
|
if (unlikely(slot2 == old_slot2))
|
|
|
|
return result;
|
|
|
|
|
ipv4: udp: optimize unicast RX path
We first locate the (local port) hash chain head
If few sockets are in this chain, we proceed with previous lookup algo.
If too many sockets are listed, we take a look at the secondary
(port, address) hash chain we added in previous patch.
We choose the shortest chain and proceed with a RCU lookup on the elected chain.
But, if we chose (port, address) chain, and fail to find a socket on given address,
we must try another lookup on (port, INADDR_ANY) chain to find socket not bound
to a particular IP.
-> No extra cost for typical setups, where the first lookup will probabbly
be performed.
RCU lookups everywhere, we dont acquire spinlock.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-11-08 17:18:11 +07:00
|
|
|
hslot2 = &udptable->hash2[slot2];
|
|
|
|
if (hslot->count < hslot2->count)
|
|
|
|
goto begin;
|
|
|
|
|
2010-04-08 11:56:48 +07:00
|
|
|
result = udp4_lib_lookup2(net, saddr, sport,
|
2017-08-07 22:44:16 +07:00
|
|
|
daddr, hnum, dif, sdif,
|
2017-01-27 01:02:24 +07:00
|
|
|
exact_dif, hslot2, skb);
|
ipv4: udp: optimize unicast RX path
We first locate the (local port) hash chain head
If few sockets are in this chain, we proceed with previous lookup algo.
If too many sockets are listed, we take a look at the secondary
(port, address) hash chain we added in previous patch.
We choose the shortest chain and proceed with a RCU lookup on the elected chain.
But, if we chose (port, address) chain, and fail to find a socket on given address,
we must try another lookup on (port, INADDR_ANY) chain to find socket not bound
to a particular IP.
-> No extra cost for typical setups, where the first lookup will probabbly
be performed.
RCU lookups everywhere, we dont acquire spinlock.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-11-08 17:18:11 +07:00
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
udp: RCU handling for Unicast packets.
Goals are :
1) Optimizing handling of incoming Unicast UDP frames, so that no memory
writes should happen in the fast path.
Note: Multicasts and broadcasts still will need to take a lock,
because doing a full lockless lookup in this case is difficult.
2) No expensive operations in the socket bind/unhash phases :
- No expensive synchronize_rcu() calls.
- No added rcu_head in socket structure, increasing memory needs,
but more important, forcing us to use call_rcu() calls,
that have the bad property of making sockets structure cold.
(rcu grace period between socket freeing and its potential reuse
make this socket being cold in CPU cache).
David did a previous patch using call_rcu() and noticed a 20%
impact on TCP connection rates.
Quoting Cristopher Lameter :
"Right. That results in cacheline cooldown. You'd want to recycle
the object as they are cache hot on a per cpu basis. That is screwed
up by the delayed regular rcu processing. We have seen multiple
regressions due to cacheline cooldown.
The only choice in cacheline hot sensitive areas is to deal with the
complexity that comes with SLAB_DESTROY_BY_RCU or give up on RCU."
- Because udp sockets are allocated from dedicated kmem_cache,
use of SLAB_DESTROY_BY_RCU can help here.
Theory of operation :
---------------------
As the lookup is lockfree (using rcu_read_lock()/rcu_read_unlock()),
special attention must be taken by readers and writers.
Use of SLAB_DESTROY_BY_RCU is tricky too, because a socket can be freed,
reused, inserted in a different chain or in worst case in the same chain
while readers could do lookups in the same time.
In order to avoid loops, a reader must check each socket found in a chain
really belongs to the chain the reader was traversing. If it finds a
mismatch, lookup must start again at the begining. This *restart* loop
is the reason we had to use rdlock for the multicast case, because
we dont want to send same message several times to the same socket.
We use RCU only for fast path.
Thus, /proc/net/udp still takes spinlocks.
Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2008-10-29 16:11:14 +07:00
|
|
|
begin:
|
|
|
|
result = NULL;
|
2013-01-22 16:50:32 +07:00
|
|
|
badness = 0;
|
2016-04-01 22:52:13 +07:00
|
|
|
sk_for_each_rcu(sk, &hslot->head) {
|
udp reuseport: fix packet of same flow hashed to different socket
There is a corner case in which udp packets belonging to a same
flow are hashed to different socket when hslot->count changes from 10
to 11:
1) When hslot->count <= 10, __udp_lib_lookup() searches udp_table->hash,
and always passes 'daddr' to udp_ehashfn().
2) When hslot->count > 10, __udp_lib_lookup() searches udp_table->hash2,
but may pass 'INADDR_ANY' to udp_ehashfn() if the sockets are bound to
INADDR_ANY instead of some specific addr.
That means when hslot->count changes from 10 to 11, the hash calculated by
udp_ehashfn() is also changed, and the udp packets belonging to a same
flow will be hashed to different socket.
This is easily reproduced:
1) Create 10 udp sockets and bind all of them to 0.0.0.0:40000.
2) From the same host send udp packets to 127.0.0.1:40000, record the
socket index which receives the packets.
3) Create 1 more udp socket and bind it to 0.0.0.0:44096. The number 44096
is 40000 + UDP_HASH_SIZE(4096), this makes the new socket put into the
same hslot as the aformentioned 10 sockets, and makes the hslot->count
change from 10 to 11.
4) From the same host send udp packets to 127.0.0.1:40000, and the socket
index which receives the packets will be different from the one received
in step 2.
This should not happen as the socket bound to 0.0.0.0:44096 should not
change the behavior of the sockets bound to 0.0.0.0:40000.
It's the same case for IPv6, and this patch also fixes that.
Signed-off-by: Su, Xuemin <suxm@chinanetcenter.com>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-13 10:02:50 +07:00
|
|
|
score = compute_score(sk, net, saddr, sport,
|
2017-08-07 22:44:16 +07:00
|
|
|
daddr, hnum, dif, sdif, exact_dif);
|
2008-10-29 15:41:45 +07:00
|
|
|
if (score > badness) {
|
2017-11-30 21:39:34 +07:00
|
|
|
if (sk->sk_reuseport) {
|
2013-10-20 02:48:51 +07:00
|
|
|
hash = udp_ehashfn(net, daddr, hnum,
|
|
|
|
saddr, sport);
|
2016-04-01 22:52:13 +07:00
|
|
|
result = reuseport_select_sock(sk, hash, skb,
|
2016-01-05 05:41:47 +07:00
|
|
|
sizeof(struct udphdr));
|
2016-04-01 22:52:13 +07:00
|
|
|
if (result)
|
|
|
|
return result;
|
2013-01-22 16:50:32 +07:00
|
|
|
}
|
2016-04-01 22:52:13 +07:00
|
|
|
result = sk;
|
|
|
|
badness = score;
|
2008-03-07 07:22:02 +07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
2011-12-09 13:23:34 +07:00
|
|
|
EXPORT_SYMBOL_GPL(__udp4_lib_lookup);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
2008-10-08 02:38:32 +07:00
|
|
|
static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb,
|
|
|
|
__be16 sport, __be16 dport,
|
2008-10-29 15:41:45 +07:00
|
|
|
struct udp_table *udptable)
|
2008-10-08 02:38:32 +07:00
|
|
|
{
|
|
|
|
const struct iphdr *iph = ip_hdr(skb);
|
|
|
|
|
2016-05-13 06:23:44 +07:00
|
|
|
return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport,
|
2013-12-11 09:07:23 +07:00
|
|
|
iph->daddr, dport, inet_iif(skb),
|
2017-08-07 22:44:16 +07:00
|
|
|
inet_sdif(skb), udptable, skb);
|
2008-10-08 02:38:32 +07:00
|
|
|
}
|
|
|
|
|
2016-04-05 22:22:50 +07:00
|
|
|
struct sock *udp4_lib_lookup_skb(struct sk_buff *skb,
|
|
|
|
__be16 sport, __be16 dport)
|
|
|
|
{
|
2016-05-13 06:23:44 +07:00
|
|
|
return __udp4_lib_lookup_skb(skb, sport, dport, &udp_table);
|
2016-04-05 22:22:50 +07:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(udp4_lib_lookup_skb);
|
|
|
|
|
2016-04-01 22:52:13 +07:00
|
|
|
/* Must be called under rcu_read_lock().
|
|
|
|
* Does increment socket refcount.
|
|
|
|
*/
|
2018-06-05 18:40:34 +07:00
|
|
|
#if IS_ENABLED(CONFIG_NF_TPROXY_IPV4) || IS_ENABLED(CONFIG_NF_SOCKET_IPV4)
|
2008-10-01 21:48:10 +07:00
|
|
|
struct sock *udp4_lib_lookup(struct net *net, __be32 saddr, __be16 sport,
|
|
|
|
__be32 daddr, __be16 dport, int dif)
|
|
|
|
{
|
2016-04-01 22:52:13 +07:00
|
|
|
struct sock *sk;
|
|
|
|
|
|
|
|
sk = __udp4_lib_lookup(net, saddr, sport, daddr, dport,
|
2017-08-07 22:44:16 +07:00
|
|
|
dif, 0, &udp_table, NULL);
|
2017-06-30 17:08:01 +07:00
|
|
|
if (sk && !refcount_inc_not_zero(&sk->sk_refcnt))
|
2016-04-01 22:52:13 +07:00
|
|
|
sk = NULL;
|
|
|
|
return sk;
|
2008-10-01 21:48:10 +07:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(udp4_lib_lookup);
|
2016-04-01 22:52:13 +07:00
|
|
|
#endif
|
2008-10-01 21:48:10 +07:00
|
|
|
|
2013-10-07 23:01:39 +07:00
|
|
|
static inline bool __udp_is_mcast_sock(struct net *net, struct sock *sk,
|
|
|
|
__be16 loc_port, __be32 loc_addr,
|
|
|
|
__be16 rmt_port, __be32 rmt_addr,
|
2017-08-07 22:44:16 +07:00
|
|
|
int dif, int sdif, unsigned short hnum)
|
2013-10-07 23:01:39 +07:00
|
|
|
{
|
|
|
|
struct inet_sock *inet = inet_sk(sk);
|
|
|
|
|
|
|
|
if (!net_eq(sock_net(sk), net) ||
|
|
|
|
udp_sk(sk)->udp_port_hash != hnum ||
|
|
|
|
(inet->inet_daddr && inet->inet_daddr != rmt_addr) ||
|
|
|
|
(inet->inet_dport != rmt_port && inet->inet_dport) ||
|
|
|
|
(inet->inet_rcv_saddr && inet->inet_rcv_saddr != loc_addr) ||
|
|
|
|
ipv6_only_sock(sk) ||
|
2017-08-07 22:44:16 +07:00
|
|
|
(sk->sk_bound_dev_if && sk->sk_bound_dev_if != dif &&
|
|
|
|
sk->sk_bound_dev_if != sdif))
|
2013-10-07 23:01:39 +07:00
|
|
|
return false;
|
2017-08-07 22:44:19 +07:00
|
|
|
if (!ip_mc_sf_allow(sk, loc_addr, rmt_addr, dif, sdif))
|
2013-10-07 23:01:39 +07:00
|
|
|
return false;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
/*
|
|
|
|
* This routine is called by the ICMP module when it gets some
|
|
|
|
* sort of error condition. If err < 0 then the socket should
|
|
|
|
* be closed and the error returned to the user. If err > 0
|
|
|
|
* it's just the icmp type << 8 | icmp code.
|
|
|
|
* Header points to the ip header of the error packet. We move
|
|
|
|
* on past this. Then (as it used to claim before adjustment)
|
|
|
|
* header points to the first 8 bytes of the udp header. We need
|
|
|
|
* to find the appropriate port.
|
|
|
|
*/
|
|
|
|
|
2008-10-29 15:41:45 +07:00
|
|
|
void __udp4_lib_err(struct sk_buff *skb, u32 info, struct udp_table *udptable)
|
2008-03-07 07:22:02 +07:00
|
|
|
{
|
|
|
|
struct inet_sock *inet;
|
2011-04-22 11:53:02 +07:00
|
|
|
const struct iphdr *iph = (const struct iphdr *)skb->data;
|
2009-07-17 07:26:32 +07:00
|
|
|
struct udphdr *uh = (struct udphdr *)(skb->data+(iph->ihl<<2));
|
2008-03-07 07:22:02 +07:00
|
|
|
const int type = icmp_hdr(skb)->type;
|
|
|
|
const int code = icmp_hdr(skb)->code;
|
|
|
|
struct sock *sk;
|
|
|
|
int harderr;
|
|
|
|
int err;
|
2008-07-15 13:01:40 +07:00
|
|
|
struct net *net = dev_net(skb->dev);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
2008-07-15 13:01:40 +07:00
|
|
|
sk = __udp4_lib_lookup(net, iph->daddr, uh->dest,
|
2017-08-07 22:44:16 +07:00
|
|
|
iph->saddr, uh->source, skb->dev->ifindex, 0,
|
|
|
|
udptable, NULL);
|
2015-04-03 15:17:26 +07:00
|
|
|
if (!sk) {
|
2016-04-28 06:44:29 +07:00
|
|
|
__ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
|
2008-03-07 07:22:02 +07:00
|
|
|
return; /* No socket for error */
|
|
|
|
}
|
|
|
|
|
|
|
|
err = 0;
|
|
|
|
harderr = 0;
|
|
|
|
inet = inet_sk(sk);
|
|
|
|
|
|
|
|
switch (type) {
|
|
|
|
default:
|
|
|
|
case ICMP_TIME_EXCEEDED:
|
|
|
|
err = EHOSTUNREACH;
|
|
|
|
break;
|
|
|
|
case ICMP_SOURCE_QUENCH:
|
|
|
|
goto out;
|
|
|
|
case ICMP_PARAMETERPROB:
|
|
|
|
err = EPROTO;
|
|
|
|
harderr = 1;
|
|
|
|
break;
|
|
|
|
case ICMP_DEST_UNREACH:
|
|
|
|
if (code == ICMP_FRAG_NEEDED) { /* Path MTU discovery */
|
2012-06-15 12:21:46 +07:00
|
|
|
ipv4_sk_update_pmtu(skb, sk, info);
|
2008-03-07 07:22:02 +07:00
|
|
|
if (inet->pmtudisc != IP_PMTUDISC_DONT) {
|
|
|
|
err = EMSGSIZE;
|
|
|
|
harderr = 1;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
err = EHOSTUNREACH;
|
|
|
|
if (code <= NR_ICMP_UNREACH) {
|
|
|
|
harderr = icmp_err_convert[code].fatal;
|
|
|
|
err = icmp_err_convert[code].errno;
|
|
|
|
}
|
|
|
|
break;
|
2012-07-12 11:27:49 +07:00
|
|
|
case ICMP_REDIRECT:
|
|
|
|
ipv4_sk_redirect(skb, sk);
|
2013-09-20 17:20:28 +07:00
|
|
|
goto out;
|
2008-03-07 07:22:02 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* RFC1122: OK. Passes ICMP errors back to application, as per
|
|
|
|
* 4.1.3.3.
|
|
|
|
*/
|
|
|
|
if (!inet->recverr) {
|
|
|
|
if (!harderr || sk->sk_state != TCP_ESTABLISHED)
|
|
|
|
goto out;
|
2010-06-01 13:44:05 +07:00
|
|
|
} else
|
2009-07-17 07:26:32 +07:00
|
|
|
ip_icmp_error(sk, skb, err, uh->dest, info, (u8 *)(uh+1));
|
2010-06-01 13:44:05 +07:00
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
sk->sk_err = err;
|
|
|
|
sk->sk_error_report(sk);
|
|
|
|
out:
|
2016-04-01 22:52:13 +07:00
|
|
|
return;
|
2008-03-07 07:22:02 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
void udp_err(struct sk_buff *skb, u32 info)
|
|
|
|
{
|
2008-10-29 15:41:45 +07:00
|
|
|
__udp4_lib_err(skb, info, &udp_table);
|
2008-03-07 07:22:02 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Throw away all pending data and cancel the corking. Socket is locked.
|
|
|
|
*/
|
2008-06-04 18:49:07 +07:00
|
|
|
void udp_flush_pending_frames(struct sock *sk)
|
2008-03-07 07:22:02 +07:00
|
|
|
{
|
|
|
|
struct udp_sock *up = udp_sk(sk);
|
|
|
|
|
|
|
|
if (up->pending) {
|
|
|
|
up->len = 0;
|
|
|
|
up->pending = 0;
|
|
|
|
ip_flush_pending_frames(sk);
|
|
|
|
}
|
|
|
|
}
|
2008-06-04 18:49:07 +07:00
|
|
|
EXPORT_SYMBOL(udp_flush_pending_frames);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
|
|
|
/**
|
2011-03-01 09:36:48 +07:00
|
|
|
* udp4_hwcsum - handle outgoing HW checksumming
|
2008-03-07 07:22:02 +07:00
|
|
|
* @skb: sk_buff containing the filled-in UDP header
|
|
|
|
* (checksum field must be zeroed out)
|
2011-03-01 09:36:48 +07:00
|
|
|
* @src: source IP address
|
|
|
|
* @dst: destination IP address
|
2008-03-07 07:22:02 +07:00
|
|
|
*/
|
2013-07-25 23:12:18 +07:00
|
|
|
void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst)
|
2008-03-07 07:22:02 +07:00
|
|
|
{
|
|
|
|
struct udphdr *uh = udp_hdr(skb);
|
2011-03-01 09:36:48 +07:00
|
|
|
int offset = skb_transport_offset(skb);
|
|
|
|
int len = skb->len - offset;
|
|
|
|
int hlen = len;
|
2008-03-07 07:22:02 +07:00
|
|
|
__wsum csum = 0;
|
|
|
|
|
2014-06-03 06:12:02 +07:00
|
|
|
if (!skb_has_frag_list(skb)) {
|
2008-03-07 07:22:02 +07:00
|
|
|
/*
|
|
|
|
* Only one fragment on the socket.
|
|
|
|
*/
|
|
|
|
skb->csum_start = skb_transport_header(skb) - skb->head;
|
|
|
|
skb->csum_offset = offsetof(struct udphdr, check);
|
2011-03-01 09:36:48 +07:00
|
|
|
uh->check = ~csum_tcpudp_magic(src, dst, len,
|
|
|
|
IPPROTO_UDP, 0);
|
2008-03-07 07:22:02 +07:00
|
|
|
} else {
|
2014-06-03 06:12:02 +07:00
|
|
|
struct sk_buff *frags;
|
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
/*
|
|
|
|
* HW-checksum won't work as there are two or more
|
|
|
|
* fragments on the socket so that all csums of sk_buffs
|
|
|
|
* should be together
|
|
|
|
*/
|
2014-06-03 06:12:02 +07:00
|
|
|
skb_walk_frags(skb, frags) {
|
2011-03-01 09:36:48 +07:00
|
|
|
csum = csum_add(csum, frags->csum);
|
|
|
|
hlen -= frags->len;
|
2014-06-03 06:12:02 +07:00
|
|
|
}
|
2008-03-07 07:22:02 +07:00
|
|
|
|
2011-03-01 09:36:48 +07:00
|
|
|
csum = skb_checksum(skb, offset, hlen, csum);
|
2008-03-07 07:22:02 +07:00
|
|
|
skb->ip_summed = CHECKSUM_NONE;
|
|
|
|
|
|
|
|
uh->check = csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, csum);
|
|
|
|
if (uh->check == 0)
|
|
|
|
uh->check = CSUM_MANGLED_0;
|
|
|
|
}
|
|
|
|
}
|
2013-07-25 23:12:18 +07:00
|
|
|
EXPORT_SYMBOL_GPL(udp4_hwcsum);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
2014-06-05 07:19:48 +07:00
|
|
|
/* Function to set UDP checksum for an IPv4 UDP packet. This is intended
|
|
|
|
* for the simple case like when setting the checksum for a UDP tunnel.
|
|
|
|
*/
|
|
|
|
void udp_set_csum(bool nocheck, struct sk_buff *skb,
|
|
|
|
__be32 saddr, __be32 daddr, int len)
|
|
|
|
{
|
|
|
|
struct udphdr *uh = udp_hdr(skb);
|
|
|
|
|
2016-02-12 03:48:04 +07:00
|
|
|
if (nocheck) {
|
2014-06-05 07:19:48 +07:00
|
|
|
uh->check = 0;
|
2016-02-12 03:48:04 +07:00
|
|
|
} else if (skb_is_gso(skb)) {
|
2014-06-05 07:19:48 +07:00
|
|
|
uh->check = ~udp_v4_check(len, saddr, daddr, 0);
|
2016-02-12 03:48:04 +07:00
|
|
|
} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
|
|
|
|
uh->check = 0;
|
|
|
|
uh->check = udp_v4_check(len, saddr, daddr, lco_csum(skb));
|
|
|
|
if (uh->check == 0)
|
|
|
|
uh->check = CSUM_MANGLED_0;
|
2016-02-12 03:49:40 +07:00
|
|
|
} else {
|
2014-06-05 07:19:48 +07:00
|
|
|
skb->ip_summed = CHECKSUM_PARTIAL;
|
|
|
|
skb->csum_start = skb_transport_header(skb) - skb->head;
|
|
|
|
skb->csum_offset = offsetof(struct udphdr, check);
|
|
|
|
uh->check = ~udp_v4_check(len, saddr, daddr, 0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(udp_set_csum);
|
|
|
|
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-27 00:42:17 +07:00
|
|
|
static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4,
|
|
|
|
struct inet_cork *cork)
|
2008-03-07 07:22:02 +07:00
|
|
|
{
|
2011-03-01 09:36:48 +07:00
|
|
|
struct sock *sk = skb->sk;
|
2008-03-07 07:22:02 +07:00
|
|
|
struct inet_sock *inet = inet_sk(sk);
|
|
|
|
struct udphdr *uh;
|
|
|
|
int err = 0;
|
|
|
|
int is_udplite = IS_UDPLITE(sk);
|
2011-03-01 09:36:48 +07:00
|
|
|
int offset = skb_transport_offset(skb);
|
|
|
|
int len = skb->len - offset;
|
2008-03-07 07:22:02 +07:00
|
|
|
__wsum csum = 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Create a UDP header
|
|
|
|
*/
|
|
|
|
uh = udp_hdr(skb);
|
2011-03-01 09:36:48 +07:00
|
|
|
uh->source = inet->inet_sport;
|
2011-05-10 03:31:04 +07:00
|
|
|
uh->dest = fl4->fl4_dport;
|
2011-03-01 09:36:48 +07:00
|
|
|
uh->len = htons(len);
|
2008-03-07 07:22:02 +07:00
|
|
|
uh->check = 0;
|
|
|
|
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-27 00:42:17 +07:00
|
|
|
if (cork->gso_size) {
|
|
|
|
const int hlen = skb_network_header_len(skb) +
|
|
|
|
sizeof(struct udphdr);
|
|
|
|
|
|
|
|
if (hlen + cork->gso_size > cork->fragsize)
|
|
|
|
return -EINVAL;
|
|
|
|
if (skb->len > cork->gso_size * UDP_MAX_SEGMENTS)
|
|
|
|
return -EINVAL;
|
2018-05-01 02:58:36 +07:00
|
|
|
if (sk->sk_no_check_tx)
|
|
|
|
return -EINVAL;
|
2018-05-22 22:34:39 +07:00
|
|
|
if (skb->ip_summed != CHECKSUM_PARTIAL || is_udplite ||
|
|
|
|
dst_xfrm(skb_dst(skb)))
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-27 00:42:17 +07:00
|
|
|
return -EIO;
|
|
|
|
|
|
|
|
skb_shinfo(skb)->gso_size = cork->gso_size;
|
|
|
|
skb_shinfo(skb)->gso_type = SKB_GSO_UDP_L4;
|
2018-05-08 01:08:22 +07:00
|
|
|
skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(len - sizeof(uh),
|
|
|
|
cork->gso_size);
|
2018-05-01 02:58:36 +07:00
|
|
|
goto csum_partial;
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-27 00:42:17 +07:00
|
|
|
}
|
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
if (is_udplite) /* UDP-Lite */
|
2011-03-01 09:36:48 +07:00
|
|
|
csum = udplite_csum(skb);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
2017-08-22 22:39:57 +07:00
|
|
|
else if (sk->sk_no_check_tx) { /* UDP csum off */
|
2008-03-07 07:22:02 +07:00
|
|
|
|
|
|
|
skb->ip_summed = CHECKSUM_NONE;
|
|
|
|
goto send;
|
|
|
|
|
|
|
|
} else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */
|
2018-05-01 02:58:36 +07:00
|
|
|
csum_partial:
|
2008-03-07 07:22:02 +07:00
|
|
|
|
2011-05-10 03:31:04 +07:00
|
|
|
udp4_hwcsum(skb, fl4->saddr, fl4->daddr);
|
2008-03-07 07:22:02 +07:00
|
|
|
goto send;
|
|
|
|
|
2011-03-01 09:36:48 +07:00
|
|
|
} else
|
|
|
|
csum = udp_csum(skb);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
|
|
|
/* add protocol-dependent pseudo-header */
|
2011-05-10 03:31:04 +07:00
|
|
|
uh->check = csum_tcpudp_magic(fl4->saddr, fl4->daddr, len,
|
2009-07-17 07:26:32 +07:00
|
|
|
sk->sk_protocol, csum);
|
2008-03-07 07:22:02 +07:00
|
|
|
if (uh->check == 0)
|
|
|
|
uh->check = CSUM_MANGLED_0;
|
|
|
|
|
|
|
|
send:
|
2012-08-10 09:22:47 +07:00
|
|
|
err = ip_send_skb(sock_net(sk), skb);
|
ip: Report qdisc packet drops
Christoph Lameter pointed out that packet drops at qdisc level where not
accounted in SNMP counters. Only if application sets IP_RECVERR, drops
are reported to user (-ENOBUFS errors) and SNMP counters updated.
IP_RECVERR is used to enable extended reliable error message passing,
but these are not needed to update system wide SNMP stats.
This patch changes things a bit to allow SNMP counters to be updated,
regardless of IP_RECVERR being set or not on the socket.
Example after an UDP tx flood
# netstat -s
...
IP:
1487048 outgoing packets dropped
...
Udp:
...
SndbufErrors: 1487048
send() syscalls, do however still return an OK status, to not
break applications.
Note : send() manual page explicitly says for -ENOBUFS error :
"The output queue for a network interface was full.
This generally indicates that the interface has stopped sending,
but may be caused by transient congestion.
(Normally, this does not occur in Linux. Packets are just silently
dropped when a device queue overflows.) "
This is not true for IP_RECVERR enabled sockets : a send() syscall
that hit a qdisc drop returns an ENOBUFS error.
Many thanks to Christoph, David, and last but not least, Alexey !
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-09-03 08:05:33 +07:00
|
|
|
if (err) {
|
|
|
|
if (err == -ENOBUFS && !inet->recverr) {
|
net: snmp: kill various STATS_USER() helpers
In the old days (before linux-3.0), SNMP counters were duplicated,
one for user context, and one for BH context.
After commit 8f0ea0fe3a03 ("snmp: reduce percpu needs by 50%")
we have a single copy, and what really matters is preemption being
enabled or disabled, since we use this_cpu_inc() or __this_cpu_inc()
respectively.
We therefore kill SNMP_INC_STATS_USER(), SNMP_ADD_STATS_USER(),
NET_INC_STATS_USER(), NET_ADD_STATS_USER(), SCTP_INC_STATS_USER(),
SNMP_INC_STATS64_USER(), SNMP_ADD_STATS64_USER(), TCP_ADD_STATS_USER(),
UDP_INC_STATS_USER(), UDP6_INC_STATS_USER(), and XFRM_INC_STATS_USER()
Following patches will rename __BH helpers to make clear their
usage is not tied to BH being disabled.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-04-28 06:44:27 +07:00
|
|
|
UDP_INC_STATS(sock_net(sk),
|
|
|
|
UDP_MIB_SNDBUFERRORS, is_udplite);
|
ip: Report qdisc packet drops
Christoph Lameter pointed out that packet drops at qdisc level where not
accounted in SNMP counters. Only if application sets IP_RECVERR, drops
are reported to user (-ENOBUFS errors) and SNMP counters updated.
IP_RECVERR is used to enable extended reliable error message passing,
but these are not needed to update system wide SNMP stats.
This patch changes things a bit to allow SNMP counters to be updated,
regardless of IP_RECVERR being set or not on the socket.
Example after an UDP tx flood
# netstat -s
...
IP:
1487048 outgoing packets dropped
...
Udp:
...
SndbufErrors: 1487048
send() syscalls, do however still return an OK status, to not
break applications.
Note : send() manual page explicitly says for -ENOBUFS error :
"The output queue for a network interface was full.
This generally indicates that the interface has stopped sending,
but may be caused by transient congestion.
(Normally, this does not occur in Linux. Packets are just silently
dropped when a device queue overflows.) "
This is not true for IP_RECVERR enabled sockets : a send() syscall
that hit a qdisc drop returns an ENOBUFS error.
Many thanks to Christoph, David, and last but not least, Alexey !
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-09-03 08:05:33 +07:00
|
|
|
err = 0;
|
|
|
|
}
|
|
|
|
} else
|
net: snmp: kill various STATS_USER() helpers
In the old days (before linux-3.0), SNMP counters were duplicated,
one for user context, and one for BH context.
After commit 8f0ea0fe3a03 ("snmp: reduce percpu needs by 50%")
we have a single copy, and what really matters is preemption being
enabled or disabled, since we use this_cpu_inc() or __this_cpu_inc()
respectively.
We therefore kill SNMP_INC_STATS_USER(), SNMP_ADD_STATS_USER(),
NET_INC_STATS_USER(), NET_ADD_STATS_USER(), SCTP_INC_STATS_USER(),
SNMP_INC_STATS64_USER(), SNMP_ADD_STATS64_USER(), TCP_ADD_STATS_USER(),
UDP_INC_STATS_USER(), UDP6_INC_STATS_USER(), and XFRM_INC_STATS_USER()
Following patches will rename __BH helpers to make clear their
usage is not tied to BH being disabled.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-04-28 06:44:27 +07:00
|
|
|
UDP_INC_STATS(sock_net(sk),
|
|
|
|
UDP_MIB_OUTDATAGRAMS, is_udplite);
|
2011-03-01 09:36:48 +07:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Push out all pending data as one UDP datagram. Socket is locked.
|
|
|
|
*/
|
2013-07-02 01:21:30 +07:00
|
|
|
int udp_push_pending_frames(struct sock *sk)
|
2011-03-01 09:36:48 +07:00
|
|
|
{
|
|
|
|
struct udp_sock *up = udp_sk(sk);
|
|
|
|
struct inet_sock *inet = inet_sk(sk);
|
2011-03-12 14:09:18 +07:00
|
|
|
struct flowi4 *fl4 = &inet->cork.fl.u.ip4;
|
2011-03-01 09:36:48 +07:00
|
|
|
struct sk_buff *skb;
|
|
|
|
int err = 0;
|
|
|
|
|
2011-05-09 07:12:19 +07:00
|
|
|
skb = ip_finish_skb(sk, fl4);
|
2011-03-01 09:36:48 +07:00
|
|
|
if (!skb)
|
|
|
|
goto out;
|
|
|
|
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-27 00:42:17 +07:00
|
|
|
err = udp_send_skb(skb, fl4, &inet->cork.base);
|
2011-03-01 09:36:48 +07:00
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
out:
|
|
|
|
up->len = 0;
|
|
|
|
up->pending = 0;
|
|
|
|
return err;
|
|
|
|
}
|
2013-07-02 01:21:30 +07:00
|
|
|
EXPORT_SYMBOL(udp_push_pending_frames);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
2018-04-27 00:42:20 +07:00
|
|
|
static int __udp_cmsg_send(struct cmsghdr *cmsg, u16 *gso_size)
|
|
|
|
{
|
|
|
|
switch (cmsg->cmsg_type) {
|
|
|
|
case UDP_SEGMENT:
|
|
|
|
if (cmsg->cmsg_len != CMSG_LEN(sizeof(__u16)))
|
|
|
|
return -EINVAL;
|
|
|
|
*gso_size = *(__u16 *)CMSG_DATA(cmsg);
|
|
|
|
return 0;
|
|
|
|
default:
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
int udp_cmsg_send(struct sock *sk, struct msghdr *msg, u16 *gso_size)
|
|
|
|
{
|
|
|
|
struct cmsghdr *cmsg;
|
|
|
|
bool need_ip = false;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
for_each_cmsghdr(cmsg, msg) {
|
|
|
|
if (!CMSG_OK(msg, cmsg))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (cmsg->cmsg_level != SOL_UDP) {
|
|
|
|
need_ip = true;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
err = __udp_cmsg_send(cmsg, gso_size);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
return need_ip;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(udp_cmsg_send);
|
|
|
|
|
2015-03-02 14:37:48 +07:00
|
|
|
int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
|
2008-03-07 07:22:02 +07:00
|
|
|
{
|
|
|
|
struct inet_sock *inet = inet_sk(sk);
|
|
|
|
struct udp_sock *up = udp_sk(sk);
|
2018-05-25 22:55:23 +07:00
|
|
|
DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name);
|
2011-05-09 06:38:45 +07:00
|
|
|
struct flowi4 fl4_stack;
|
2011-03-12 14:09:18 +07:00
|
|
|
struct flowi4 *fl4;
|
2008-03-07 07:22:02 +07:00
|
|
|
int ulen = len;
|
|
|
|
struct ipcm_cookie ipc;
|
|
|
|
struct rtable *rt = NULL;
|
|
|
|
int free = 0;
|
|
|
|
int connected = 0;
|
|
|
|
__be32 daddr, faddr, saddr;
|
|
|
|
__be16 dport;
|
|
|
|
u8 tos;
|
|
|
|
int err, is_udplite = IS_UDPLITE(sk);
|
|
|
|
int corkreq = up->corkflag || msg->msg_flags&MSG_MORE;
|
|
|
|
int (*getfrag)(void *, char *, int, int, int, struct sk_buff *);
|
2011-03-01 09:36:48 +07:00
|
|
|
struct sk_buff *skb;
|
2011-04-21 16:45:37 +07:00
|
|
|
struct ip_options_data opt_copy;
|
2008-03-07 07:22:02 +07:00
|
|
|
|
|
|
|
if (len > 0xFFFF)
|
|
|
|
return -EMSGSIZE;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Check the flags.
|
|
|
|
*/
|
|
|
|
|
2009-07-17 07:26:32 +07:00
|
|
|
if (msg->msg_flags & MSG_OOB) /* Mirror BSD error message compatibility */
|
2008-03-07 07:22:02 +07:00
|
|
|
return -EOPNOTSUPP;
|
|
|
|
|
2011-03-01 09:36:48 +07:00
|
|
|
getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag;
|
|
|
|
|
2011-05-09 07:24:10 +07:00
|
|
|
fl4 = &inet->cork.fl.u.ip4;
|
2008-03-07 07:22:02 +07:00
|
|
|
if (up->pending) {
|
|
|
|
/*
|
|
|
|
* There are pending frames.
|
|
|
|
* The socket lock must be held while it's corked.
|
|
|
|
*/
|
|
|
|
lock_sock(sk);
|
|
|
|
if (likely(up->pending)) {
|
|
|
|
if (unlikely(up->pending != AF_INET)) {
|
|
|
|
release_sock(sk);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
goto do_append_data;
|
|
|
|
}
|
|
|
|
release_sock(sk);
|
|
|
|
}
|
|
|
|
ulen += sizeof(struct udphdr);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get and verify the address.
|
|
|
|
*/
|
2018-05-25 22:55:23 +07:00
|
|
|
if (usin) {
|
2008-03-07 07:22:02 +07:00
|
|
|
if (msg->msg_namelen < sizeof(*usin))
|
|
|
|
return -EINVAL;
|
|
|
|
if (usin->sin_family != AF_INET) {
|
|
|
|
if (usin->sin_family != AF_UNSPEC)
|
|
|
|
return -EAFNOSUPPORT;
|
|
|
|
}
|
|
|
|
|
|
|
|
daddr = usin->sin_addr.s_addr;
|
|
|
|
dport = usin->sin_port;
|
|
|
|
if (dport == 0)
|
|
|
|
return -EINVAL;
|
|
|
|
} else {
|
|
|
|
if (sk->sk_state != TCP_ESTABLISHED)
|
|
|
|
return -EDESTADDRREQ;
|
2009-10-15 13:30:45 +07:00
|
|
|
daddr = inet->inet_daddr;
|
|
|
|
dport = inet->inet_dport;
|
2008-03-07 07:22:02 +07:00
|
|
|
/* Open fast path for connected socket.
|
|
|
|
Route will not be used, if at least one option is set.
|
|
|
|
*/
|
|
|
|
connected = 1;
|
|
|
|
}
|
|
|
|
|
2018-07-06 21:12:54 +07:00
|
|
|
ipcm_init_sk(&ipc, inet);
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-27 00:42:17 +07:00
|
|
|
ipc.gso_size = up->gso_size;
|
2013-04-14 15:08:13 +07:00
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
if (msg->msg_controllen) {
|
2018-04-27 00:42:20 +07:00
|
|
|
err = udp_cmsg_send(sk, msg, &ipc.gso_size);
|
|
|
|
if (err > 0)
|
|
|
|
err = ip_cmsg_send(sk, msg, &ipc,
|
|
|
|
sk->sk_family == AF_INET6);
|
|
|
|
if (unlikely(err < 0)) {
|
2016-02-04 21:23:28 +07:00
|
|
|
kfree(ipc.opt);
|
2008-03-07 07:22:02 +07:00
|
|
|
return err;
|
2016-02-04 21:23:28 +07:00
|
|
|
}
|
2008-03-07 07:22:02 +07:00
|
|
|
if (ipc.opt)
|
|
|
|
free = 1;
|
|
|
|
connected = 0;
|
|
|
|
}
|
2011-04-21 16:45:37 +07:00
|
|
|
if (!ipc.opt) {
|
|
|
|
struct ip_options_rcu *inet_opt;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
inet_opt = rcu_dereference(inet->inet_opt);
|
|
|
|
if (inet_opt) {
|
|
|
|
memcpy(&opt_copy, inet_opt,
|
|
|
|
sizeof(*inet_opt) + inet_opt->opt.optlen);
|
|
|
|
ipc.opt = &opt_copy.opt;
|
|
|
|
}
|
|
|
|
rcu_read_unlock();
|
|
|
|
}
|
2008-03-07 07:22:02 +07:00
|
|
|
|
2018-05-25 22:55:23 +07:00
|
|
|
if (cgroup_bpf_enabled && !connected) {
|
|
|
|
err = BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk,
|
|
|
|
(struct sockaddr *)usin, &ipc.addr);
|
|
|
|
if (err)
|
|
|
|
goto out_free;
|
|
|
|
if (usin) {
|
|
|
|
if (usin->sin_port == 0) {
|
|
|
|
/* BPF program set invalid port. Reject it. */
|
|
|
|
err = -EINVAL;
|
|
|
|
goto out_free;
|
|
|
|
}
|
|
|
|
daddr = usin->sin_addr.s_addr;
|
|
|
|
dport = usin->sin_port;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
saddr = ipc.addr;
|
|
|
|
ipc.addr = faddr = daddr;
|
|
|
|
|
2011-04-21 16:45:37 +07:00
|
|
|
if (ipc.opt && ipc.opt->opt.srr) {
|
2018-05-11 00:59:34 +07:00
|
|
|
if (!daddr) {
|
|
|
|
err = -EINVAL;
|
|
|
|
goto out_free;
|
|
|
|
}
|
2011-04-21 16:45:37 +07:00
|
|
|
faddr = ipc.opt->opt.faddr;
|
2008-03-07 07:22:02 +07:00
|
|
|
connected = 0;
|
|
|
|
}
|
2013-09-24 20:43:09 +07:00
|
|
|
tos = get_rttos(&ipc, inet);
|
2008-03-07 07:22:02 +07:00
|
|
|
if (sock_flag(sk, SOCK_LOCALROUTE) ||
|
|
|
|
(msg->msg_flags & MSG_DONTROUTE) ||
|
2011-04-21 16:45:37 +07:00
|
|
|
(ipc.opt && ipc.opt->opt.is_strictroute)) {
|
2008-03-07 07:22:02 +07:00
|
|
|
tos |= RTO_ONLINK;
|
|
|
|
connected = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (ipv4_is_multicast(daddr)) {
|
|
|
|
if (!ipc.oif)
|
|
|
|
ipc.oif = inet->mc_index;
|
|
|
|
if (!saddr)
|
|
|
|
saddr = inet->mc_addr;
|
|
|
|
connected = 0;
|
2018-01-25 10:37:38 +07:00
|
|
|
} else if (!ipc.oif) {
|
2012-02-08 16:11:07 +07:00
|
|
|
ipc.oif = inet->uc_index;
|
2018-01-25 10:37:38 +07:00
|
|
|
} else if (ipv4_is_lbcast(daddr) && inet->uc_index) {
|
|
|
|
/* oif is set, packet is to local broadcast and
|
|
|
|
* and uc_index is set. oif is most likely set
|
|
|
|
* by sk_bound_dev_if. If uc_index != oif check if the
|
|
|
|
* oif is an L3 master and uc_index is an L3 slave.
|
|
|
|
* If so, we want to allow the send using the uc_index.
|
|
|
|
*/
|
|
|
|
if (ipc.oif != inet->uc_index &&
|
|
|
|
ipc.oif == l3mdev_master_ifindex_by_index(sock_net(sk),
|
|
|
|
inet->uc_index)) {
|
|
|
|
ipc.oif = inet->uc_index;
|
|
|
|
}
|
|
|
|
}
|
2008-03-07 07:22:02 +07:00
|
|
|
|
|
|
|
if (connected)
|
2009-07-17 07:26:32 +07:00
|
|
|
rt = (struct rtable *)sk_dst_check(sk, 0);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
2015-04-03 15:17:26 +07:00
|
|
|
if (!rt) {
|
2008-07-17 10:19:08 +07:00
|
|
|
struct net *net = sock_net(sk);
|
2015-08-14 03:59:03 +07:00
|
|
|
__u8 flow_flags = inet_sk_flowi_flags(sk);
|
2008-07-17 10:19:08 +07:00
|
|
|
|
2011-05-09 06:38:45 +07:00
|
|
|
fl4 = &fl4_stack;
|
2015-08-14 03:59:03 +07:00
|
|
|
|
2011-05-09 06:38:45 +07:00
|
|
|
flowi4_init_output(fl4, ipc.oif, sk->sk_mark, tos,
|
2011-03-31 18:54:27 +07:00
|
|
|
RT_SCOPE_UNIVERSE, sk->sk_protocol,
|
2015-08-14 03:59:03 +07:00
|
|
|
flow_flags,
|
2016-11-04 00:23:43 +07:00
|
|
|
faddr, saddr, dport, inet->inet_sport,
|
|
|
|
sk->sk_uid);
|
2011-03-31 18:54:27 +07:00
|
|
|
|
2011-05-09 06:38:45 +07:00
|
|
|
security_sk_classify_flow(sk, flowi4_to_flowi(fl4));
|
|
|
|
rt = ip_route_output_flow(net, fl4, sk);
|
2011-03-03 05:31:35 +07:00
|
|
|
if (IS_ERR(rt)) {
|
|
|
|
err = PTR_ERR(rt);
|
2011-03-04 01:38:01 +07:00
|
|
|
rt = NULL;
|
2008-03-07 07:22:02 +07:00
|
|
|
if (err == -ENETUNREACH)
|
2013-11-29 00:51:22 +07:00
|
|
|
IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
|
2008-03-07 07:22:02 +07:00
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
err = -EACCES;
|
|
|
|
if ((rt->rt_flags & RTCF_BROADCAST) &&
|
|
|
|
!sock_flag(sk, SOCK_BROADCAST))
|
|
|
|
goto out;
|
|
|
|
if (connected)
|
2010-06-11 13:31:35 +07:00
|
|
|
sk_dst_set(sk, dst_clone(&rt->dst));
|
2008-03-07 07:22:02 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
if (msg->msg_flags&MSG_CONFIRM)
|
|
|
|
goto do_confirm;
|
|
|
|
back_from_confirm:
|
|
|
|
|
2011-05-09 06:38:45 +07:00
|
|
|
saddr = fl4->saddr;
|
2008-03-07 07:22:02 +07:00
|
|
|
if (!ipc.addr)
|
2011-05-09 06:38:45 +07:00
|
|
|
daddr = ipc.addr = fl4->daddr;
|
2008-03-07 07:22:02 +07:00
|
|
|
|
2011-03-01 09:36:48 +07:00
|
|
|
/* Lockless fast path for the non-corking case. */
|
|
|
|
if (!corkreq) {
|
2018-04-27 00:42:15 +07:00
|
|
|
struct inet_cork cork;
|
|
|
|
|
2014-11-25 01:23:40 +07:00
|
|
|
skb = ip_make_skb(sk, fl4, getfrag, msg, ulen,
|
2011-03-01 09:36:48 +07:00
|
|
|
sizeof(struct udphdr), &ipc, &rt,
|
2018-04-27 00:42:15 +07:00
|
|
|
&cork, msg->msg_flags);
|
2011-03-01 09:36:48 +07:00
|
|
|
err = PTR_ERR(skb);
|
2013-01-22 13:32:49 +07:00
|
|
|
if (!IS_ERR_OR_NULL(skb))
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-27 00:42:17 +07:00
|
|
|
err = udp_send_skb(skb, fl4, &cork);
|
2011-03-01 09:36:48 +07:00
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
lock_sock(sk);
|
|
|
|
if (unlikely(up->pending)) {
|
|
|
|
/* The socket is already corked while preparing it. */
|
|
|
|
/* ... which is an evident application bug. --ANK */
|
|
|
|
release_sock(sk);
|
|
|
|
|
2017-10-19 19:22:17 +07:00
|
|
|
net_dbg_ratelimited("socket already corked\n");
|
2008-03-07 07:22:02 +07:00
|
|
|
err = -EINVAL;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* Now cork the socket to pend data.
|
|
|
|
*/
|
2011-03-12 14:09:18 +07:00
|
|
|
fl4 = &inet->cork.fl.u.ip4;
|
|
|
|
fl4->daddr = daddr;
|
|
|
|
fl4->saddr = saddr;
|
2011-03-12 15:00:33 +07:00
|
|
|
fl4->fl4_dport = dport;
|
|
|
|
fl4->fl4_sport = inet->inet_sport;
|
2008-03-07 07:22:02 +07:00
|
|
|
up->pending = AF_INET;
|
|
|
|
|
|
|
|
do_append_data:
|
|
|
|
up->len += ulen;
|
2014-11-25 01:23:40 +07:00
|
|
|
err = ip_append_data(sk, fl4, getfrag, msg, ulen,
|
2011-05-09 07:24:10 +07:00
|
|
|
sizeof(struct udphdr), &ipc, &rt,
|
|
|
|
corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);
|
2008-03-07 07:22:02 +07:00
|
|
|
if (err)
|
|
|
|
udp_flush_pending_frames(sk);
|
|
|
|
else if (!corkreq)
|
|
|
|
err = udp_push_pending_frames(sk);
|
|
|
|
else if (unlikely(skb_queue_empty(&sk->sk_write_queue)))
|
|
|
|
up->pending = 0;
|
|
|
|
release_sock(sk);
|
|
|
|
|
|
|
|
out:
|
|
|
|
ip_rt_put(rt);
|
2018-05-11 00:59:34 +07:00
|
|
|
out_free:
|
2008-03-07 07:22:02 +07:00
|
|
|
if (free)
|
|
|
|
kfree(ipc.opt);
|
|
|
|
if (!err)
|
|
|
|
return len;
|
|
|
|
/*
|
|
|
|
* ENOBUFS = no kernel mem, SOCK_NOSPACE = no sndbuf space. Reporting
|
|
|
|
* ENOBUFS might not be good (it's not tunable per se), but otherwise
|
|
|
|
* we don't have a good statistic (IpOutDiscards but it can be too many
|
|
|
|
* things). We could add another new stat but at least for now that
|
|
|
|
* seems like overkill.
|
|
|
|
*/
|
|
|
|
if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
|
net: snmp: kill various STATS_USER() helpers
In the old days (before linux-3.0), SNMP counters were duplicated,
one for user context, and one for BH context.
After commit 8f0ea0fe3a03 ("snmp: reduce percpu needs by 50%")
we have a single copy, and what really matters is preemption being
enabled or disabled, since we use this_cpu_inc() or __this_cpu_inc()
respectively.
We therefore kill SNMP_INC_STATS_USER(), SNMP_ADD_STATS_USER(),
NET_INC_STATS_USER(), NET_ADD_STATS_USER(), SCTP_INC_STATS_USER(),
SNMP_INC_STATS64_USER(), SNMP_ADD_STATS64_USER(), TCP_ADD_STATS_USER(),
UDP_INC_STATS_USER(), UDP6_INC_STATS_USER(), and XFRM_INC_STATS_USER()
Following patches will rename __BH helpers to make clear their
usage is not tied to BH being disabled.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-04-28 06:44:27 +07:00
|
|
|
UDP_INC_STATS(sock_net(sk),
|
|
|
|
UDP_MIB_SNDBUFERRORS, is_udplite);
|
2008-03-07 07:22:02 +07:00
|
|
|
}
|
|
|
|
return err;
|
|
|
|
|
|
|
|
do_confirm:
|
2017-02-07 04:14:16 +07:00
|
|
|
if (msg->msg_flags & MSG_PROBE)
|
|
|
|
dst_confirm_neigh(&rt->dst, &fl4->daddr);
|
2008-03-07 07:22:02 +07:00
|
|
|
if (!(msg->msg_flags&MSG_PROBE) || len)
|
|
|
|
goto back_from_confirm;
|
|
|
|
err = 0;
|
|
|
|
goto out;
|
|
|
|
}
|
2009-07-17 07:26:32 +07:00
|
|
|
EXPORT_SYMBOL(udp_sendmsg);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
|
|
|
int udp_sendpage(struct sock *sk, struct page *page, int offset,
|
|
|
|
size_t size, int flags)
|
|
|
|
{
|
2011-05-09 07:24:10 +07:00
|
|
|
struct inet_sock *inet = inet_sk(sk);
|
2008-03-07 07:22:02 +07:00
|
|
|
struct udp_sock *up = udp_sk(sk);
|
|
|
|
int ret;
|
|
|
|
|
2013-11-25 13:36:28 +07:00
|
|
|
if (flags & MSG_SENDPAGE_NOTLAST)
|
|
|
|
flags |= MSG_MORE;
|
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
if (!up->pending) {
|
|
|
|
struct msghdr msg = { .msg_flags = flags|MSG_MORE };
|
|
|
|
|
|
|
|
/* Call udp_sendmsg to specify destination address which
|
|
|
|
* sendpage interface can't pass.
|
|
|
|
* This will succeed only when the socket is connected.
|
|
|
|
*/
|
2015-03-02 14:37:48 +07:00
|
|
|
ret = udp_sendmsg(sk, &msg, 0);
|
2008-03-07 07:22:02 +07:00
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
lock_sock(sk);
|
|
|
|
|
|
|
|
if (unlikely(!up->pending)) {
|
|
|
|
release_sock(sk);
|
|
|
|
|
2017-10-19 19:22:17 +07:00
|
|
|
net_dbg_ratelimited("cork failed\n");
|
2008-03-07 07:22:02 +07:00
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2011-05-09 07:24:10 +07:00
|
|
|
ret = ip_append_page(sk, &inet->cork.fl.u.ip4,
|
|
|
|
page, offset, size, flags);
|
2008-03-07 07:22:02 +07:00
|
|
|
if (ret == -EOPNOTSUPP) {
|
|
|
|
release_sock(sk);
|
|
|
|
return sock_no_sendpage(sk->sk_socket, page, offset,
|
|
|
|
size, flags);
|
|
|
|
}
|
|
|
|
if (ret < 0) {
|
|
|
|
udp_flush_pending_frames(sk);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
up->len += size;
|
|
|
|
if (!(up->corkflag || (flags&MSG_MORE)))
|
|
|
|
ret = udp_push_pending_frames(sk);
|
|
|
|
if (!ret)
|
|
|
|
ret = size;
|
|
|
|
out:
|
|
|
|
release_sock(sk);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2017-07-25 22:57:47 +07:00
|
|
|
#define UDP_SKB_IS_STATELESS 0x80000000
|
|
|
|
|
2017-06-12 16:23:43 +07:00
|
|
|
static void udp_set_dev_scratch(struct sk_buff *skb)
|
|
|
|
{
|
2017-07-25 22:57:47 +07:00
|
|
|
struct udp_dev_scratch *scratch = udp_skb_scratch(skb);
|
2017-06-12 16:23:43 +07:00
|
|
|
|
|
|
|
BUILD_BUG_ON(sizeof(struct udp_dev_scratch) > sizeof(long));
|
2017-07-25 22:57:47 +07:00
|
|
|
scratch->_tsize_state = skb->truesize;
|
|
|
|
#if BITS_PER_LONG == 64
|
2017-06-12 16:23:43 +07:00
|
|
|
scratch->len = skb->len;
|
|
|
|
scratch->csum_unnecessary = !!skb_csum_unnecessary(skb);
|
|
|
|
scratch->is_linear = !skb_is_nonlinear(skb);
|
2017-07-25 22:57:47 +07:00
|
|
|
#endif
|
2017-08-03 23:07:08 +07:00
|
|
|
/* all head states execept sp (dst, sk, nf) are always cleared by
|
|
|
|
* udp_rcv() and we need to preserve secpath, if present, to eventually
|
|
|
|
* process IP_CMSG_PASSSEC at recvmsg() time
|
|
|
|
*/
|
|
|
|
if (likely(!skb_sec_path(skb)))
|
2017-07-25 22:57:47 +07:00
|
|
|
scratch->_tsize_state |= UDP_SKB_IS_STATELESS;
|
2017-06-12 16:23:43 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static int udp_skb_truesize(struct sk_buff *skb)
|
|
|
|
{
|
2017-07-25 22:57:47 +07:00
|
|
|
return udp_skb_scratch(skb)->_tsize_state & ~UDP_SKB_IS_STATELESS;
|
2017-06-12 16:23:43 +07:00
|
|
|
}
|
|
|
|
|
2017-07-25 22:57:47 +07:00
|
|
|
static bool udp_skb_has_head_state(struct sk_buff *skb)
|
2017-06-12 16:23:43 +07:00
|
|
|
{
|
2017-07-25 22:57:47 +07:00
|
|
|
return !(udp_skb_scratch(skb)->_tsize_state & UDP_SKB_IS_STATELESS);
|
2017-06-12 16:23:43 +07:00
|
|
|
}
|
|
|
|
|
2016-11-04 17:28:59 +07:00
|
|
|
/* fully reclaim rmem/fwd memory allocated for skb */
|
2017-05-16 16:20:15 +07:00
|
|
|
static void udp_rmem_release(struct sock *sk, int size, int partial,
|
|
|
|
bool rx_queue_lock_held)
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 18:55:46 +07:00
|
|
|
{
|
2016-12-09 02:41:56 +07:00
|
|
|
struct udp_sock *up = udp_sk(sk);
|
2017-05-16 16:20:14 +07:00
|
|
|
struct sk_buff_head *sk_queue;
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 18:55:46 +07:00
|
|
|
int amt;
|
|
|
|
|
2016-12-09 02:41:56 +07:00
|
|
|
if (likely(partial)) {
|
|
|
|
up->forward_deficit += size;
|
|
|
|
size = up->forward_deficit;
|
2017-09-19 17:11:43 +07:00
|
|
|
if (size < (sk->sk_rcvbuf >> 2))
|
2016-12-09 02:41:56 +07:00
|
|
|
return;
|
|
|
|
} else {
|
|
|
|
size += up->forward_deficit;
|
|
|
|
}
|
|
|
|
up->forward_deficit = 0;
|
|
|
|
|
2017-05-16 16:20:15 +07:00
|
|
|
/* acquire the sk_receive_queue for fwd allocated memory scheduling,
|
|
|
|
* if the called don't held it already
|
|
|
|
*/
|
2017-05-16 16:20:14 +07:00
|
|
|
sk_queue = &sk->sk_receive_queue;
|
2017-05-16 16:20:15 +07:00
|
|
|
if (!rx_queue_lock_held)
|
|
|
|
spin_lock(&sk_queue->lock);
|
|
|
|
|
2017-05-16 16:20:14 +07:00
|
|
|
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 18:55:46 +07:00
|
|
|
sk->sk_forward_alloc += size;
|
|
|
|
amt = (sk->sk_forward_alloc - partial) & ~(SK_MEM_QUANTUM - 1);
|
|
|
|
sk->sk_forward_alloc -= amt;
|
|
|
|
|
|
|
|
if (amt)
|
|
|
|
__sk_mem_reduce_allocated(sk, amt >> SK_MEM_QUANTUM_SHIFT);
|
2016-12-09 02:41:57 +07:00
|
|
|
|
|
|
|
atomic_sub(size, &sk->sk_rmem_alloc);
|
2017-05-16 16:20:14 +07:00
|
|
|
|
|
|
|
/* this can save us from acquiring the rx queue lock on next receive */
|
|
|
|
skb_queue_splice_tail_init(sk_queue, &up->reader_queue);
|
|
|
|
|
2017-05-16 16:20:15 +07:00
|
|
|
if (!rx_queue_lock_held)
|
|
|
|
spin_unlock(&sk_queue->lock);
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 18:55:46 +07:00
|
|
|
}
|
|
|
|
|
2017-05-16 16:20:14 +07:00
|
|
|
/* Note: called with reader_queue.lock held.
|
2016-12-09 02:41:55 +07:00
|
|
|
* Instead of using skb->truesize here, find a copy of it in skb->dev_scratch
|
|
|
|
* This avoids a cache line miss while receive_queue lock is held.
|
|
|
|
* Look at __udp_enqueue_schedule_skb() to find where this copy is done.
|
|
|
|
*/
|
2016-11-04 17:28:59 +07:00
|
|
|
void udp_skb_destructor(struct sock *sk, struct sk_buff *skb)
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 18:55:46 +07:00
|
|
|
{
|
2017-06-12 16:23:43 +07:00
|
|
|
prefetch(&skb->data);
|
|
|
|
udp_rmem_release(sk, udp_skb_truesize(skb), 1, false);
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 18:55:46 +07:00
|
|
|
}
|
2016-11-04 17:28:59 +07:00
|
|
|
EXPORT_SYMBOL(udp_skb_destructor);
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 18:55:46 +07:00
|
|
|
|
2017-05-16 16:20:15 +07:00
|
|
|
/* as above, but the caller held the rx queue lock, too */
|
2017-05-17 15:50:36 +07:00
|
|
|
static void udp_skb_dtor_locked(struct sock *sk, struct sk_buff *skb)
|
2017-05-16 16:20:15 +07:00
|
|
|
{
|
2017-06-12 16:23:43 +07:00
|
|
|
prefetch(&skb->data);
|
|
|
|
udp_rmem_release(sk, udp_skb_truesize(skb), 1, true);
|
2017-05-16 16:20:15 +07:00
|
|
|
}
|
|
|
|
|
2016-12-09 02:41:54 +07:00
|
|
|
/* Idea of busylocks is to let producers grab an extra spinlock
|
|
|
|
* to relieve pressure on the receive_queue spinlock shared by consumer.
|
|
|
|
* Under flood, this means that only one producer can be in line
|
|
|
|
* trying to acquire the receive_queue spinlock.
|
|
|
|
* These busylock can be allocated on a per cpu manner, instead of a
|
|
|
|
* per socket one (that would consume a cache line per socket)
|
|
|
|
*/
|
|
|
|
static int udp_busylocks_log __read_mostly;
|
|
|
|
static spinlock_t *udp_busylocks __read_mostly;
|
|
|
|
|
|
|
|
static spinlock_t *busylock_acquire(void *ptr)
|
|
|
|
{
|
|
|
|
spinlock_t *busy;
|
|
|
|
|
|
|
|
busy = udp_busylocks + hash_ptr(ptr, udp_busylocks_log);
|
|
|
|
spin_lock(busy);
|
|
|
|
return busy;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void busylock_release(spinlock_t *busy)
|
|
|
|
{
|
|
|
|
if (busy)
|
|
|
|
spin_unlock(busy);
|
|
|
|
}
|
|
|
|
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 18:55:46 +07:00
|
|
|
int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
struct sk_buff_head *list = &sk->sk_receive_queue;
|
|
|
|
int rmem, delta, amt, err = -ENOMEM;
|
2016-12-09 02:41:54 +07:00
|
|
|
spinlock_t *busy = NULL;
|
udp: under rx pressure, try to condense skbs
Under UDP flood, many softirq producers try to add packets to
UDP receive queue, and one user thread is burning one cpu trying
to dequeue packets as fast as possible.
Two parts of the per packet cost are :
- copying payload from kernel space to user space,
- freeing memory pieces associated with skb.
If socket is under pressure, softirq handler(s) can try to pull in
skb->head the payload of the packet if it fits.
Meaning the softirq handler(s) can free/reuse the page fragment
immediately, instead of letting udp_recvmsg() do this hundreds of usec
later, possibly from another node.
Additional gains :
- We reduce skb->truesize and thus can store more packets per SO_RCVBUF
- We avoid cache line misses at copyout() time and consume_skb() time,
and avoid one put_page() with potential alien freeing on NUMA hosts.
This comes at the cost of a copy, bounded to available tail room, which
is usually small. (We might have to fix GRO_MAX_HEAD which looks bigger
than necessary)
This patch gave me about 5 % increase in throughput in my tests.
skb_condense() helper could probably used in other contexts.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-12-08 00:19:33 +07:00
|
|
|
int size;
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 18:55:46 +07:00
|
|
|
|
|
|
|
/* try to avoid the costly atomic add/sub pair when the receive
|
|
|
|
* queue is full; always allow at least a packet
|
|
|
|
*/
|
|
|
|
rmem = atomic_read(&sk->sk_rmem_alloc);
|
2016-12-02 23:35:49 +07:00
|
|
|
if (rmem > sk->sk_rcvbuf)
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 18:55:46 +07:00
|
|
|
goto drop;
|
|
|
|
|
udp: under rx pressure, try to condense skbs
Under UDP flood, many softirq producers try to add packets to
UDP receive queue, and one user thread is burning one cpu trying
to dequeue packets as fast as possible.
Two parts of the per packet cost are :
- copying payload from kernel space to user space,
- freeing memory pieces associated with skb.
If socket is under pressure, softirq handler(s) can try to pull in
skb->head the payload of the packet if it fits.
Meaning the softirq handler(s) can free/reuse the page fragment
immediately, instead of letting udp_recvmsg() do this hundreds of usec
later, possibly from another node.
Additional gains :
- We reduce skb->truesize and thus can store more packets per SO_RCVBUF
- We avoid cache line misses at copyout() time and consume_skb() time,
and avoid one put_page() with potential alien freeing on NUMA hosts.
This comes at the cost of a copy, bounded to available tail room, which
is usually small. (We might have to fix GRO_MAX_HEAD which looks bigger
than necessary)
This patch gave me about 5 % increase in throughput in my tests.
skb_condense() helper could probably used in other contexts.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-12-08 00:19:33 +07:00
|
|
|
/* Under mem pressure, it might be helpful to help udp_recvmsg()
|
|
|
|
* having linear skbs :
|
|
|
|
* - Reduce memory overhead and thus increase receive queue capacity
|
|
|
|
* - Less cache line misses at copyout() time
|
|
|
|
* - Less work at consume_skb() (less alien page frag freeing)
|
|
|
|
*/
|
2016-12-09 02:41:54 +07:00
|
|
|
if (rmem > (sk->sk_rcvbuf >> 1)) {
|
udp: under rx pressure, try to condense skbs
Under UDP flood, many softirq producers try to add packets to
UDP receive queue, and one user thread is burning one cpu trying
to dequeue packets as fast as possible.
Two parts of the per packet cost are :
- copying payload from kernel space to user space,
- freeing memory pieces associated with skb.
If socket is under pressure, softirq handler(s) can try to pull in
skb->head the payload of the packet if it fits.
Meaning the softirq handler(s) can free/reuse the page fragment
immediately, instead of letting udp_recvmsg() do this hundreds of usec
later, possibly from another node.
Additional gains :
- We reduce skb->truesize and thus can store more packets per SO_RCVBUF
- We avoid cache line misses at copyout() time and consume_skb() time,
and avoid one put_page() with potential alien freeing on NUMA hosts.
This comes at the cost of a copy, bounded to available tail room, which
is usually small. (We might have to fix GRO_MAX_HEAD which looks bigger
than necessary)
This patch gave me about 5 % increase in throughput in my tests.
skb_condense() helper could probably used in other contexts.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-12-08 00:19:33 +07:00
|
|
|
skb_condense(skb);
|
2016-12-09 02:41:54 +07:00
|
|
|
|
|
|
|
busy = busylock_acquire(sk);
|
|
|
|
}
|
udp: under rx pressure, try to condense skbs
Under UDP flood, many softirq producers try to add packets to
UDP receive queue, and one user thread is burning one cpu trying
to dequeue packets as fast as possible.
Two parts of the per packet cost are :
- copying payload from kernel space to user space,
- freeing memory pieces associated with skb.
If socket is under pressure, softirq handler(s) can try to pull in
skb->head the payload of the packet if it fits.
Meaning the softirq handler(s) can free/reuse the page fragment
immediately, instead of letting udp_recvmsg() do this hundreds of usec
later, possibly from another node.
Additional gains :
- We reduce skb->truesize and thus can store more packets per SO_RCVBUF
- We avoid cache line misses at copyout() time and consume_skb() time,
and avoid one put_page() with potential alien freeing on NUMA hosts.
This comes at the cost of a copy, bounded to available tail room, which
is usually small. (We might have to fix GRO_MAX_HEAD which looks bigger
than necessary)
This patch gave me about 5 % increase in throughput in my tests.
skb_condense() helper could probably used in other contexts.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-12-08 00:19:33 +07:00
|
|
|
size = skb->truesize;
|
2017-06-12 16:23:43 +07:00
|
|
|
udp_set_dev_scratch(skb);
|
udp: under rx pressure, try to condense skbs
Under UDP flood, many softirq producers try to add packets to
UDP receive queue, and one user thread is burning one cpu trying
to dequeue packets as fast as possible.
Two parts of the per packet cost are :
- copying payload from kernel space to user space,
- freeing memory pieces associated with skb.
If socket is under pressure, softirq handler(s) can try to pull in
skb->head the payload of the packet if it fits.
Meaning the softirq handler(s) can free/reuse the page fragment
immediately, instead of letting udp_recvmsg() do this hundreds of usec
later, possibly from another node.
Additional gains :
- We reduce skb->truesize and thus can store more packets per SO_RCVBUF
- We avoid cache line misses at copyout() time and consume_skb() time,
and avoid one put_page() with potential alien freeing on NUMA hosts.
This comes at the cost of a copy, bounded to available tail room, which
is usually small. (We might have to fix GRO_MAX_HEAD which looks bigger
than necessary)
This patch gave me about 5 % increase in throughput in my tests.
skb_condense() helper could probably used in other contexts.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-12-08 00:19:33 +07:00
|
|
|
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 18:55:46 +07:00
|
|
|
/* we drop only if the receive buf is full and the receive
|
|
|
|
* queue contains some other skb
|
|
|
|
*/
|
|
|
|
rmem = atomic_add_return(size, &sk->sk_rmem_alloc);
|
2016-12-02 23:35:49 +07:00
|
|
|
if (rmem > (size + sk->sk_rcvbuf))
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 18:55:46 +07:00
|
|
|
goto uncharge_drop;
|
|
|
|
|
|
|
|
spin_lock(&list->lock);
|
|
|
|
if (size >= sk->sk_forward_alloc) {
|
|
|
|
amt = sk_mem_pages(size);
|
|
|
|
delta = amt << SK_MEM_QUANTUM_SHIFT;
|
|
|
|
if (!__sk_mem_raise_allocated(sk, delta, amt, SK_MEM_RECV)) {
|
|
|
|
err = -ENOBUFS;
|
|
|
|
spin_unlock(&list->lock);
|
|
|
|
goto uncharge_drop;
|
|
|
|
}
|
|
|
|
|
|
|
|
sk->sk_forward_alloc += delta;
|
|
|
|
}
|
|
|
|
|
|
|
|
sk->sk_forward_alloc -= size;
|
|
|
|
|
2016-11-04 17:28:59 +07:00
|
|
|
/* no need to setup a destructor, we will explicitly release the
|
|
|
|
* forward allocated memory on dequeue
|
|
|
|
*/
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 18:55:46 +07:00
|
|
|
sock_skb_set_dropcount(sk, skb);
|
|
|
|
|
|
|
|
__skb_queue_tail(list, skb);
|
|
|
|
spin_unlock(&list->lock);
|
|
|
|
|
|
|
|
if (!sock_flag(sk, SOCK_DEAD))
|
|
|
|
sk->sk_data_ready(sk);
|
|
|
|
|
2016-12-09 02:41:54 +07:00
|
|
|
busylock_release(busy);
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 18:55:46 +07:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
uncharge_drop:
|
|
|
|
atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
|
|
|
|
|
|
|
|
drop:
|
|
|
|
atomic_inc(&sk->sk_drops);
|
2016-12-09 02:41:54 +07:00
|
|
|
busylock_release(busy);
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 18:55:46 +07:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(__udp_enqueue_schedule_skb);
|
|
|
|
|
2016-11-15 22:37:53 +07:00
|
|
|
void udp_destruct_sock(struct sock *sk)
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 18:55:46 +07:00
|
|
|
{
|
|
|
|
/* reclaim completely the forward allocated memory */
|
2017-05-16 16:20:14 +07:00
|
|
|
struct udp_sock *up = udp_sk(sk);
|
2016-11-04 17:28:59 +07:00
|
|
|
unsigned int total = 0;
|
|
|
|
struct sk_buff *skb;
|
|
|
|
|
2017-05-16 16:20:14 +07:00
|
|
|
skb_queue_splice_tail_init(&sk->sk_receive_queue, &up->reader_queue);
|
|
|
|
while ((skb = __skb_dequeue(&up->reader_queue)) != NULL) {
|
2016-11-04 17:28:59 +07:00
|
|
|
total += skb->truesize;
|
|
|
|
kfree_skb(skb);
|
|
|
|
}
|
2017-05-16 16:20:15 +07:00
|
|
|
udp_rmem_release(sk, total, 0, true);
|
2016-11-04 17:28:59 +07:00
|
|
|
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 18:55:46 +07:00
|
|
|
inet_sock_destruct(sk);
|
|
|
|
}
|
2016-11-15 22:37:53 +07:00
|
|
|
EXPORT_SYMBOL_GPL(udp_destruct_sock);
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 18:55:46 +07:00
|
|
|
|
|
|
|
int udp_init_sock(struct sock *sk)
|
|
|
|
{
|
2017-05-16 16:20:14 +07:00
|
|
|
skb_queue_head_init(&udp_sk(sk)->reader_queue);
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 18:55:46 +07:00
|
|
|
sk->sk_destruct = udp_destruct_sock;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(udp_init_sock);
|
|
|
|
|
|
|
|
void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len)
|
|
|
|
{
|
|
|
|
if (unlikely(READ_ONCE(sk->sk_peek_off) >= 0)) {
|
|
|
|
bool slow = lock_sock_fast(sk);
|
|
|
|
|
|
|
|
sk_peek_offset_bwd(sk, len);
|
|
|
|
unlock_sock_fast(sk, slow);
|
|
|
|
}
|
2017-06-12 16:23:42 +07:00
|
|
|
|
2017-09-06 19:44:36 +07:00
|
|
|
if (!skb_unref(skb))
|
|
|
|
return;
|
|
|
|
|
2017-07-25 22:57:47 +07:00
|
|
|
/* In the more common cases we cleared the head states previously,
|
|
|
|
* see __udp_queue_rcv_skb().
|
2017-07-18 16:57:55 +07:00
|
|
|
*/
|
2017-07-25 22:57:47 +07:00
|
|
|
if (unlikely(udp_skb_has_head_state(skb)))
|
2017-07-18 16:57:55 +07:00
|
|
|
skb_release_head_state(skb);
|
2017-09-06 19:44:36 +07:00
|
|
|
__consume_stateless_skb(skb);
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 18:55:46 +07:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(skb_consume_udp);
|
|
|
|
|
2017-05-16 16:20:14 +07:00
|
|
|
static struct sk_buff *__first_packet_length(struct sock *sk,
|
|
|
|
struct sk_buff_head *rcvq,
|
|
|
|
int *total)
|
|
|
|
{
|
|
|
|
struct sk_buff *skb;
|
|
|
|
|
2017-06-23 19:19:51 +07:00
|
|
|
while ((skb = skb_peek(rcvq)) != NULL) {
|
|
|
|
if (udp_lib_checksum_complete(skb)) {
|
|
|
|
__UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS,
|
|
|
|
IS_UDPLITE(sk));
|
|
|
|
__UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS,
|
|
|
|
IS_UDPLITE(sk));
|
|
|
|
atomic_inc(&sk->sk_drops);
|
|
|
|
__skb_unlink(skb, rcvq);
|
|
|
|
*total += skb->truesize;
|
|
|
|
kfree_skb(skb);
|
|
|
|
} else {
|
|
|
|
/* the csum related bits could be changed, refresh
|
|
|
|
* the scratch area
|
|
|
|
*/
|
|
|
|
udp_set_dev_scratch(skb);
|
|
|
|
break;
|
|
|
|
}
|
2017-05-16 16:20:14 +07:00
|
|
|
}
|
|
|
|
return skb;
|
|
|
|
}
|
|
|
|
|
2009-10-09 11:43:40 +07:00
|
|
|
/**
|
|
|
|
* first_packet_length - return length of first packet in receive queue
|
|
|
|
* @sk: socket
|
|
|
|
*
|
|
|
|
* Drops all bad checksum frames, until a valid one is found.
|
2016-08-24 03:59:33 +07:00
|
|
|
* Returns the length of found skb, or -1 if none is found.
|
2009-10-09 11:43:40 +07:00
|
|
|
*/
|
2016-08-24 03:59:33 +07:00
|
|
|
static int first_packet_length(struct sock *sk)
|
2009-10-09 11:43:40 +07:00
|
|
|
{
|
2017-05-16 16:20:14 +07:00
|
|
|
struct sk_buff_head *rcvq = &udp_sk(sk)->reader_queue;
|
|
|
|
struct sk_buff_head *sk_queue = &sk->sk_receive_queue;
|
2009-10-09 11:43:40 +07:00
|
|
|
struct sk_buff *skb;
|
2016-11-04 17:28:59 +07:00
|
|
|
int total = 0;
|
2016-08-24 03:59:33 +07:00
|
|
|
int res;
|
2009-10-09 11:43:40 +07:00
|
|
|
|
|
|
|
spin_lock_bh(&rcvq->lock);
|
2017-05-16 16:20:14 +07:00
|
|
|
skb = __first_packet_length(sk, rcvq, &total);
|
|
|
|
if (!skb && !skb_queue_empty(sk_queue)) {
|
|
|
|
spin_lock(&sk_queue->lock);
|
|
|
|
skb_queue_splice_tail_init(sk_queue, rcvq);
|
|
|
|
spin_unlock(&sk_queue->lock);
|
|
|
|
|
|
|
|
skb = __first_packet_length(sk, rcvq, &total);
|
2009-10-09 11:43:40 +07:00
|
|
|
}
|
2016-08-24 03:59:33 +07:00
|
|
|
res = skb ? skb->len : -1;
|
2016-11-04 17:28:59 +07:00
|
|
|
if (total)
|
2017-05-16 16:20:15 +07:00
|
|
|
udp_rmem_release(sk, total, 1, false);
|
2009-10-09 11:43:40 +07:00
|
|
|
spin_unlock_bh(&rcvq->lock);
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
2005-04-17 05:20:36 +07:00
|
|
|
/*
|
|
|
|
* IOCTL requests applicable to the UDP protocol
|
|
|
|
*/
|
2007-02-09 21:24:47 +07:00
|
|
|
|
2005-04-17 05:20:36 +07:00
|
|
|
int udp_ioctl(struct sock *sk, int cmd, unsigned long arg)
|
|
|
|
{
|
2007-03-09 11:41:55 +07:00
|
|
|
switch (cmd) {
|
|
|
|
case SIOCOUTQ:
|
2005-04-17 05:20:36 +07:00
|
|
|
{
|
2009-06-18 09:05:41 +07:00
|
|
|
int amount = sk_wmem_alloc_get(sk);
|
|
|
|
|
2007-03-09 11:41:55 +07:00
|
|
|
return put_user(amount, (int __user *)arg);
|
|
|
|
}
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2007-03-09 11:41:55 +07:00
|
|
|
case SIOCINQ:
|
|
|
|
{
|
2016-08-24 03:59:33 +07:00
|
|
|
int amount = max_t(int, 0, first_packet_length(sk));
|
2007-03-09 11:41:55 +07:00
|
|
|
|
|
|
|
return put_user(amount, (int __user *)arg);
|
|
|
|
}
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2007-03-09 11:41:55 +07:00
|
|
|
default:
|
|
|
|
return -ENOIOCTLCMD;
|
2005-04-17 05:20:36 +07:00
|
|
|
}
|
2007-03-09 11:41:55 +07:00
|
|
|
|
|
|
|
return 0;
|
2005-04-17 05:20:36 +07:00
|
|
|
}
|
2009-07-17 07:26:32 +07:00
|
|
|
EXPORT_SYMBOL(udp_ioctl);
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2017-05-16 16:20:14 +07:00
|
|
|
struct sk_buff *__skb_recv_udp(struct sock *sk, unsigned int flags,
|
|
|
|
int noblock, int *peeked, int *off, int *err)
|
|
|
|
{
|
|
|
|
struct sk_buff_head *sk_queue = &sk->sk_receive_queue;
|
|
|
|
struct sk_buff_head *queue;
|
|
|
|
struct sk_buff *last;
|
|
|
|
long timeo;
|
|
|
|
int error;
|
|
|
|
|
|
|
|
queue = &udp_sk(sk)->reader_queue;
|
|
|
|
flags |= noblock ? MSG_DONTWAIT : 0;
|
|
|
|
timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
|
|
|
|
do {
|
|
|
|
struct sk_buff *skb;
|
|
|
|
|
|
|
|
error = sock_error(sk);
|
|
|
|
if (error)
|
|
|
|
break;
|
|
|
|
|
|
|
|
error = -EAGAIN;
|
|
|
|
*peeked = 0;
|
|
|
|
do {
|
|
|
|
spin_lock_bh(&queue->lock);
|
|
|
|
skb = __skb_try_recv_from_queue(sk, queue, flags,
|
|
|
|
udp_skb_destructor,
|
2017-05-18 01:39:05 +07:00
|
|
|
peeked, off, err,
|
2017-05-16 16:20:14 +07:00
|
|
|
&last);
|
|
|
|
if (skb) {
|
|
|
|
spin_unlock_bh(&queue->lock);
|
|
|
|
return skb;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (skb_queue_empty(sk_queue)) {
|
|
|
|
spin_unlock_bh(&queue->lock);
|
|
|
|
goto busy_check;
|
|
|
|
}
|
|
|
|
|
2017-05-16 16:20:15 +07:00
|
|
|
/* refill the reader queue and walk it again
|
|
|
|
* keep both queues locked to avoid re-acquiring
|
|
|
|
* the sk_receive_queue lock if fwd memory scheduling
|
|
|
|
* is needed.
|
|
|
|
*/
|
2017-05-16 16:20:14 +07:00
|
|
|
spin_lock(&sk_queue->lock);
|
|
|
|
skb_queue_splice_tail_init(sk_queue, queue);
|
|
|
|
|
|
|
|
skb = __skb_try_recv_from_queue(sk, queue, flags,
|
2017-05-16 16:20:15 +07:00
|
|
|
udp_skb_dtor_locked,
|
2017-05-18 01:39:05 +07:00
|
|
|
peeked, off, err,
|
2017-05-16 16:20:14 +07:00
|
|
|
&last);
|
2017-05-16 16:20:15 +07:00
|
|
|
spin_unlock(&sk_queue->lock);
|
2017-05-16 16:20:14 +07:00
|
|
|
spin_unlock_bh(&queue->lock);
|
2017-05-18 01:39:05 +07:00
|
|
|
if (skb)
|
2017-05-16 16:20:14 +07:00
|
|
|
return skb;
|
|
|
|
|
|
|
|
busy_check:
|
|
|
|
if (!sk_can_busy_loop(sk))
|
|
|
|
break;
|
|
|
|
|
|
|
|
sk_busy_loop(sk, flags & MSG_DONTWAIT);
|
|
|
|
} while (!skb_queue_empty(sk_queue));
|
|
|
|
|
|
|
|
/* sk_queue is empty, reader_queue may contain peeked packets */
|
|
|
|
} while (timeo &&
|
|
|
|
!__skb_wait_for_more_packets(sk, &error, &timeo,
|
|
|
|
(struct sk_buff *)sk_queue));
|
|
|
|
|
|
|
|
*err = error;
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(__skb_recv_udp);
|
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
/*
|
|
|
|
* This should be easy, if there is something there we
|
|
|
|
* return it, otherwise we block.
|
|
|
|
*/
|
|
|
|
|
2015-03-02 14:37:48 +07:00
|
|
|
int udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int noblock,
|
|
|
|
int flags, int *addr_len)
|
2008-03-07 07:22:02 +07:00
|
|
|
{
|
|
|
|
struct inet_sock *inet = inet_sk(sk);
|
2014-01-18 04:53:15 +07:00
|
|
|
DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name);
|
2008-03-07 07:22:02 +07:00
|
|
|
struct sk_buff *skb;
|
2011-12-02 02:12:55 +07:00
|
|
|
unsigned int ulen, copied;
|
2016-04-05 23:41:16 +07:00
|
|
|
int peeked, peeking, off;
|
2008-03-07 07:22:02 +07:00
|
|
|
int err;
|
|
|
|
int is_udplite = IS_UDPLITE(sk);
|
udp: properly support MSG_PEEK with truncated buffers
Backport of this upstream commit into stable kernels :
89c22d8c3b27 ("net: Fix skb csum races when peeking")
exposed a bug in udp stack vs MSG_PEEK support, when user provides
a buffer smaller than skb payload.
In this case,
skb_copy_and_csum_datagram_iovec(skb, sizeof(struct udphdr),
msg->msg_iov);
returns -EFAULT.
This bug does not happen in upstream kernels since Al Viro did a great
job to replace this into :
skb_copy_and_csum_datagram_msg(skb, sizeof(struct udphdr), msg);
This variant is safe vs short buffers.
For the time being, instead reverting Herbert Xu patch and add back
skb->ip_summed invalid changes, simply store the result of
udp_lib_checksum_complete() so that we avoid computing the checksum a
second time, and avoid the problematic
skb_copy_and_csum_datagram_iovec() call.
This patch can be applied on recent kernels as it avoids a double
checksumming, then backported to stable kernels as a bug fix.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Acked-by: Herbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-12-30 20:51:12 +07:00
|
|
|
bool checksum_valid = false;
|
2008-03-07 07:22:02 +07:00
|
|
|
|
|
|
|
if (flags & MSG_ERRQUEUE)
|
2013-11-23 06:46:12 +07:00
|
|
|
return ip_recv_error(sk, msg, len, addr_len);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
|
|
|
try_again:
|
datagram: When peeking datagrams with offset < 0 don't skip empty skbs
Due to commit e6afc8ace6dd5cef5e812f26c72579da8806f5ac ("udp: remove
headers from UDP packets before queueing"), when udp packets are being
peeked the requested extra offset is always 0 as there is no need to skip
the udp header. However, when the offset is 0 and the next skb is
of length 0, it is only returned once. The behaviour can be seen with
the following python script:
from socket import *;
f=socket(AF_INET6, SOCK_DGRAM | SOCK_NONBLOCK, 0);
g=socket(AF_INET6, SOCK_DGRAM | SOCK_NONBLOCK, 0);
f.bind(('::', 0));
addr=('::1', f.getsockname()[1]);
g.sendto(b'', addr)
g.sendto(b'b', addr)
print(f.recvfrom(10, MSG_PEEK));
print(f.recvfrom(10, MSG_PEEK));
Where the expected output should be the empty string twice.
Instead, make sk_peek_offset return negative values, and pass those values
to __skb_try_recv_datagram/__skb_try_recv_from_queue. If the passed offset
to __skb_try_recv_from_queue is negative, the checked skb is never skipped.
__skb_try_recv_from_queue will then ensure the offset is reset back to 0
if a peek is requested without an offset, unless no packets are found.
Also simplify the if condition in __skb_try_recv_from_queue. If _off is
greater then 0, and off is greater then or equal to skb->len, then
(_off || skb->len) must always be true assuming skb->len >= 0 is always
true.
Also remove a redundant check around a call to sk_peek_offset in af_unix.c,
as it double checked if MSG_PEEK was set in the flags.
V2:
- Moved the negative fixup into __skb_try_recv_from_queue, and remove now
redundant checks
- Fix peeking in udp{,v6}_recvmsg to report the right value when the
offset is 0
V3:
- Marked new branch in __skb_try_recv_from_queue as unlikely.
Signed-off-by: Matthew Dawson <matthew@mjdsystems.ca>
Acked-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-08-19 02:04:54 +07:00
|
|
|
peeking = flags & MSG_PEEK;
|
|
|
|
off = sk_peek_offset(sk, flags);
|
2016-11-04 17:28:59 +07:00
|
|
|
skb = __skb_recv_udp(sk, flags, noblock, &peeked, &off, &err);
|
2008-03-07 07:22:02 +07:00
|
|
|
if (!skb)
|
2016-04-05 23:41:16 +07:00
|
|
|
return err;
|
2008-03-07 07:22:02 +07:00
|
|
|
|
2017-06-12 16:23:43 +07:00
|
|
|
ulen = udp_skb_len(skb);
|
2011-12-02 02:12:55 +07:00
|
|
|
copied = len;
|
2016-04-05 23:41:16 +07:00
|
|
|
if (copied > ulen - off)
|
|
|
|
copied = ulen - off;
|
2011-12-02 02:12:55 +07:00
|
|
|
else if (copied < ulen)
|
2008-03-07 07:22:02 +07:00
|
|
|
msg->msg_flags |= MSG_TRUNC;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If checksum is needed at all, try to do it while copying the
|
|
|
|
* data. If the data is truncated, or if we only want a partial
|
|
|
|
* coverage checksum (UDP-Lite), do it before the copy.
|
|
|
|
*/
|
|
|
|
|
2016-11-19 08:18:03 +07:00
|
|
|
if (copied < ulen || peeking ||
|
|
|
|
(is_udplite && UDP_SKB_CB(skb)->partial_cov)) {
|
2017-06-12 16:23:43 +07:00
|
|
|
checksum_valid = udp_skb_csum_unnecessary(skb) ||
|
|
|
|
!__udp_lib_checksum_complete(skb);
|
udp: properly support MSG_PEEK with truncated buffers
Backport of this upstream commit into stable kernels :
89c22d8c3b27 ("net: Fix skb csum races when peeking")
exposed a bug in udp stack vs MSG_PEEK support, when user provides
a buffer smaller than skb payload.
In this case,
skb_copy_and_csum_datagram_iovec(skb, sizeof(struct udphdr),
msg->msg_iov);
returns -EFAULT.
This bug does not happen in upstream kernels since Al Viro did a great
job to replace this into :
skb_copy_and_csum_datagram_msg(skb, sizeof(struct udphdr), msg);
This variant is safe vs short buffers.
For the time being, instead reverting Herbert Xu patch and add back
skb->ip_summed invalid changes, simply store the result of
udp_lib_checksum_complete() so that we avoid computing the checksum a
second time, and avoid the problematic
skb_copy_and_csum_datagram_iovec() call.
This patch can be applied on recent kernels as it avoids a double
checksumming, then backported to stable kernels as a bug fix.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Acked-by: Herbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-12-30 20:51:12 +07:00
|
|
|
if (!checksum_valid)
|
2008-03-07 07:22:02 +07:00
|
|
|
goto csum_copy_err;
|
|
|
|
}
|
|
|
|
|
2017-06-12 16:23:43 +07:00
|
|
|
if (checksum_valid || udp_skb_csum_unnecessary(skb)) {
|
|
|
|
if (udp_skb_is_linear(skb))
|
|
|
|
err = copy_linear_skb(skb, copied, off, &msg->msg_iter);
|
|
|
|
else
|
|
|
|
err = skb_copy_datagram_msg(skb, off, msg, copied);
|
|
|
|
} else {
|
2016-04-05 23:41:16 +07:00
|
|
|
err = skb_copy_and_csum_datagram_msg(skb, off, msg);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
|
|
|
if (err == -EINVAL)
|
|
|
|
goto csum_copy_err;
|
|
|
|
}
|
|
|
|
|
2012-06-27 07:23:44 +07:00
|
|
|
if (unlikely(err)) {
|
2012-09-06 06:34:44 +07:00
|
|
|
if (!peeked) {
|
|
|
|
atomic_inc(&sk->sk_drops);
|
net: snmp: kill various STATS_USER() helpers
In the old days (before linux-3.0), SNMP counters were duplicated,
one for user context, and one for BH context.
After commit 8f0ea0fe3a03 ("snmp: reduce percpu needs by 50%")
we have a single copy, and what really matters is preemption being
enabled or disabled, since we use this_cpu_inc() or __this_cpu_inc()
respectively.
We therefore kill SNMP_INC_STATS_USER(), SNMP_ADD_STATS_USER(),
NET_INC_STATS_USER(), NET_ADD_STATS_USER(), SCTP_INC_STATS_USER(),
SNMP_INC_STATS64_USER(), SNMP_ADD_STATS64_USER(), TCP_ADD_STATS_USER(),
UDP_INC_STATS_USER(), UDP6_INC_STATS_USER(), and XFRM_INC_STATS_USER()
Following patches will rename __BH helpers to make clear their
usage is not tied to BH being disabled.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-04-28 06:44:27 +07:00
|
|
|
UDP_INC_STATS(sock_net(sk),
|
|
|
|
UDP_MIB_INERRORS, is_udplite);
|
2012-09-06 06:34:44 +07:00
|
|
|
}
|
2016-10-21 18:55:47 +07:00
|
|
|
kfree_skb(skb);
|
2016-04-05 23:41:16 +07:00
|
|
|
return err;
|
2012-06-27 07:23:44 +07:00
|
|
|
}
|
2008-03-07 07:22:02 +07:00
|
|
|
|
|
|
|
if (!peeked)
|
net: snmp: kill various STATS_USER() helpers
In the old days (before linux-3.0), SNMP counters were duplicated,
one for user context, and one for BH context.
After commit 8f0ea0fe3a03 ("snmp: reduce percpu needs by 50%")
we have a single copy, and what really matters is preemption being
enabled or disabled, since we use this_cpu_inc() or __this_cpu_inc()
respectively.
We therefore kill SNMP_INC_STATS_USER(), SNMP_ADD_STATS_USER(),
NET_INC_STATS_USER(), NET_ADD_STATS_USER(), SCTP_INC_STATS_USER(),
SNMP_INC_STATS64_USER(), SNMP_ADD_STATS64_USER(), TCP_ADD_STATS_USER(),
UDP_INC_STATS_USER(), UDP6_INC_STATS_USER(), and XFRM_INC_STATS_USER()
Following patches will rename __BH helpers to make clear their
usage is not tied to BH being disabled.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-04-28 06:44:27 +07:00
|
|
|
UDP_INC_STATS(sock_net(sk),
|
|
|
|
UDP_MIB_INDATAGRAMS, is_udplite);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
net: Generalize socket rx gap / receive queue overflow cmsg
Create a new socket level option to report number of queue overflows
Recently I augmented the AF_PACKET protocol to report the number of frames lost
on the socket receive queue between any two enqueued frames. This value was
exported via a SOL_PACKET level cmsg. AFter I completed that work it was
requested that this feature be generalized so that any datagram oriented socket
could make use of this option. As such I've created this patch, It creates a
new SOL_SOCKET level option called SO_RXQ_OVFL, which when enabled exports a
SOL_SOCKET level cmsg that reports the nubmer of times the sk_receive_queue
overflowed between any two given frames. It also augments the AF_PACKET
protocol to take advantage of this new feature (as it previously did not touch
sk->sk_drops, which this patch uses to record the overflow count). Tested
successfully by me.
Notes:
1) Unlike my previous patch, this patch simply records the sk_drops value, which
is not a number of drops between packets, but rather a total number of drops.
Deltas must be computed in user space.
2) While this patch currently works with datagram oriented protocols, it will
also be accepted by non-datagram oriented protocols. I'm not sure if thats
agreeable to everyone, but my argument in favor of doing so is that, for those
protocols which aren't applicable to this option, sk_drops will always be zero,
and reporting no drops on a receive queue that isn't used for those
non-participating protocols seems reasonable to me. This also saves us having
to code in a per-protocol opt in mechanism.
3) This applies cleanly to net-next assuming that commit
977750076d98c7ff6cbda51858bb5a5894a9d9ab (my af packet cmsg patch) is reverted
Signed-off-by: Neil Horman <nhorman@tuxdriver.com>
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-10-13 03:26:31 +07:00
|
|
|
sock_recv_ts_and_drops(msg, sk, skb);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
|
|
|
/* Copy the address. */
|
2009-07-17 07:26:32 +07:00
|
|
|
if (sin) {
|
2008-03-07 07:22:02 +07:00
|
|
|
sin->sin_family = AF_INET;
|
|
|
|
sin->sin_port = udp_hdr(skb)->source;
|
|
|
|
sin->sin_addr.s_addr = ip_hdr(skb)->saddr;
|
|
|
|
memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
|
2013-11-18 10:20:45 +07:00
|
|
|
*addr_len = sizeof(*sin);
|
2008-03-07 07:22:02 +07:00
|
|
|
}
|
|
|
|
if (inet->cmsg_flags)
|
2016-11-04 17:28:58 +07:00
|
|
|
ip_cmsg_recv_offset(msg, sk, skb, sizeof(struct udphdr), off);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
2011-12-02 02:12:55 +07:00
|
|
|
err = copied;
|
2008-03-07 07:22:02 +07:00
|
|
|
if (flags & MSG_TRUNC)
|
|
|
|
err = ulen;
|
|
|
|
|
2016-10-21 18:55:47 +07:00
|
|
|
skb_consume_udp(sk, skb, peeking ? -err : err);
|
2008-03-07 07:22:02 +07:00
|
|
|
return err;
|
|
|
|
|
|
|
|
csum_copy_err:
|
2017-05-16 16:20:14 +07:00
|
|
|
if (!__sk_queue_drop_skb(sk, &udp_sk(sk)->reader_queue, skb, flags,
|
|
|
|
udp_skb_destructor)) {
|
net: snmp: kill various STATS_USER() helpers
In the old days (before linux-3.0), SNMP counters were duplicated,
one for user context, and one for BH context.
After commit 8f0ea0fe3a03 ("snmp: reduce percpu needs by 50%")
we have a single copy, and what really matters is preemption being
enabled or disabled, since we use this_cpu_inc() or __this_cpu_inc()
respectively.
We therefore kill SNMP_INC_STATS_USER(), SNMP_ADD_STATS_USER(),
NET_INC_STATS_USER(), NET_ADD_STATS_USER(), SCTP_INC_STATS_USER(),
SNMP_INC_STATS64_USER(), SNMP_ADD_STATS64_USER(), TCP_ADD_STATS_USER(),
UDP_INC_STATS_USER(), UDP6_INC_STATS_USER(), and XFRM_INC_STATS_USER()
Following patches will rename __BH helpers to make clear their
usage is not tied to BH being disabled.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-04-28 06:44:27 +07:00
|
|
|
UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
|
|
|
|
UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
|
2013-04-29 15:39:56 +07:00
|
|
|
}
|
2016-10-21 18:55:47 +07:00
|
|
|
kfree_skb(skb);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
2015-05-30 23:16:53 +07:00
|
|
|
/* starting over for a new packet, but check if we need to yield */
|
|
|
|
cond_resched();
|
2011-06-21 17:43:40 +07:00
|
|
|
msg->msg_flags &= ~MSG_TRUNC;
|
2008-03-07 07:22:02 +07:00
|
|
|
goto try_again;
|
|
|
|
}
|
|
|
|
|
2018-03-31 05:08:05 +07:00
|
|
|
int udp_pre_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
|
|
|
|
{
|
|
|
|
/* This check is replicated from __ip4_datagram_connect() and
|
|
|
|
* intended to prevent BPF program called below from accessing bytes
|
|
|
|
* that are out of the bound specified by user in addr_len.
|
|
|
|
*/
|
|
|
|
if (addr_len < sizeof(struct sockaddr_in))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
return BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(udp_pre_connect);
|
|
|
|
|
2016-10-20 23:39:40 +07:00
|
|
|
int __udp_disconnect(struct sock *sk, int flags)
|
2005-04-17 05:20:36 +07:00
|
|
|
{
|
|
|
|
struct inet_sock *inet = inet_sk(sk);
|
|
|
|
/*
|
|
|
|
* 1003.1g - break association.
|
|
|
|
*/
|
2007-02-09 21:24:47 +07:00
|
|
|
|
2005-04-17 05:20:36 +07:00
|
|
|
sk->sk_state = TCP_CLOSE;
|
2009-10-15 13:30:45 +07:00
|
|
|
inet->inet_daddr = 0;
|
|
|
|
inet->inet_dport = 0;
|
2011-08-15 02:45:55 +07:00
|
|
|
sock_rps_reset_rxhash(sk);
|
2005-04-17 05:20:36 +07:00
|
|
|
sk->sk_bound_dev_if = 0;
|
|
|
|
if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
|
|
|
|
inet_reset_saddr(sk);
|
|
|
|
|
|
|
|
if (!(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) {
|
|
|
|
sk->sk_prot->unhash(sk);
|
2009-10-15 13:30:45 +07:00
|
|
|
inet->inet_sport = 0;
|
2005-04-17 05:20:36 +07:00
|
|
|
}
|
|
|
|
sk_dst_reset(sk);
|
|
|
|
return 0;
|
|
|
|
}
|
2016-10-20 23:39:40 +07:00
|
|
|
EXPORT_SYMBOL(__udp_disconnect);
|
|
|
|
|
|
|
|
int udp_disconnect(struct sock *sk, int flags)
|
|
|
|
{
|
|
|
|
lock_sock(sk);
|
|
|
|
__udp_disconnect(sk, flags);
|
|
|
|
release_sock(sk);
|
|
|
|
return 0;
|
|
|
|
}
|
2009-07-17 07:26:32 +07:00
|
|
|
EXPORT_SYMBOL(udp_disconnect);
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2008-10-29 15:41:45 +07:00
|
|
|
void udp_lib_unhash(struct sock *sk)
|
|
|
|
{
|
2008-11-26 04:55:15 +07:00
|
|
|
if (sk_hashed(sk)) {
|
|
|
|
struct udp_table *udptable = sk->sk_prot->h.udp_table;
|
2009-11-08 17:17:58 +07:00
|
|
|
struct udp_hslot *hslot, *hslot2;
|
|
|
|
|
|
|
|
hslot = udp_hashslot(udptable, sock_net(sk),
|
|
|
|
udp_sk(sk)->udp_port_hash);
|
|
|
|
hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
|
2008-10-29 15:41:45 +07:00
|
|
|
|
2008-11-26 04:55:15 +07:00
|
|
|
spin_lock_bh(&hslot->lock);
|
2016-01-05 05:41:46 +07:00
|
|
|
if (rcu_access_pointer(sk->sk_reuseport_cb))
|
|
|
|
reuseport_detach_sock(sk);
|
2016-04-01 22:52:13 +07:00
|
|
|
if (sk_del_node_init_rcu(sk)) {
|
2009-11-08 17:17:05 +07:00
|
|
|
hslot->count--;
|
2009-10-15 13:30:45 +07:00
|
|
|
inet_sk(sk)->inet_num = 0;
|
2008-11-26 04:55:15 +07:00
|
|
|
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
|
2009-11-08 17:17:58 +07:00
|
|
|
|
|
|
|
spin_lock(&hslot2->lock);
|
2016-04-01 22:52:13 +07:00
|
|
|
hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
|
2009-11-08 17:17:58 +07:00
|
|
|
hslot2->count--;
|
|
|
|
spin_unlock(&hslot2->lock);
|
2008-11-26 04:55:15 +07:00
|
|
|
}
|
|
|
|
spin_unlock_bh(&hslot->lock);
|
2008-10-29 15:41:45 +07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(udp_lib_unhash);
|
|
|
|
|
udp: add rehash on connect()
commit 30fff923 introduced in linux-2.6.33 (udp: bind() optimisation)
added a secondary hash on UDP, hashed on (local addr, local port).
Problem is that following sequence :
fd = socket(...)
connect(fd, &remote, ...)
not only selects remote end point (address and port), but also sets
local address, while UDP stack stored in secondary hash table the socket
while its local address was INADDR_ANY (or ipv6 equivalent)
Sequence is :
- autobind() : choose a random local port, insert socket in hash tables
[while local address is INADDR_ANY]
- connect() : set remote address and port, change local address to IP
given by a route lookup.
When an incoming UDP frame comes, if more than 10 sockets are found in
primary hash table, we switch to secondary table, and fail to find
socket because its local address changed.
One solution to this problem is to rehash datagram socket if needed.
We add a new rehash(struct socket *) method in "struct proto", and
implement this method for UDP v4 & v6, using a common helper.
This rehashing only takes care of secondary hash table, since primary
hash (based on local port only) is not changed.
Reported-by: Krzysztof Piotr Oledzki <ole@ans.pl>
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Tested-by: Krzysztof Piotr Oledzki <ole@ans.pl>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-09-08 12:08:44 +07:00
|
|
|
/*
|
|
|
|
* inet_rcv_saddr was changed, we must rehash secondary hash
|
|
|
|
*/
|
|
|
|
void udp_lib_rehash(struct sock *sk, u16 newhash)
|
|
|
|
{
|
|
|
|
if (sk_hashed(sk)) {
|
|
|
|
struct udp_table *udptable = sk->sk_prot->h.udp_table;
|
|
|
|
struct udp_hslot *hslot, *hslot2, *nhslot2;
|
|
|
|
|
|
|
|
hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
|
|
|
|
nhslot2 = udp_hashslot2(udptable, newhash);
|
|
|
|
udp_sk(sk)->udp_portaddr_hash = newhash;
|
2016-01-05 05:41:46 +07:00
|
|
|
|
|
|
|
if (hslot2 != nhslot2 ||
|
|
|
|
rcu_access_pointer(sk->sk_reuseport_cb)) {
|
udp: add rehash on connect()
commit 30fff923 introduced in linux-2.6.33 (udp: bind() optimisation)
added a secondary hash on UDP, hashed on (local addr, local port).
Problem is that following sequence :
fd = socket(...)
connect(fd, &remote, ...)
not only selects remote end point (address and port), but also sets
local address, while UDP stack stored in secondary hash table the socket
while its local address was INADDR_ANY (or ipv6 equivalent)
Sequence is :
- autobind() : choose a random local port, insert socket in hash tables
[while local address is INADDR_ANY]
- connect() : set remote address and port, change local address to IP
given by a route lookup.
When an incoming UDP frame comes, if more than 10 sockets are found in
primary hash table, we switch to secondary table, and fail to find
socket because its local address changed.
One solution to this problem is to rehash datagram socket if needed.
We add a new rehash(struct socket *) method in "struct proto", and
implement this method for UDP v4 & v6, using a common helper.
This rehashing only takes care of secondary hash table, since primary
hash (based on local port only) is not changed.
Reported-by: Krzysztof Piotr Oledzki <ole@ans.pl>
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Tested-by: Krzysztof Piotr Oledzki <ole@ans.pl>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-09-08 12:08:44 +07:00
|
|
|
hslot = udp_hashslot(udptable, sock_net(sk),
|
|
|
|
udp_sk(sk)->udp_port_hash);
|
|
|
|
/* we must lock primary chain too */
|
|
|
|
spin_lock_bh(&hslot->lock);
|
2016-01-05 05:41:46 +07:00
|
|
|
if (rcu_access_pointer(sk->sk_reuseport_cb))
|
|
|
|
reuseport_detach_sock(sk);
|
|
|
|
|
|
|
|
if (hslot2 != nhslot2) {
|
|
|
|
spin_lock(&hslot2->lock);
|
2016-04-01 22:52:13 +07:00
|
|
|
hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
|
2016-01-05 05:41:46 +07:00
|
|
|
hslot2->count--;
|
|
|
|
spin_unlock(&hslot2->lock);
|
|
|
|
|
|
|
|
spin_lock(&nhslot2->lock);
|
2016-04-01 22:52:13 +07:00
|
|
|
hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
|
2016-01-05 05:41:46 +07:00
|
|
|
&nhslot2->head);
|
|
|
|
nhslot2->count++;
|
|
|
|
spin_unlock(&nhslot2->lock);
|
|
|
|
}
|
udp: add rehash on connect()
commit 30fff923 introduced in linux-2.6.33 (udp: bind() optimisation)
added a secondary hash on UDP, hashed on (local addr, local port).
Problem is that following sequence :
fd = socket(...)
connect(fd, &remote, ...)
not only selects remote end point (address and port), but also sets
local address, while UDP stack stored in secondary hash table the socket
while its local address was INADDR_ANY (or ipv6 equivalent)
Sequence is :
- autobind() : choose a random local port, insert socket in hash tables
[while local address is INADDR_ANY]
- connect() : set remote address and port, change local address to IP
given by a route lookup.
When an incoming UDP frame comes, if more than 10 sockets are found in
primary hash table, we switch to secondary table, and fail to find
socket because its local address changed.
One solution to this problem is to rehash datagram socket if needed.
We add a new rehash(struct socket *) method in "struct proto", and
implement this method for UDP v4 & v6, using a common helper.
This rehashing only takes care of secondary hash table, since primary
hash (based on local port only) is not changed.
Reported-by: Krzysztof Piotr Oledzki <ole@ans.pl>
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Tested-by: Krzysztof Piotr Oledzki <ole@ans.pl>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-09-08 12:08:44 +07:00
|
|
|
|
|
|
|
spin_unlock_bh(&hslot->lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(udp_lib_rehash);
|
|
|
|
|
|
|
|
static void udp_v4_rehash(struct sock *sk)
|
|
|
|
{
|
2017-12-02 03:52:30 +07:00
|
|
|
u16 new_hash = ipv4_portaddr_hash(sock_net(sk),
|
udp: add rehash on connect()
commit 30fff923 introduced in linux-2.6.33 (udp: bind() optimisation)
added a secondary hash on UDP, hashed on (local addr, local port).
Problem is that following sequence :
fd = socket(...)
connect(fd, &remote, ...)
not only selects remote end point (address and port), but also sets
local address, while UDP stack stored in secondary hash table the socket
while its local address was INADDR_ANY (or ipv6 equivalent)
Sequence is :
- autobind() : choose a random local port, insert socket in hash tables
[while local address is INADDR_ANY]
- connect() : set remote address and port, change local address to IP
given by a route lookup.
When an incoming UDP frame comes, if more than 10 sockets are found in
primary hash table, we switch to secondary table, and fail to find
socket because its local address changed.
One solution to this problem is to rehash datagram socket if needed.
We add a new rehash(struct socket *) method in "struct proto", and
implement this method for UDP v4 & v6, using a common helper.
This rehashing only takes care of secondary hash table, since primary
hash (based on local port only) is not changed.
Reported-by: Krzysztof Piotr Oledzki <ole@ans.pl>
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Tested-by: Krzysztof Piotr Oledzki <ole@ans.pl>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-09-08 12:08:44 +07:00
|
|
|
inet_sk(sk)->inet_rcv_saddr,
|
|
|
|
inet_sk(sk)->inet_num);
|
|
|
|
udp_lib_rehash(sk, new_hash);
|
|
|
|
}
|
|
|
|
|
2017-05-17 19:52:16 +07:00
|
|
|
static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
|
2008-09-16 01:48:46 +07:00
|
|
|
{
|
rfs: Receive Flow Steering
This patch implements receive flow steering (RFS). RFS steers
received packets for layer 3 and 4 processing to the CPU where
the application for the corresponding flow is running. RFS is an
extension of Receive Packet Steering (RPS).
The basic idea of RFS is that when an application calls recvmsg
(or sendmsg) the application's running CPU is stored in a hash
table that is indexed by the connection's rxhash which is stored in
the socket structure. The rxhash is passed in skb's received on
the connection from netif_receive_skb. For each received packet,
the associated rxhash is used to look up the CPU in the hash table,
if a valid CPU is set then the packet is steered to that CPU using
the RPS mechanisms.
The convolution of the simple approach is that it would potentially
allow OOO packets. If threads are thrashing around CPUs or multiple
threads are trying to read from the same sockets, a quickly changing
CPU value in the hash table could cause rampant OOO packets--
we consider this a non-starter.
To avoid OOO packets, this solution implements two types of hash
tables: rps_sock_flow_table and rps_dev_flow_table.
rps_sock_table is a global hash table. Each entry is just a CPU
number and it is populated in recvmsg and sendmsg as described above.
This table contains the "desired" CPUs for flows.
rps_dev_flow_table is specific to each device queue. Each entry
contains a CPU and a tail queue counter. The CPU is the "current"
CPU for a matching flow. The tail queue counter holds the value
of a tail queue counter for the associated CPU's backlog queue at
the time of last enqueue for a flow matching the entry.
Each backlog queue has a queue head counter which is incremented
on dequeue, and so a queue tail counter is computed as queue head
count + queue length. When a packet is enqueued on a backlog queue,
the current value of the queue tail counter is saved in the hash
entry of the rps_dev_flow_table.
And now the trick: when selecting the CPU for RPS (get_rps_cpu)
the rps_sock_flow table and the rps_dev_flow table for the RX queue
are consulted. When the desired CPU for the flow (found in the
rps_sock_flow table) does not match the current CPU (found in the
rps_dev_flow table), the current CPU is changed to the desired CPU
if one of the following is true:
- The current CPU is unset (equal to RPS_NO_CPU)
- Current CPU is offline
- The current CPU's queue head counter >= queue tail counter in the
rps_dev_flow table. This checks if the queue tail has advanced
beyond the last packet that was enqueued using this table entry.
This guarantees that all packets queued using this entry have been
dequeued, thus preserving in order delivery.
Making each queue have its own rps_dev_flow table has two advantages:
1) the tail queue counters will be written on each receive, so
keeping the table local to interrupting CPU s good for locality. 2)
this allows lockless access to the table-- the CPU number and queue
tail counter need to be accessed together under mutual exclusion
from netif_receive_skb, we assume that this is only called from
device napi_poll which is non-reentrant.
This patch implements RFS for TCP and connected UDP sockets.
It should be usable for other flow oriented protocols.
There are two configuration parameters for RFS. The
"rps_flow_entries" kernel init parameter sets the number of
entries in the rps_sock_flow_table, the per rxqueue sysfs entry
"rps_flow_cnt" contains the number of entries in the rps_dev_flow
table for the rxqueue. Both are rounded to power of two.
The obvious benefit of RFS (over just RPS) is that it achieves
CPU locality between the receive processing for a flow and the
applications processing; this can result in increased performance
(higher pps, lower latency).
The benefits of RFS are dependent on cache hierarchy, application
load, and other factors. On simple benchmarks, we don't necessarily
see improvement and sometimes see degradation. However, for more
complex benchmarks and for applications where cache pressure is
much higher this technique seems to perform very well.
Below are some benchmark results which show the potential benfit of
this patch. The netperf test has 500 instances of netperf TCP_RR
test with 1 byte req. and resp. The RPC test is an request/response
test similar in structure to netperf RR test ith 100 threads on
each host, but does more work in userspace that netperf.
e1000e on 8 core Intel
No RFS or RPS 104K tps at 30% CPU
No RFS (best RPS config): 290K tps at 63% CPU
RFS 303K tps at 61% CPU
RPC test tps CPU% 50/90/99% usec latency Latency StdDev
No RFS/RPS 103K 48% 757/900/3185 4472.35
RPS only: 174K 73% 415/993/2468 491.66
RFS 223K 73% 379/651/1382 315.61
Signed-off-by: Tom Herbert <therbert@google.com>
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-04-17 06:01:27 +07:00
|
|
|
int rc;
|
2009-10-15 10:40:11 +07:00
|
|
|
|
2013-10-07 23:01:38 +07:00
|
|
|
if (inet_sk(sk)->inet_daddr) {
|
2011-08-15 02:45:55 +07:00
|
|
|
sock_rps_save_rxhash(sk, skb);
|
2013-10-07 23:01:38 +07:00
|
|
|
sk_mark_napi_id(sk, skb);
|
net: introduce SO_INCOMING_CPU
Alternative to RPS/RFS is to use hardware support for multiple
queues.
Then split a set of million of sockets into worker threads, each
one using epoll() to manage events on its own socket pool.
Ideally, we want one thread per RX/TX queue/cpu, but we have no way to
know after accept() or connect() on which queue/cpu a socket is managed.
We normally use one cpu per RX queue (IRQ smp_affinity being properly
set), so remembering on socket structure which cpu delivered last packet
is enough to solve the problem.
After accept(), connect(), or even file descriptor passing around
processes, applications can use :
int cpu;
socklen_t len = sizeof(cpu);
getsockopt(fd, SOL_SOCKET, SO_INCOMING_CPU, &cpu, &len);
And use this information to put the socket into the right silo
for optimal performance, as all networking stack should run
on the appropriate cpu, without need to send IPI (RPS/RFS).
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-11-11 20:54:28 +07:00
|
|
|
sk_incoming_cpu_update(sk);
|
2016-11-17 00:10:42 +07:00
|
|
|
} else {
|
|
|
|
sk_mark_napi_id_once(sk, skb);
|
2013-10-07 23:01:38 +07:00
|
|
|
}
|
rfs: Receive Flow Steering
This patch implements receive flow steering (RFS). RFS steers
received packets for layer 3 and 4 processing to the CPU where
the application for the corresponding flow is running. RFS is an
extension of Receive Packet Steering (RPS).
The basic idea of RFS is that when an application calls recvmsg
(or sendmsg) the application's running CPU is stored in a hash
table that is indexed by the connection's rxhash which is stored in
the socket structure. The rxhash is passed in skb's received on
the connection from netif_receive_skb. For each received packet,
the associated rxhash is used to look up the CPU in the hash table,
if a valid CPU is set then the packet is steered to that CPU using
the RPS mechanisms.
The convolution of the simple approach is that it would potentially
allow OOO packets. If threads are thrashing around CPUs or multiple
threads are trying to read from the same sockets, a quickly changing
CPU value in the hash table could cause rampant OOO packets--
we consider this a non-starter.
To avoid OOO packets, this solution implements two types of hash
tables: rps_sock_flow_table and rps_dev_flow_table.
rps_sock_table is a global hash table. Each entry is just a CPU
number and it is populated in recvmsg and sendmsg as described above.
This table contains the "desired" CPUs for flows.
rps_dev_flow_table is specific to each device queue. Each entry
contains a CPU and a tail queue counter. The CPU is the "current"
CPU for a matching flow. The tail queue counter holds the value
of a tail queue counter for the associated CPU's backlog queue at
the time of last enqueue for a flow matching the entry.
Each backlog queue has a queue head counter which is incremented
on dequeue, and so a queue tail counter is computed as queue head
count + queue length. When a packet is enqueued on a backlog queue,
the current value of the queue tail counter is saved in the hash
entry of the rps_dev_flow_table.
And now the trick: when selecting the CPU for RPS (get_rps_cpu)
the rps_sock_flow table and the rps_dev_flow table for the RX queue
are consulted. When the desired CPU for the flow (found in the
rps_sock_flow table) does not match the current CPU (found in the
rps_dev_flow table), the current CPU is changed to the desired CPU
if one of the following is true:
- The current CPU is unset (equal to RPS_NO_CPU)
- Current CPU is offline
- The current CPU's queue head counter >= queue tail counter in the
rps_dev_flow table. This checks if the queue tail has advanced
beyond the last packet that was enqueued using this table entry.
This guarantees that all packets queued using this entry have been
dequeued, thus preserving in order delivery.
Making each queue have its own rps_dev_flow table has two advantages:
1) the tail queue counters will be written on each receive, so
keeping the table local to interrupting CPU s good for locality. 2)
this allows lockless access to the table-- the CPU number and queue
tail counter need to be accessed together under mutual exclusion
from netif_receive_skb, we assume that this is only called from
device napi_poll which is non-reentrant.
This patch implements RFS for TCP and connected UDP sockets.
It should be usable for other flow oriented protocols.
There are two configuration parameters for RFS. The
"rps_flow_entries" kernel init parameter sets the number of
entries in the rps_sock_flow_table, the per rxqueue sysfs entry
"rps_flow_cnt" contains the number of entries in the rps_dev_flow
table for the rxqueue. Both are rounded to power of two.
The obvious benefit of RFS (over just RPS) is that it achieves
CPU locality between the receive processing for a flow and the
applications processing; this can result in increased performance
(higher pps, lower latency).
The benefits of RFS are dependent on cache hierarchy, application
load, and other factors. On simple benchmarks, we don't necessarily
see improvement and sometimes see degradation. However, for more
complex benchmarks and for applications where cache pressure is
much higher this technique seems to perform very well.
Below are some benchmark results which show the potential benfit of
this patch. The netperf test has 500 instances of netperf TCP_RR
test with 1 byte req. and resp. The RPC test is an request/response
test similar in structure to netperf RR test ith 100 threads on
each host, but does more work in userspace that netperf.
e1000e on 8 core Intel
No RFS or RPS 104K tps at 30% CPU
No RFS (best RPS config): 290K tps at 63% CPU
RFS 303K tps at 61% CPU
RPC test tps CPU% 50/90/99% usec latency Latency StdDev
No RFS/RPS 103K 48% 757/900/3185 4472.35
RPS only: 174K 73% 415/993/2468 491.66
RFS 223K 73% 379/651/1382 315.61
Signed-off-by: Tom Herbert <therbert@google.com>
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-04-17 06:01:27 +07:00
|
|
|
|
2016-10-21 18:55:47 +07:00
|
|
|
rc = __udp_enqueue_schedule_skb(sk, skb);
|
2009-10-15 10:40:11 +07:00
|
|
|
if (rc < 0) {
|
|
|
|
int is_udplite = IS_UDPLITE(sk);
|
2008-09-16 01:48:46 +07:00
|
|
|
|
|
|
|
/* Note that an ENOMEM error is charged twice */
|
2009-10-15 10:40:11 +07:00
|
|
|
if (rc == -ENOMEM)
|
2016-04-30 04:16:50 +07:00
|
|
|
UDP_INC_STATS(sock_net(sk), UDP_MIB_RCVBUFERRORS,
|
2016-04-28 06:44:30 +07:00
|
|
|
is_udplite);
|
2016-04-30 04:16:50 +07:00
|
|
|
UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
|
2009-10-15 10:40:11 +07:00
|
|
|
kfree_skb(skb);
|
2011-06-17 18:58:39 +07:00
|
|
|
trace_udp_fail_queue_rcv_skb(rc, sk);
|
2009-10-15 10:40:11 +07:00
|
|
|
return -1;
|
2008-09-16 01:48:46 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-05-08 23:07:03 +07:00
|
|
|
static DEFINE_STATIC_KEY_FALSE(udp_encap_needed_key);
|
2012-04-12 06:05:28 +07:00
|
|
|
void udp_encap_enable(void)
|
|
|
|
{
|
2018-05-08 23:07:03 +07:00
|
|
|
static_branch_enable(&udp_encap_needed_key);
|
2012-04-12 06:05:28 +07:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(udp_encap_enable);
|
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
/* returns:
|
|
|
|
* -1: error
|
|
|
|
* 0: success
|
|
|
|
* >0: "udp encap" protocol resubmission
|
|
|
|
*
|
|
|
|
* Note that in the success and error cases, the skb is assumed to
|
|
|
|
* have either been requeued or freed.
|
|
|
|
*/
|
2017-05-17 19:52:16 +07:00
|
|
|
static int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
|
2008-03-07 07:22:02 +07:00
|
|
|
{
|
|
|
|
struct udp_sock *up = udp_sk(sk);
|
|
|
|
int is_udplite = IS_UDPLITE(sk);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Charge it to the socket, dropping if the queue is full.
|
|
|
|
*/
|
|
|
|
if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
|
|
|
|
goto drop;
|
|
|
|
nf_reset(skb);
|
|
|
|
|
2018-05-08 23:07:03 +07:00
|
|
|
if (static_branch_unlikely(&udp_encap_needed_key) && up->encap_type) {
|
2011-11-01 19:56:59 +07:00
|
|
|
int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
|
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
/*
|
|
|
|
* This is an encapsulation socket so pass the skb to
|
|
|
|
* the socket's udp_encap_rcv() hook. Otherwise, just
|
|
|
|
* fall through and pass this up the UDP socket.
|
|
|
|
* up->encap_rcv() returns the following value:
|
|
|
|
* =0 if skb was successfully passed to the encap
|
|
|
|
* handler or was discarded by it.
|
|
|
|
* >0 if skb should be passed on to UDP.
|
|
|
|
* <0 if skb should be resubmitted as proto -N
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* if we're overly short, let UDP handle it */
|
locking/atomics: COCCINELLE/treewide: Convert trivial ACCESS_ONCE() patterns to READ_ONCE()/WRITE_ONCE()
Please do not apply this to mainline directly, instead please re-run the
coccinelle script shown below and apply its output.
For several reasons, it is desirable to use {READ,WRITE}_ONCE() in
preference to ACCESS_ONCE(), and new code is expected to use one of the
former. So far, there's been no reason to change most existing uses of
ACCESS_ONCE(), as these aren't harmful, and changing them results in
churn.
However, for some features, the read/write distinction is critical to
correct operation. To distinguish these cases, separate read/write
accessors must be used. This patch migrates (most) remaining
ACCESS_ONCE() instances to {READ,WRITE}_ONCE(), using the following
coccinelle script:
----
// Convert trivial ACCESS_ONCE() uses to equivalent READ_ONCE() and
// WRITE_ONCE()
// $ make coccicheck COCCI=/home/mark/once.cocci SPFLAGS="--include-headers" MODE=patch
virtual patch
@ depends on patch @
expression E1, E2;
@@
- ACCESS_ONCE(E1) = E2
+ WRITE_ONCE(E1, E2)
@ depends on patch @
expression E;
@@
- ACCESS_ONCE(E)
+ READ_ONCE(E)
----
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: davem@davemloft.net
Cc: linux-arch@vger.kernel.org
Cc: mpe@ellerman.id.au
Cc: shuah@kernel.org
Cc: snitzer@redhat.com
Cc: thor.thayer@linux.intel.com
Cc: tj@kernel.org
Cc: viro@zeniv.linux.org.uk
Cc: will.deacon@arm.com
Link: http://lkml.kernel.org/r/1508792849-3115-19-git-send-email-paulmck@linux.vnet.ibm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-10-24 04:07:29 +07:00
|
|
|
encap_rcv = READ_ONCE(up->encap_rcv);
|
2016-05-19 20:58:33 +07:00
|
|
|
if (encap_rcv) {
|
2008-03-07 07:22:02 +07:00
|
|
|
int ret;
|
|
|
|
|
2014-05-08 06:52:39 +07:00
|
|
|
/* Verify checksum before giving to encap */
|
|
|
|
if (udp_lib_checksum_complete(skb))
|
|
|
|
goto csum_error;
|
|
|
|
|
2011-11-01 19:56:59 +07:00
|
|
|
ret = encap_rcv(sk, skb);
|
2008-03-07 07:22:02 +07:00
|
|
|
if (ret <= 0) {
|
2016-04-28 06:44:30 +07:00
|
|
|
__UDP_INC_STATS(sock_net(sk),
|
|
|
|
UDP_MIB_INDATAGRAMS,
|
|
|
|
is_udplite);
|
2008-03-07 07:22:02 +07:00
|
|
|
return -ret;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* FALLTHROUGH -- it's a UDP Packet */
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* UDP-Lite specific tests, ignored on UDP sockets
|
|
|
|
*/
|
|
|
|
if ((is_udplite & UDPLITE_RECV_CC) && UDP_SKB_CB(skb)->partial_cov) {
|
|
|
|
|
|
|
|
/*
|
|
|
|
* MIB statistics other than incrementing the error count are
|
|
|
|
* disabled for the following two types of errors: these depend
|
|
|
|
* on the application settings, not on the functioning of the
|
|
|
|
* protocol stack as such.
|
|
|
|
*
|
|
|
|
* RFC 3828 here recommends (sec 3.3): "There should also be a
|
|
|
|
* way ... to ... at least let the receiving application block
|
|
|
|
* delivery of packets with coverage values less than a value
|
|
|
|
* provided by the application."
|
|
|
|
*/
|
|
|
|
if (up->pcrlen == 0) { /* full coverage was set */
|
2014-11-12 01:59:17 +07:00
|
|
|
net_dbg_ratelimited("UDPLite: partial coverage %d while full coverage %d requested\n",
|
|
|
|
UDP_SKB_CB(skb)->cscov, skb->len);
|
2008-03-07 07:22:02 +07:00
|
|
|
goto drop;
|
|
|
|
}
|
|
|
|
/* The next case involves violating the min. coverage requested
|
|
|
|
* by the receiver. This is subtle: if receiver wants x and x is
|
|
|
|
* greater than the buffersize/MTU then receiver will complain
|
|
|
|
* that it wants x while sender emits packets of smaller size y.
|
|
|
|
* Therefore the above ...()->partial_cov statement is essential.
|
|
|
|
*/
|
|
|
|
if (UDP_SKB_CB(skb)->cscov < up->pcrlen) {
|
2014-11-12 01:59:17 +07:00
|
|
|
net_dbg_ratelimited("UDPLite: coverage %d too small, need min %d\n",
|
|
|
|
UDP_SKB_CB(skb)->cscov, up->pcrlen);
|
2008-03-07 07:22:02 +07:00
|
|
|
goto drop;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-06-21 15:24:40 +07:00
|
|
|
prefetch(&sk->sk_rmem_alloc);
|
2016-06-03 04:52:43 +07:00
|
|
|
if (rcu_access_pointer(sk->sk_filter) &&
|
|
|
|
udp_lib_checksum_complete(skb))
|
2016-04-05 23:41:15 +07:00
|
|
|
goto csum_error;
|
2016-06-03 04:52:43 +07:00
|
|
|
|
2016-07-25 23:06:12 +07:00
|
|
|
if (sk_filter_trim_cap(sk, skb, sizeof(struct udphdr)))
|
2016-07-08 22:52:33 +07:00
|
|
|
goto drop;
|
2008-03-07 07:22:02 +07:00
|
|
|
|
2016-04-05 23:41:15 +07:00
|
|
|
udp_csum_pull_header(skb);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
2013-10-07 23:01:40 +07:00
|
|
|
ipv4_pktinfo_prepare(sk, skb);
|
2016-10-21 18:55:47 +07:00
|
|
|
return __udp_queue_rcv_skb(sk, skb);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
2013-04-29 15:39:56 +07:00
|
|
|
csum_error:
|
2016-04-28 06:44:30 +07:00
|
|
|
__UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
|
2008-03-07 07:22:02 +07:00
|
|
|
drop:
|
2016-04-28 06:44:30 +07:00
|
|
|
__UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
|
2009-10-15 07:12:40 +07:00
|
|
|
atomic_inc(&sk->sk_drops);
|
2008-03-07 07:22:02 +07:00
|
|
|
kfree_skb(skb);
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2013-12-12 05:46:51 +07:00
|
|
|
/* For TCP sockets, sk_rx_dst is protected by socket lock
|
2013-12-16 01:53:46 +07:00
|
|
|
* For UDP, we use xchg() to guard against concurrent changes.
|
2013-12-12 05:46:51 +07:00
|
|
|
*/
|
2017-08-25 19:31:01 +07:00
|
|
|
bool udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst)
|
2013-10-07 23:01:39 +07:00
|
|
|
{
|
2013-12-12 05:46:51 +07:00
|
|
|
struct dst_entry *old;
|
|
|
|
|
2017-06-18 00:42:25 +07:00
|
|
|
if (dst_hold_safe(dst)) {
|
|
|
|
old = xchg(&sk->sk_rx_dst, dst);
|
|
|
|
dst_release(old);
|
2017-08-25 19:31:01 +07:00
|
|
|
return old != dst;
|
2017-06-18 00:42:25 +07:00
|
|
|
}
|
2017-08-25 19:31:01 +07:00
|
|
|
return false;
|
2013-10-07 23:01:39 +07:00
|
|
|
}
|
2017-07-27 19:45:09 +07:00
|
|
|
EXPORT_SYMBOL(udp_sk_rx_dst_set);
|
2013-10-07 23:01:39 +07:00
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
/*
|
|
|
|
* Multicasts and broadcasts go to each listener.
|
|
|
|
*
|
2009-11-08 17:18:44 +07:00
|
|
|
* Note: called only from the BH handler context.
|
2008-03-07 07:22:02 +07:00
|
|
|
*/
|
2008-06-17 07:12:11 +07:00
|
|
|
static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
|
2008-03-07 07:22:02 +07:00
|
|
|
struct udphdr *uh,
|
|
|
|
__be32 saddr, __be32 daddr,
|
2014-11-07 01:37:54 +07:00
|
|
|
struct udp_table *udptable,
|
|
|
|
int proto)
|
2008-03-07 07:22:02 +07:00
|
|
|
{
|
2016-04-01 22:52:13 +07:00
|
|
|
struct sock *sk, *first = NULL;
|
2014-07-16 10:28:31 +07:00
|
|
|
unsigned short hnum = ntohs(uh->dest);
|
|
|
|
struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum);
|
2014-07-16 10:28:32 +07:00
|
|
|
unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10);
|
2016-04-01 22:52:13 +07:00
|
|
|
unsigned int offset = offsetof(typeof(*sk), sk_node);
|
|
|
|
int dif = skb->dev->ifindex;
|
2017-08-07 22:44:16 +07:00
|
|
|
int sdif = inet_sdif(skb);
|
2016-04-01 22:52:13 +07:00
|
|
|
struct hlist_node *node;
|
|
|
|
struct sk_buff *nskb;
|
2014-07-16 10:28:32 +07:00
|
|
|
|
|
|
|
if (use_hash2) {
|
2017-12-02 03:52:30 +07:00
|
|
|
hash2_any = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum) &
|
2016-11-15 05:40:30 +07:00
|
|
|
udptable->mask;
|
2017-12-02 03:52:30 +07:00
|
|
|
hash2 = ipv4_portaddr_hash(net, daddr, hnum) & udptable->mask;
|
2014-07-16 10:28:32 +07:00
|
|
|
start_lookup:
|
2016-11-15 05:40:30 +07:00
|
|
|
hslot = &udptable->hash2[hash2];
|
2014-07-16 10:28:32 +07:00
|
|
|
offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
|
|
|
|
}
|
2008-03-07 07:22:02 +07:00
|
|
|
|
2016-04-01 22:52:13 +07:00
|
|
|
sk_for_each_entry_offset_rcu(sk, node, &hslot->head, offset) {
|
|
|
|
if (!__udp_is_mcast_sock(net, sk, uh->dest, daddr,
|
2017-08-07 22:44:16 +07:00
|
|
|
uh->source, saddr, dif, sdif, hnum))
|
2016-04-01 22:52:13 +07:00
|
|
|
continue;
|
|
|
|
|
|
|
|
if (!first) {
|
|
|
|
first = sk;
|
|
|
|
continue;
|
2009-11-08 17:18:44 +07:00
|
|
|
}
|
2016-04-01 22:52:13 +07:00
|
|
|
nskb = skb_clone(skb, GFP_ATOMIC);
|
2009-11-08 17:18:44 +07:00
|
|
|
|
2016-04-01 22:52:13 +07:00
|
|
|
if (unlikely(!nskb)) {
|
|
|
|
atomic_inc(&sk->sk_drops);
|
2016-04-28 06:44:30 +07:00
|
|
|
__UDP_INC_STATS(net, UDP_MIB_RCVBUFERRORS,
|
|
|
|
IS_UDPLITE(sk));
|
|
|
|
__UDP_INC_STATS(net, UDP_MIB_INERRORS,
|
|
|
|
IS_UDPLITE(sk));
|
2016-04-01 22:52:13 +07:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (udp_queue_rcv_skb(sk, nskb) > 0)
|
|
|
|
consume_skb(nskb);
|
|
|
|
}
|
2009-11-08 17:18:44 +07:00
|
|
|
|
2014-07-16 10:28:32 +07:00
|
|
|
/* Also lookup *:port if we are using hash2 and haven't done so yet. */
|
|
|
|
if (use_hash2 && hash2 != hash2_any) {
|
|
|
|
hash2 = hash2_any;
|
|
|
|
goto start_lookup;
|
|
|
|
}
|
|
|
|
|
2016-04-01 22:52:13 +07:00
|
|
|
if (first) {
|
|
|
|
if (udp_queue_rcv_skb(first, skb) > 0)
|
|
|
|
consume_skb(skb);
|
2009-11-08 17:18:44 +07:00
|
|
|
} else {
|
2016-04-01 22:52:13 +07:00
|
|
|
kfree_skb(skb);
|
2016-04-28 06:44:30 +07:00
|
|
|
__UDP_INC_STATS(net, UDP_MIB_IGNOREDMULTI,
|
|
|
|
proto == IPPROTO_UDPLITE);
|
2009-11-08 17:18:44 +07:00
|
|
|
}
|
2008-03-07 07:22:02 +07:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Initialize UDP checksum. If exited with zero value (success),
|
|
|
|
* CHECKSUM_UNNECESSARY means, that no more checks are required.
|
|
|
|
* Otherwise, csum completion requires chacksumming packet body,
|
|
|
|
* including udp header and folding it to skb->csum.
|
|
|
|
*/
|
|
|
|
static inline int udp4_csum_init(struct sk_buff *skb, struct udphdr *uh,
|
|
|
|
int proto)
|
|
|
|
{
|
|
|
|
int err;
|
|
|
|
|
|
|
|
UDP_SKB_CB(skb)->partial_cov = 0;
|
|
|
|
UDP_SKB_CB(skb)->cscov = skb->len;
|
|
|
|
|
|
|
|
if (proto == IPPROTO_UDPLITE) {
|
|
|
|
err = udplite_checksum_init(skb, uh);
|
|
|
|
if (err)
|
|
|
|
return err;
|
2018-02-16 00:18:43 +07:00
|
|
|
|
|
|
|
if (UDP_SKB_CB(skb)->partial_cov) {
|
|
|
|
skb->csum = inet_compute_pseudo(skb, proto);
|
|
|
|
return 0;
|
|
|
|
}
|
2008-03-07 07:22:02 +07:00
|
|
|
}
|
|
|
|
|
2016-06-12 17:02:46 +07:00
|
|
|
/* Note, we are only interested in != 0 or == 0, thus the
|
|
|
|
* force to int.
|
|
|
|
*/
|
|
|
|
return (__force int)skb_checksum_init_zero_check(skb, proto, uh->check,
|
|
|
|
inet_compute_pseudo);
|
2008-03-07 07:22:02 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* All we need to do is get the socket, and then do a checksum.
|
|
|
|
*/
|
|
|
|
|
2008-10-29 15:41:45 +07:00
|
|
|
int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
|
2008-03-07 07:22:02 +07:00
|
|
|
int proto)
|
|
|
|
{
|
|
|
|
struct sock *sk;
|
2009-02-06 06:05:45 +07:00
|
|
|
struct udphdr *uh;
|
2008-03-07 07:22:02 +07:00
|
|
|
unsigned short ulen;
|
2009-06-02 12:19:30 +07:00
|
|
|
struct rtable *rt = skb_rtable(skb);
|
2009-02-06 16:59:12 +07:00
|
|
|
__be32 saddr, daddr;
|
2008-07-06 11:18:48 +07:00
|
|
|
struct net *net = dev_net(skb->dev);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Validate the packet.
|
|
|
|
*/
|
|
|
|
if (!pskb_may_pull(skb, sizeof(struct udphdr)))
|
|
|
|
goto drop; /* No space for header. */
|
|
|
|
|
2009-02-06 06:05:45 +07:00
|
|
|
uh = udp_hdr(skb);
|
2008-03-07 07:22:02 +07:00
|
|
|
ulen = ntohs(uh->len);
|
2010-05-06 10:44:34 +07:00
|
|
|
saddr = ip_hdr(skb)->saddr;
|
|
|
|
daddr = ip_hdr(skb)->daddr;
|
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
if (ulen > skb->len)
|
|
|
|
goto short_packet;
|
|
|
|
|
|
|
|
if (proto == IPPROTO_UDP) {
|
|
|
|
/* UDP validates ulen. */
|
|
|
|
if (ulen < sizeof(*uh) || pskb_trim_rcsum(skb, ulen))
|
|
|
|
goto short_packet;
|
|
|
|
uh = udp_hdr(skb);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (udp4_csum_init(skb, uh, proto))
|
|
|
|
goto csum_error;
|
|
|
|
|
2013-12-11 09:07:23 +07:00
|
|
|
sk = skb_steal_sock(skb);
|
|
|
|
if (sk) {
|
2013-12-12 05:46:51 +07:00
|
|
|
struct dst_entry *dst = skb_dst(skb);
|
2013-10-07 23:01:39 +07:00
|
|
|
int ret;
|
|
|
|
|
2013-12-12 05:46:51 +07:00
|
|
|
if (unlikely(sk->sk_rx_dst != dst))
|
|
|
|
udp_sk_rx_dst_set(sk, dst);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
2013-10-07 23:01:39 +07:00
|
|
|
ret = udp_queue_rcv_skb(sk, skb);
|
2013-12-11 09:07:23 +07:00
|
|
|
sock_put(sk);
|
2013-10-07 23:01:39 +07:00
|
|
|
/* a return value > 0 means to resubmit the input, but
|
|
|
|
* it wants the return to be -protocol, or 0
|
|
|
|
*/
|
|
|
|
if (ret > 0)
|
|
|
|
return -ret;
|
|
|
|
return 0;
|
|
|
|
}
|
2008-03-07 07:22:02 +07:00
|
|
|
|
2014-11-05 02:48:41 +07:00
|
|
|
if (rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST))
|
|
|
|
return __udp4_lib_mcast_deliver(net, skb, uh,
|
2014-11-07 01:37:54 +07:00
|
|
|
saddr, daddr, udptable, proto);
|
2014-11-05 02:48:41 +07:00
|
|
|
|
|
|
|
sk = __udp4_lib_lookup_skb(skb, uh->source, uh->dest, udptable);
|
2015-04-03 15:17:27 +07:00
|
|
|
if (sk) {
|
2013-06-10 15:40:00 +07:00
|
|
|
int ret;
|
|
|
|
|
2015-01-06 04:56:14 +07:00
|
|
|
if (inet_get_convert_csum(sk) && uh->check && !IS_UDPLITE(sk))
|
2014-09-01 05:12:43 +07:00
|
|
|
skb_checksum_try_convert(skb, IPPROTO_UDP, uh->check,
|
|
|
|
inet_compute_pseudo);
|
|
|
|
|
2013-06-10 15:40:00 +07:00
|
|
|
ret = udp_queue_rcv_skb(sk, skb);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
|
|
|
/* a return value > 0 means to resubmit the input, but
|
|
|
|
* it wants the return to be -protocol, or 0
|
|
|
|
*/
|
|
|
|
if (ret > 0)
|
|
|
|
return -ret;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
|
|
|
|
goto drop;
|
|
|
|
nf_reset(skb);
|
|
|
|
|
|
|
|
/* No socket. Drop packet silently, if checksum is wrong */
|
|
|
|
if (udp_lib_checksum_complete(skb))
|
|
|
|
goto csum_error;
|
|
|
|
|
2016-04-28 06:44:30 +07:00
|
|
|
__UDP_INC_STATS(net, UDP_MIB_NOPORTS, proto == IPPROTO_UDPLITE);
|
2008-03-07 07:22:02 +07:00
|
|
|
icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Hmm. We got an UDP packet to a port to which we
|
|
|
|
* don't wanna listen. Ignore it.
|
|
|
|
*/
|
|
|
|
kfree_skb(skb);
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
short_packet:
|
2014-11-12 01:59:17 +07:00
|
|
|
net_dbg_ratelimited("UDP%s: short packet: From %pI4:%u %d/%d to %pI4:%u\n",
|
|
|
|
proto == IPPROTO_UDPLITE ? "Lite" : "",
|
|
|
|
&saddr, ntohs(uh->source),
|
|
|
|
ulen, skb->len,
|
|
|
|
&daddr, ntohs(uh->dest));
|
2008-03-07 07:22:02 +07:00
|
|
|
goto drop;
|
|
|
|
|
|
|
|
csum_error:
|
|
|
|
/*
|
|
|
|
* RFC1122: OK. Discards the bad packet silently (as far as
|
|
|
|
* the network is concerned, anyway) as per 4.1.3.4 (MUST).
|
|
|
|
*/
|
2014-11-12 01:59:17 +07:00
|
|
|
net_dbg_ratelimited("UDP%s: bad checksum. From %pI4:%u to %pI4:%u ulen %d\n",
|
|
|
|
proto == IPPROTO_UDPLITE ? "Lite" : "",
|
|
|
|
&saddr, ntohs(uh->source), &daddr, ntohs(uh->dest),
|
|
|
|
ulen);
|
2016-04-28 06:44:30 +07:00
|
|
|
__UDP_INC_STATS(net, UDP_MIB_CSUMERRORS, proto == IPPROTO_UDPLITE);
|
2008-03-07 07:22:02 +07:00
|
|
|
drop:
|
2016-04-28 06:44:30 +07:00
|
|
|
__UDP_INC_STATS(net, UDP_MIB_INERRORS, proto == IPPROTO_UDPLITE);
|
2008-03-07 07:22:02 +07:00
|
|
|
kfree_skb(skb);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2013-10-07 23:01:39 +07:00
|
|
|
/* We can only early demux multicast if there is a single matching socket.
|
|
|
|
* If more than one socket found returns NULL
|
|
|
|
*/
|
|
|
|
static struct sock *__udp4_lib_mcast_demux_lookup(struct net *net,
|
|
|
|
__be16 loc_port, __be32 loc_addr,
|
|
|
|
__be16 rmt_port, __be32 rmt_addr,
|
2017-08-07 22:44:16 +07:00
|
|
|
int dif, int sdif)
|
2013-10-07 23:01:39 +07:00
|
|
|
{
|
|
|
|
struct sock *sk, *result;
|
|
|
|
unsigned short hnum = ntohs(loc_port);
|
2016-04-01 22:52:13 +07:00
|
|
|
unsigned int slot = udp_hashfn(net, hnum, udp_table.mask);
|
2013-10-07 23:01:39 +07:00
|
|
|
struct udp_hslot *hslot = &udp_table.hash[slot];
|
|
|
|
|
2014-06-13 06:13:06 +07:00
|
|
|
/* Do not bother scanning a too big list */
|
|
|
|
if (hslot->count > 10)
|
|
|
|
return NULL;
|
|
|
|
|
2013-10-07 23:01:39 +07:00
|
|
|
result = NULL;
|
2016-04-01 22:52:13 +07:00
|
|
|
sk_for_each_rcu(sk, &hslot->head) {
|
|
|
|
if (__udp_is_mcast_sock(net, sk, loc_port, loc_addr,
|
2017-08-07 22:44:16 +07:00
|
|
|
rmt_port, rmt_addr, dif, sdif, hnum)) {
|
2016-04-01 22:52:13 +07:00
|
|
|
if (result)
|
|
|
|
return NULL;
|
2013-10-07 23:01:39 +07:00
|
|
|
result = sk;
|
|
|
|
}
|
|
|
|
}
|
2016-04-01 22:52:13 +07:00
|
|
|
|
2013-10-07 23:01:39 +07:00
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* For unicast we should only early demux connected sockets or we can
|
|
|
|
* break forwarding setups. The chains here can be long so only check
|
|
|
|
* if the first socket is an exact match and if not move on.
|
|
|
|
*/
|
|
|
|
static struct sock *__udp4_lib_demux_lookup(struct net *net,
|
|
|
|
__be16 loc_port, __be32 loc_addr,
|
|
|
|
__be16 rmt_port, __be32 rmt_addr,
|
2017-08-07 22:44:17 +07:00
|
|
|
int dif, int sdif)
|
2013-10-07 23:01:39 +07:00
|
|
|
{
|
|
|
|
unsigned short hnum = ntohs(loc_port);
|
2017-12-02 03:52:30 +07:00
|
|
|
unsigned int hash2 = ipv4_portaddr_hash(net, loc_addr, hnum);
|
2013-10-07 23:01:39 +07:00
|
|
|
unsigned int slot2 = hash2 & udp_table.mask;
|
|
|
|
struct udp_hslot *hslot2 = &udp_table.hash2[slot2];
|
2014-05-14 10:30:07 +07:00
|
|
|
INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr);
|
2013-10-07 23:01:39 +07:00
|
|
|
const __portpair ports = INET_COMBINED_PORTS(rmt_port, hnum);
|
2016-04-01 22:52:13 +07:00
|
|
|
struct sock *sk;
|
2013-10-07 23:01:39 +07:00
|
|
|
|
2016-04-01 22:52:13 +07:00
|
|
|
udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
|
|
|
|
if (INET_MATCH(sk, net, acookie, rmt_addr,
|
2017-08-07 22:44:17 +07:00
|
|
|
loc_addr, ports, dif, sdif))
|
2016-04-01 22:52:13 +07:00
|
|
|
return sk;
|
2013-10-07 23:01:39 +07:00
|
|
|
/* Only check first socket in chain */
|
|
|
|
break;
|
|
|
|
}
|
2016-04-01 22:52:13 +07:00
|
|
|
return NULL;
|
2013-10-07 23:01:39 +07:00
|
|
|
}
|
|
|
|
|
2017-09-28 20:51:36 +07:00
|
|
|
int udp_v4_early_demux(struct sk_buff *skb)
|
2013-10-07 23:01:39 +07:00
|
|
|
{
|
2013-12-11 23:10:05 +07:00
|
|
|
struct net *net = dev_net(skb->dev);
|
2017-09-28 20:51:37 +07:00
|
|
|
struct in_device *in_dev = NULL;
|
2013-12-11 23:10:05 +07:00
|
|
|
const struct iphdr *iph;
|
|
|
|
const struct udphdr *uh;
|
2016-04-01 22:52:13 +07:00
|
|
|
struct sock *sk = NULL;
|
2013-10-07 23:01:39 +07:00
|
|
|
struct dst_entry *dst;
|
|
|
|
int dif = skb->dev->ifindex;
|
2017-08-07 22:44:16 +07:00
|
|
|
int sdif = inet_sdif(skb);
|
2015-06-04 04:27:38 +07:00
|
|
|
int ours;
|
2013-10-07 23:01:39 +07:00
|
|
|
|
|
|
|
/* validate the packet */
|
|
|
|
if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct udphdr)))
|
2017-09-28 20:51:36 +07:00
|
|
|
return 0;
|
2013-10-07 23:01:39 +07:00
|
|
|
|
2013-12-11 23:10:05 +07:00
|
|
|
iph = ip_hdr(skb);
|
|
|
|
uh = udp_hdr(skb);
|
|
|
|
|
2017-10-09 19:52:10 +07:00
|
|
|
if (skb->pkt_type == PACKET_MULTICAST) {
|
2017-09-28 20:51:37 +07:00
|
|
|
in_dev = __in_dev_get_rcu(skb->dev);
|
2015-06-04 04:27:38 +07:00
|
|
|
|
|
|
|
if (!in_dev)
|
2017-09-28 20:51:36 +07:00
|
|
|
return 0;
|
2015-06-04 04:27:38 +07:00
|
|
|
|
2017-10-09 19:52:10 +07:00
|
|
|
ours = ip_check_mc_rcu(in_dev, iph->daddr, iph->saddr,
|
|
|
|
iph->protocol);
|
|
|
|
if (!ours)
|
|
|
|
return 0;
|
2016-03-22 15:19:38 +07:00
|
|
|
|
2013-10-07 23:01:39 +07:00
|
|
|
sk = __udp4_lib_mcast_demux_lookup(net, uh->dest, iph->daddr,
|
2017-08-07 22:44:16 +07:00
|
|
|
uh->source, iph->saddr,
|
|
|
|
dif, sdif);
|
2015-06-04 04:27:38 +07:00
|
|
|
} else if (skb->pkt_type == PACKET_HOST) {
|
2013-10-07 23:01:39 +07:00
|
|
|
sk = __udp4_lib_demux_lookup(net, uh->dest, iph->daddr,
|
2017-08-07 22:44:17 +07:00
|
|
|
uh->source, iph->saddr, dif, sdif);
|
2015-06-04 04:27:38 +07:00
|
|
|
}
|
2013-10-07 23:01:39 +07:00
|
|
|
|
2017-06-30 17:08:01 +07:00
|
|
|
if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt))
|
2017-09-28 20:51:36 +07:00
|
|
|
return 0;
|
2013-10-07 23:01:39 +07:00
|
|
|
|
|
|
|
skb->sk = sk;
|
2014-09-05 00:32:11 +07:00
|
|
|
skb->destructor = sock_efree;
|
2015-08-01 17:14:33 +07:00
|
|
|
dst = READ_ONCE(sk->sk_rx_dst);
|
2013-10-07 23:01:39 +07:00
|
|
|
|
|
|
|
if (dst)
|
|
|
|
dst = dst_check(dst, 0);
|
2015-08-01 17:14:33 +07:00
|
|
|
if (dst) {
|
2017-09-28 20:51:37 +07:00
|
|
|
u32 itag = 0;
|
|
|
|
|
2017-06-18 00:42:25 +07:00
|
|
|
/* set noref for now.
|
|
|
|
* any place which wants to hold dst has to call
|
|
|
|
* dst_hold_safe()
|
|
|
|
*/
|
|
|
|
skb_dst_set_noref(skb, dst);
|
2017-09-28 20:51:37 +07:00
|
|
|
|
|
|
|
/* for unconnected multicast sockets we need to validate
|
|
|
|
* the source on each packet
|
|
|
|
*/
|
|
|
|
if (!inet_sk(sk)->inet_daddr && in_dev)
|
|
|
|
return ip_mc_validate_source(skb, iph->daddr,
|
|
|
|
iph->saddr, iph->tos,
|
|
|
|
skb->dev, in_dev, &itag);
|
2015-08-01 17:14:33 +07:00
|
|
|
}
|
2017-09-28 20:51:36 +07:00
|
|
|
return 0;
|
2013-10-07 23:01:39 +07:00
|
|
|
}
|
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
int udp_rcv(struct sk_buff *skb)
|
|
|
|
{
|
2008-10-29 15:41:45 +07:00
|
|
|
return __udp4_lib_rcv(skb, &udp_table, IPPROTO_UDP);
|
2008-03-07 07:22:02 +07:00
|
|
|
}
|
|
|
|
|
2008-06-15 07:04:49 +07:00
|
|
|
void udp_destroy_sock(struct sock *sk)
|
2008-03-07 07:22:02 +07:00
|
|
|
{
|
2013-03-19 13:11:12 +07:00
|
|
|
struct udp_sock *up = udp_sk(sk);
|
2010-05-27 02:20:18 +07:00
|
|
|
bool slow = lock_sock_fast(sk);
|
2008-03-07 07:22:02 +07:00
|
|
|
udp_flush_pending_frames(sk);
|
2010-05-27 02:20:18 +07:00
|
|
|
unlock_sock_fast(sk, slow);
|
2018-05-08 23:07:03 +07:00
|
|
|
if (static_branch_unlikely(&udp_encap_needed_key) && up->encap_type) {
|
2013-03-19 13:11:12 +07:00
|
|
|
void (*encap_destroy)(struct sock *sk);
|
locking/atomics: COCCINELLE/treewide: Convert trivial ACCESS_ONCE() patterns to READ_ONCE()/WRITE_ONCE()
Please do not apply this to mainline directly, instead please re-run the
coccinelle script shown below and apply its output.
For several reasons, it is desirable to use {READ,WRITE}_ONCE() in
preference to ACCESS_ONCE(), and new code is expected to use one of the
former. So far, there's been no reason to change most existing uses of
ACCESS_ONCE(), as these aren't harmful, and changing them results in
churn.
However, for some features, the read/write distinction is critical to
correct operation. To distinguish these cases, separate read/write
accessors must be used. This patch migrates (most) remaining
ACCESS_ONCE() instances to {READ,WRITE}_ONCE(), using the following
coccinelle script:
----
// Convert trivial ACCESS_ONCE() uses to equivalent READ_ONCE() and
// WRITE_ONCE()
// $ make coccicheck COCCI=/home/mark/once.cocci SPFLAGS="--include-headers" MODE=patch
virtual patch
@ depends on patch @
expression E1, E2;
@@
- ACCESS_ONCE(E1) = E2
+ WRITE_ONCE(E1, E2)
@ depends on patch @
expression E;
@@
- ACCESS_ONCE(E)
+ READ_ONCE(E)
----
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: davem@davemloft.net
Cc: linux-arch@vger.kernel.org
Cc: mpe@ellerman.id.au
Cc: shuah@kernel.org
Cc: snitzer@redhat.com
Cc: thor.thayer@linux.intel.com
Cc: tj@kernel.org
Cc: viro@zeniv.linux.org.uk
Cc: will.deacon@arm.com
Link: http://lkml.kernel.org/r/1508792849-3115-19-git-send-email-paulmck@linux.vnet.ibm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-10-24 04:07:29 +07:00
|
|
|
encap_destroy = READ_ONCE(up->encap_destroy);
|
2013-03-19 13:11:12 +07:00
|
|
|
if (encap_destroy)
|
|
|
|
encap_destroy(sk);
|
|
|
|
}
|
2008-03-07 07:22:02 +07:00
|
|
|
}
|
|
|
|
|
2005-04-17 05:20:36 +07:00
|
|
|
/*
|
|
|
|
* Socket option code for UDP
|
|
|
|
*/
|
2006-11-28 00:29:59 +07:00
|
|
|
int udp_lib_setsockopt(struct sock *sk, int level, int optname,
|
2009-10-01 06:12:20 +07:00
|
|
|
char __user *optval, unsigned int optlen,
|
2006-11-28 00:29:59 +07:00
|
|
|
int (*push_pending_frames)(struct sock *))
|
2005-04-17 05:20:36 +07:00
|
|
|
{
|
|
|
|
struct udp_sock *up = udp_sk(sk);
|
2014-05-23 22:47:32 +07:00
|
|
|
int val, valbool;
|
2005-04-17 05:20:36 +07:00
|
|
|
int err = 0;
|
2007-12-03 18:34:16 +07:00
|
|
|
int is_udplite = IS_UDPLITE(sk);
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2009-07-17 07:26:32 +07:00
|
|
|
if (optlen < sizeof(int))
|
2005-04-17 05:20:36 +07:00
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (get_user(val, (int __user *)optval))
|
|
|
|
return -EFAULT;
|
|
|
|
|
2014-05-23 22:47:32 +07:00
|
|
|
valbool = val ? 1 : 0;
|
|
|
|
|
2007-03-09 11:41:55 +07:00
|
|
|
switch (optname) {
|
2005-04-17 05:20:36 +07:00
|
|
|
case UDP_CORK:
|
|
|
|
if (val != 0) {
|
|
|
|
up->corkflag = 1;
|
|
|
|
} else {
|
|
|
|
up->corkflag = 0;
|
|
|
|
lock_sock(sk);
|
2014-11-12 12:59:20 +07:00
|
|
|
push_pending_frames(sk);
|
2005-04-17 05:20:36 +07:00
|
|
|
release_sock(sk);
|
|
|
|
}
|
|
|
|
break;
|
2007-02-09 21:24:47 +07:00
|
|
|
|
2005-04-17 05:20:36 +07:00
|
|
|
case UDP_ENCAP:
|
|
|
|
switch (val) {
|
|
|
|
case 0:
|
|
|
|
case UDP_ENCAP_ESPINUDP:
|
|
|
|
case UDP_ENCAP_ESPINUDP_NON_IKE:
|
2007-07-06 07:08:05 +07:00
|
|
|
up->encap_rcv = xfrm4_udp_encap_rcv;
|
|
|
|
/* FALLTHROUGH */
|
2007-06-28 05:37:46 +07:00
|
|
|
case UDP_ENCAP_L2TPINUDP:
|
2005-04-17 05:20:36 +07:00
|
|
|
up->encap_type = val;
|
2012-04-12 06:05:28 +07:00
|
|
|
udp_encap_enable();
|
2005-04-17 05:20:36 +07:00
|
|
|
break;
|
|
|
|
default:
|
|
|
|
err = -ENOPROTOOPT;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
2014-05-23 22:47:32 +07:00
|
|
|
case UDP_NO_CHECK6_TX:
|
|
|
|
up->no_check6_tx = valbool;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case UDP_NO_CHECK6_RX:
|
|
|
|
up->no_check6_rx = valbool;
|
|
|
|
break;
|
|
|
|
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-27 00:42:17 +07:00
|
|
|
case UDP_SEGMENT:
|
|
|
|
if (val < 0 || val > USHRT_MAX)
|
|
|
|
return -EINVAL;
|
|
|
|
up->gso_size = val;
|
|
|
|
break;
|
|
|
|
|
2006-11-28 02:10:57 +07:00
|
|
|
/*
|
|
|
|
* UDP-Lite's partial checksum coverage (RFC 3828).
|
|
|
|
*/
|
|
|
|
/* The sender sets actual checksum coverage length via this option.
|
|
|
|
* The case coverage > packet length is handled by send module. */
|
|
|
|
case UDPLITE_SEND_CSCOV:
|
2007-12-03 18:34:16 +07:00
|
|
|
if (!is_udplite) /* Disable the option on UDP sockets */
|
2006-11-28 02:10:57 +07:00
|
|
|
return -ENOPROTOOPT;
|
|
|
|
if (val != 0 && val < 8) /* Illegal coverage: use default (8) */
|
|
|
|
val = 8;
|
2010-05-25 04:33:03 +07:00
|
|
|
else if (val > USHRT_MAX)
|
|
|
|
val = USHRT_MAX;
|
2006-11-28 02:10:57 +07:00
|
|
|
up->pcslen = val;
|
|
|
|
up->pcflag |= UDPLITE_SEND_CC;
|
|
|
|
break;
|
|
|
|
|
2007-02-09 21:24:47 +07:00
|
|
|
/* The receiver specifies a minimum checksum coverage value. To make
|
|
|
|
* sense, this should be set to at least 8 (as done below). If zero is
|
2006-11-28 02:10:57 +07:00
|
|
|
* used, this again means full checksum coverage. */
|
|
|
|
case UDPLITE_RECV_CSCOV:
|
2007-12-03 18:34:16 +07:00
|
|
|
if (!is_udplite) /* Disable the option on UDP sockets */
|
2006-11-28 02:10:57 +07:00
|
|
|
return -ENOPROTOOPT;
|
|
|
|
if (val != 0 && val < 8) /* Avoid silly minimal values. */
|
|
|
|
val = 8;
|
2010-05-25 04:33:03 +07:00
|
|
|
else if (val > USHRT_MAX)
|
|
|
|
val = USHRT_MAX;
|
2006-11-28 02:10:57 +07:00
|
|
|
up->pcrlen = val;
|
|
|
|
up->pcflag |= UDPLITE_RECV_CC;
|
|
|
|
break;
|
|
|
|
|
2005-04-17 05:20:36 +07:00
|
|
|
default:
|
|
|
|
err = -ENOPROTOOPT;
|
|
|
|
break;
|
2007-03-09 11:41:55 +07:00
|
|
|
}
|
2005-04-17 05:20:36 +07:00
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
2009-07-17 07:26:32 +07:00
|
|
|
EXPORT_SYMBOL(udp_lib_setsockopt);
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
int udp_setsockopt(struct sock *sk, int level, int optname,
|
2009-10-01 06:12:20 +07:00
|
|
|
char __user *optval, unsigned int optlen)
|
2008-03-07 07:22:02 +07:00
|
|
|
{
|
|
|
|
if (level == SOL_UDP || level == SOL_UDPLITE)
|
|
|
|
return udp_lib_setsockopt(sk, level, optname, optval, optlen,
|
|
|
|
udp_push_pending_frames);
|
|
|
|
return ip_setsockopt(sk, level, optname, optval, optlen);
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_COMPAT
|
|
|
|
int compat_udp_setsockopt(struct sock *sk, int level, int optname,
|
2009-10-01 06:12:20 +07:00
|
|
|
char __user *optval, unsigned int optlen)
|
2008-03-07 07:22:02 +07:00
|
|
|
{
|
|
|
|
if (level == SOL_UDP || level == SOL_UDPLITE)
|
|
|
|
return udp_lib_setsockopt(sk, level, optname, optval, optlen,
|
|
|
|
udp_push_pending_frames);
|
|
|
|
return compat_ip_setsockopt(sk, level, optname, optval, optlen);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2006-11-28 00:29:59 +07:00
|
|
|
int udp_lib_getsockopt(struct sock *sk, int level, int optname,
|
|
|
|
char __user *optval, int __user *optlen)
|
2005-04-17 05:20:36 +07:00
|
|
|
{
|
|
|
|
struct udp_sock *up = udp_sk(sk);
|
|
|
|
int val, len;
|
|
|
|
|
2009-07-17 07:26:32 +07:00
|
|
|
if (get_user(len, optlen))
|
2005-04-17 05:20:36 +07:00
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
len = min_t(unsigned int, len, sizeof(int));
|
2007-02-09 21:24:47 +07:00
|
|
|
|
2007-03-09 11:41:55 +07:00
|
|
|
if (len < 0)
|
2005-04-17 05:20:36 +07:00
|
|
|
return -EINVAL;
|
|
|
|
|
2007-03-09 11:41:55 +07:00
|
|
|
switch (optname) {
|
2005-04-17 05:20:36 +07:00
|
|
|
case UDP_CORK:
|
|
|
|
val = up->corkflag;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case UDP_ENCAP:
|
|
|
|
val = up->encap_type;
|
|
|
|
break;
|
|
|
|
|
2014-05-23 22:47:32 +07:00
|
|
|
case UDP_NO_CHECK6_TX:
|
|
|
|
val = up->no_check6_tx;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case UDP_NO_CHECK6_RX:
|
|
|
|
val = up->no_check6_rx;
|
|
|
|
break;
|
|
|
|
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-27 00:42:17 +07:00
|
|
|
case UDP_SEGMENT:
|
|
|
|
val = up->gso_size;
|
|
|
|
break;
|
|
|
|
|
2006-11-28 02:10:57 +07:00
|
|
|
/* The following two cannot be changed on UDP sockets, the return is
|
|
|
|
* always 0 (which corresponds to the full checksum coverage of UDP). */
|
|
|
|
case UDPLITE_SEND_CSCOV:
|
|
|
|
val = up->pcslen;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case UDPLITE_RECV_CSCOV:
|
|
|
|
val = up->pcrlen;
|
|
|
|
break;
|
|
|
|
|
2005-04-17 05:20:36 +07:00
|
|
|
default:
|
|
|
|
return -ENOPROTOOPT;
|
2007-03-09 11:41:55 +07:00
|
|
|
}
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2007-03-09 11:41:55 +07:00
|
|
|
if (put_user(len, optlen))
|
2007-02-09 21:24:47 +07:00
|
|
|
return -EFAULT;
|
2009-07-17 07:26:32 +07:00
|
|
|
if (copy_to_user(optval, &val, len))
|
2005-04-17 05:20:36 +07:00
|
|
|
return -EFAULT;
|
2007-02-09 21:24:47 +07:00
|
|
|
return 0;
|
2005-04-17 05:20:36 +07:00
|
|
|
}
|
2009-07-17 07:26:32 +07:00
|
|
|
EXPORT_SYMBOL(udp_lib_getsockopt);
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
int udp_getsockopt(struct sock *sk, int level, int optname,
|
|
|
|
char __user *optval, int __user *optlen)
|
|
|
|
{
|
|
|
|
if (level == SOL_UDP || level == SOL_UDPLITE)
|
|
|
|
return udp_lib_getsockopt(sk, level, optname, optval, optlen);
|
|
|
|
return ip_getsockopt(sk, level, optname, optval, optlen);
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_COMPAT
|
|
|
|
int compat_udp_getsockopt(struct sock *sk, int level, int optname,
|
|
|
|
char __user *optval, int __user *optlen)
|
|
|
|
{
|
|
|
|
if (level == SOL_UDP || level == SOL_UDPLITE)
|
|
|
|
return udp_lib_getsockopt(sk, level, optname, optval, optlen);
|
|
|
|
return compat_ip_getsockopt(sk, level, optname, optval, optlen);
|
|
|
|
}
|
|
|
|
#endif
|
2005-04-17 05:20:36 +07:00
|
|
|
/**
|
|
|
|
* udp_poll - wait for a UDP event.
|
|
|
|
* @file - file struct
|
|
|
|
* @sock - socket
|
2018-06-28 23:43:44 +07:00
|
|
|
* @wait - poll table
|
2005-04-17 05:20:36 +07:00
|
|
|
*
|
2007-02-09 21:24:47 +07:00
|
|
|
* This is same as datagram poll, except for the special case of
|
2005-04-17 05:20:36 +07:00
|
|
|
* blocking sockets. If application is using a blocking fd
|
|
|
|
* and a packet with checksum error is in the queue;
|
|
|
|
* then it could get return from select indicating data available
|
|
|
|
* but then block when reading it. Add special case code
|
|
|
|
* to work around these arguably broken applications.
|
|
|
|
*/
|
2018-06-28 23:43:44 +07:00
|
|
|
__poll_t udp_poll(struct file *file, struct socket *sock, poll_table *wait)
|
2005-04-17 05:20:36 +07:00
|
|
|
{
|
2018-06-28 23:43:44 +07:00
|
|
|
__poll_t mask = datagram_poll(file, sock, wait);
|
2005-04-17 05:20:36 +07:00
|
|
|
struct sock *sk = sock->sk;
|
2006-11-28 02:10:57 +07:00
|
|
|
|
2017-05-16 16:20:14 +07:00
|
|
|
if (!skb_queue_empty(&udp_sk(sk)->reader_queue))
|
2018-02-12 05:34:03 +07:00
|
|
|
mask |= EPOLLIN | EPOLLRDNORM;
|
2017-05-16 16:20:14 +07:00
|
|
|
|
2005-04-17 05:20:36 +07:00
|
|
|
/* Check for false positives due to checksum errors */
|
2018-06-28 23:43:44 +07:00
|
|
|
if ((mask & EPOLLRDNORM) && !(file->f_flags & O_NONBLOCK) &&
|
2016-08-24 03:59:33 +07:00
|
|
|
!(sk->sk_shutdown & RCV_SHUTDOWN) && first_packet_length(sk) == -1)
|
2018-02-12 05:34:03 +07:00
|
|
|
mask &= ~(EPOLLIN | EPOLLRDNORM);
|
2005-04-17 05:20:36 +07:00
|
|
|
|
|
|
|
return mask;
|
2007-02-09 21:24:47 +07:00
|
|
|
|
2005-04-17 05:20:36 +07:00
|
|
|
}
|
2018-06-28 23:43:44 +07:00
|
|
|
EXPORT_SYMBOL(udp_poll);
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2016-08-24 11:06:33 +07:00
|
|
|
int udp_abort(struct sock *sk, int err)
|
|
|
|
{
|
|
|
|
lock_sock(sk);
|
|
|
|
|
|
|
|
sk->sk_err = err;
|
|
|
|
sk->sk_error_report(sk);
|
2016-10-20 23:39:40 +07:00
|
|
|
__udp_disconnect(sk, 0);
|
2016-08-24 11:06:33 +07:00
|
|
|
|
|
|
|
release_sock(sk);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(udp_abort);
|
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
struct proto udp_prot = {
|
2018-03-14 11:57:16 +07:00
|
|
|
.name = "UDP",
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
.close = udp_lib_close,
|
2018-03-31 05:08:05 +07:00
|
|
|
.pre_connect = udp_pre_connect,
|
2018-03-14 11:57:16 +07:00
|
|
|
.connect = ip4_datagram_connect,
|
|
|
|
.disconnect = udp_disconnect,
|
|
|
|
.ioctl = udp_ioctl,
|
|
|
|
.init = udp_init_sock,
|
|
|
|
.destroy = udp_destroy_sock,
|
|
|
|
.setsockopt = udp_setsockopt,
|
|
|
|
.getsockopt = udp_getsockopt,
|
|
|
|
.sendmsg = udp_sendmsg,
|
|
|
|
.recvmsg = udp_recvmsg,
|
|
|
|
.sendpage = udp_sendpage,
|
|
|
|
.release_cb = ip4_datagram_release_cb,
|
|
|
|
.hash = udp_lib_hash,
|
|
|
|
.unhash = udp_lib_unhash,
|
|
|
|
.rehash = udp_v4_rehash,
|
|
|
|
.get_port = udp_v4_get_port,
|
|
|
|
.memory_allocated = &udp_memory_allocated,
|
|
|
|
.sysctl_mem = sysctl_udp_mem,
|
|
|
|
.sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min),
|
|
|
|
.sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_udp_rmem_min),
|
|
|
|
.obj_size = sizeof(struct udp_sock),
|
|
|
|
.h.udp_table = &udp_table,
|
2008-03-07 07:22:02 +07:00
|
|
|
#ifdef CONFIG_COMPAT
|
2018-03-14 11:57:16 +07:00
|
|
|
.compat_setsockopt = compat_udp_setsockopt,
|
|
|
|
.compat_getsockopt = compat_udp_getsockopt,
|
2008-03-07 07:22:02 +07:00
|
|
|
#endif
|
2018-03-14 11:57:16 +07:00
|
|
|
.diag_destroy = udp_abort,
|
2008-03-07 07:22:02 +07:00
|
|
|
};
|
2009-07-17 07:26:32 +07:00
|
|
|
EXPORT_SYMBOL(udp_prot);
|
2005-04-17 05:20:36 +07:00
|
|
|
|
|
|
|
/* ------------------------------------------------------------------------ */
|
|
|
|
#ifdef CONFIG_PROC_FS
|
|
|
|
|
2008-10-29 15:41:45 +07:00
|
|
|
static struct sock *udp_get_first(struct seq_file *seq, int start)
|
2005-04-17 05:20:36 +07:00
|
|
|
{
|
|
|
|
struct sock *sk;
|
2018-04-11 02:31:50 +07:00
|
|
|
struct udp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
|
2005-04-17 05:20:36 +07:00
|
|
|
struct udp_iter_state *state = seq->private;
|
2008-03-29 08:23:33 +07:00
|
|
|
struct net *net = seq_file_net(seq);
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2018-04-11 02:31:50 +07:00
|
|
|
for (state->bucket = start; state->bucket <= afinfo->udp_table->mask;
|
2009-10-07 07:37:59 +07:00
|
|
|
++state->bucket) {
|
2018-04-11 02:31:50 +07:00
|
|
|
struct udp_hslot *hslot = &afinfo->udp_table->hash[state->bucket];
|
2009-10-07 07:37:59 +07:00
|
|
|
|
2016-04-01 22:52:13 +07:00
|
|
|
if (hlist_empty(&hslot->head))
|
2009-10-07 07:37:59 +07:00
|
|
|
continue;
|
|
|
|
|
2008-10-29 15:41:45 +07:00
|
|
|
spin_lock_bh(&hslot->lock);
|
2016-04-01 22:52:13 +07:00
|
|
|
sk_for_each(sk, &hslot->head) {
|
2008-03-26 01:57:35 +07:00
|
|
|
if (!net_eq(sock_net(sk), net))
|
2008-03-21 18:11:58 +07:00
|
|
|
continue;
|
2018-04-11 02:31:50 +07:00
|
|
|
if (sk->sk_family == afinfo->family)
|
2005-04-17 05:20:36 +07:00
|
|
|
goto found;
|
|
|
|
}
|
2008-10-29 15:41:45 +07:00
|
|
|
spin_unlock_bh(&hslot->lock);
|
2005-04-17 05:20:36 +07:00
|
|
|
}
|
|
|
|
sk = NULL;
|
|
|
|
found:
|
|
|
|
return sk;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct sock *udp_get_next(struct seq_file *seq, struct sock *sk)
|
|
|
|
{
|
2018-04-11 02:31:50 +07:00
|
|
|
struct udp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
|
2005-04-17 05:20:36 +07:00
|
|
|
struct udp_iter_state *state = seq->private;
|
2008-03-29 08:23:33 +07:00
|
|
|
struct net *net = seq_file_net(seq);
|
2005-04-17 05:20:36 +07:00
|
|
|
|
|
|
|
do {
|
2016-04-01 22:52:13 +07:00
|
|
|
sk = sk_next(sk);
|
2018-04-11 02:31:50 +07:00
|
|
|
} while (sk && (!net_eq(sock_net(sk), net) || sk->sk_family != afinfo->family));
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2008-10-29 15:41:45 +07:00
|
|
|
if (!sk) {
|
2018-04-11 02:31:50 +07:00
|
|
|
if (state->bucket <= afinfo->udp_table->mask)
|
|
|
|
spin_unlock_bh(&afinfo->udp_table->hash[state->bucket].lock);
|
2008-10-29 15:41:45 +07:00
|
|
|
return udp_get_first(seq, state->bucket + 1);
|
2005-04-17 05:20:36 +07:00
|
|
|
}
|
|
|
|
return sk;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct sock *udp_get_idx(struct seq_file *seq, loff_t pos)
|
|
|
|
{
|
2008-10-29 15:41:45 +07:00
|
|
|
struct sock *sk = udp_get_first(seq, 0);
|
2005-04-17 05:20:36 +07:00
|
|
|
|
|
|
|
if (sk)
|
2007-03-09 11:41:55 +07:00
|
|
|
while (pos && (sk = udp_get_next(seq, sk)) != NULL)
|
2005-04-17 05:20:36 +07:00
|
|
|
--pos;
|
|
|
|
return pos ? NULL : sk;
|
|
|
|
}
|
|
|
|
|
2018-04-11 02:31:50 +07:00
|
|
|
void *udp_seq_start(struct seq_file *seq, loff_t *pos)
|
2005-04-17 05:20:36 +07:00
|
|
|
{
|
2009-03-24 05:22:33 +07:00
|
|
|
struct udp_iter_state *state = seq->private;
|
2009-10-07 07:37:59 +07:00
|
|
|
state->bucket = MAX_UDP_PORTS;
|
2009-03-24 05:22:33 +07:00
|
|
|
|
2008-04-01 09:38:15 +07:00
|
|
|
return *pos ? udp_get_idx(seq, *pos-1) : SEQ_START_TOKEN;
|
2005-04-17 05:20:36 +07:00
|
|
|
}
|
2018-04-11 02:31:50 +07:00
|
|
|
EXPORT_SYMBOL(udp_seq_start);
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2018-04-11 02:31:50 +07:00
|
|
|
void *udp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
|
2005-04-17 05:20:36 +07:00
|
|
|
{
|
|
|
|
struct sock *sk;
|
|
|
|
|
2008-04-01 09:38:15 +07:00
|
|
|
if (v == SEQ_START_TOKEN)
|
2005-04-17 05:20:36 +07:00
|
|
|
sk = udp_get_idx(seq, 0);
|
|
|
|
else
|
|
|
|
sk = udp_get_next(seq, v);
|
|
|
|
|
|
|
|
++*pos;
|
|
|
|
return sk;
|
|
|
|
}
|
2018-04-11 02:31:50 +07:00
|
|
|
EXPORT_SYMBOL(udp_seq_next);
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2018-04-11 02:31:50 +07:00
|
|
|
void udp_seq_stop(struct seq_file *seq, void *v)
|
2005-04-17 05:20:36 +07:00
|
|
|
{
|
2018-04-11 02:31:50 +07:00
|
|
|
struct udp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
|
2008-10-29 15:41:45 +07:00
|
|
|
struct udp_iter_state *state = seq->private;
|
|
|
|
|
2018-04-11 02:31:50 +07:00
|
|
|
if (state->bucket <= afinfo->udp_table->mask)
|
|
|
|
spin_unlock_bh(&afinfo->udp_table->hash[state->bucket].lock);
|
2005-04-17 05:20:36 +07:00
|
|
|
}
|
2018-04-11 02:31:50 +07:00
|
|
|
EXPORT_SYMBOL(udp_seq_stop);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
|
|
|
/* ------------------------------------------------------------------------ */
|
2008-04-24 15:02:16 +07:00
|
|
|
static void udp4_format_sock(struct sock *sp, struct seq_file *f,
|
2013-11-15 05:31:57 +07:00
|
|
|
int bucket)
|
2008-03-07 07:22:02 +07:00
|
|
|
{
|
|
|
|
struct inet_sock *inet = inet_sk(sp);
|
2009-10-15 13:30:45 +07:00
|
|
|
__be32 dest = inet->inet_daddr;
|
|
|
|
__be32 src = inet->inet_rcv_saddr;
|
|
|
|
__u16 destp = ntohs(inet->inet_dport);
|
|
|
|
__u16 srcp = ntohs(inet->inet_sport);
|
2008-03-07 07:22:02 +07:00
|
|
|
|
2009-10-07 07:37:59 +07:00
|
|
|
seq_printf(f, "%5d: %08X:%04X %08X:%04X"
|
2013-11-15 05:31:57 +07:00
|
|
|
" %02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %d",
|
2008-03-07 07:22:02 +07:00
|
|
|
bucket, src, srcp, dest, destp, sp->sk_state,
|
2009-06-18 09:05:41 +07:00
|
|
|
sk_wmem_alloc_get(sp),
|
2018-06-08 16:35:40 +07:00
|
|
|
udp_rqueue_get(sp),
|
2012-05-24 14:10:10 +07:00
|
|
|
0, 0L, 0,
|
|
|
|
from_kuid_munged(seq_user_ns(f), sock_i_uid(sp)),
|
|
|
|
0, sock_i_ino(sp),
|
2017-06-30 17:08:01 +07:00
|
|
|
refcount_read(&sp->sk_refcnt), sp,
|
2013-11-15 05:31:57 +07:00
|
|
|
atomic_read(&sp->sk_drops));
|
2008-03-07 07:22:02 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
int udp4_seq_show(struct seq_file *seq, void *v)
|
|
|
|
{
|
2013-11-15 05:31:57 +07:00
|
|
|
seq_setwidth(seq, 127);
|
2008-03-07 07:22:02 +07:00
|
|
|
if (v == SEQ_START_TOKEN)
|
2013-11-15 05:31:57 +07:00
|
|
|
seq_puts(seq, " sl local_address rem_address st tx_queue "
|
2008-03-07 07:22:02 +07:00
|
|
|
"rx_queue tr tm->when retrnsmt uid timeout "
|
2008-06-18 11:04:56 +07:00
|
|
|
"inode ref pointer drops");
|
2008-03-07 07:22:02 +07:00
|
|
|
else {
|
|
|
|
struct udp_iter_state *state = seq->private;
|
|
|
|
|
2013-11-15 05:31:57 +07:00
|
|
|
udp4_format_sock(v, seq, state->bucket);
|
2008-03-07 07:22:02 +07:00
|
|
|
}
|
2013-11-15 05:31:57 +07:00
|
|
|
seq_pad(seq, '\n');
|
2008-03-07 07:22:02 +07:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-04-11 00:42:55 +07:00
|
|
|
const struct seq_operations udp_seq_ops = {
|
2018-04-11 02:31:50 +07:00
|
|
|
.start = udp_seq_start,
|
|
|
|
.next = udp_seq_next,
|
|
|
|
.stop = udp_seq_stop,
|
|
|
|
.show = udp4_seq_show,
|
|
|
|
};
|
2018-04-11 00:42:55 +07:00
|
|
|
EXPORT_SYMBOL(udp_seq_ops);
|
2011-10-30 13:46:30 +07:00
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
static struct udp_seq_afinfo udp4_seq_afinfo = {
|
|
|
|
.family = AF_INET,
|
2008-10-29 15:41:45 +07:00
|
|
|
.udp_table = &udp_table,
|
2008-03-07 07:22:02 +07:00
|
|
|
};
|
|
|
|
|
2010-01-17 10:35:32 +07:00
|
|
|
static int __net_init udp4_proc_init_net(struct net *net)
|
2008-03-25 04:53:49 +07:00
|
|
|
{
|
2018-04-11 00:42:55 +07:00
|
|
|
if (!proc_create_net_data("udp", 0444, net->proc_net, &udp_seq_ops,
|
|
|
|
sizeof(struct udp_iter_state), &udp4_seq_afinfo))
|
2018-04-11 02:31:50 +07:00
|
|
|
return -ENOMEM;
|
|
|
|
return 0;
|
2008-03-25 04:53:49 +07:00
|
|
|
}
|
|
|
|
|
2010-01-17 10:35:32 +07:00
|
|
|
static void __net_exit udp4_proc_exit_net(struct net *net)
|
2008-03-25 04:53:49 +07:00
|
|
|
{
|
2018-04-11 02:31:50 +07:00
|
|
|
remove_proc_entry("udp", net->proc_net);
|
2008-03-25 04:53:49 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct pernet_operations udp4_net_ops = {
|
|
|
|
.init = udp4_proc_init_net,
|
|
|
|
.exit = udp4_proc_exit_net,
|
|
|
|
};
|
|
|
|
|
2008-03-07 07:22:02 +07:00
|
|
|
int __init udp4_proc_init(void)
|
|
|
|
{
|
2008-03-25 04:53:49 +07:00
|
|
|
return register_pernet_subsys(&udp4_net_ops);
|
2008-03-07 07:22:02 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
void udp4_proc_exit(void)
|
|
|
|
{
|
2008-03-25 04:53:49 +07:00
|
|
|
unregister_pernet_subsys(&udp4_net_ops);
|
2008-03-07 07:22:02 +07:00
|
|
|
}
|
2005-04-17 05:20:36 +07:00
|
|
|
#endif /* CONFIG_PROC_FS */
|
|
|
|
|
2009-10-07 07:37:59 +07:00
|
|
|
static __initdata unsigned long uhash_entries;
|
|
|
|
static int __init set_uhash_entries(char *str)
|
2008-10-29 15:41:45 +07:00
|
|
|
{
|
2012-05-19 21:13:18 +07:00
|
|
|
ssize_t ret;
|
|
|
|
|
2009-10-07 07:37:59 +07:00
|
|
|
if (!str)
|
|
|
|
return 0;
|
2012-05-19 21:13:18 +07:00
|
|
|
|
|
|
|
ret = kstrtoul(str, 0, &uhash_entries);
|
|
|
|
if (ret)
|
|
|
|
return 0;
|
|
|
|
|
2009-10-07 07:37:59 +07:00
|
|
|
if (uhash_entries && uhash_entries < UDP_HTABLE_SIZE_MIN)
|
|
|
|
uhash_entries = UDP_HTABLE_SIZE_MIN;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
__setup("uhash_entries=", set_uhash_entries);
|
2008-10-29 15:41:45 +07:00
|
|
|
|
2009-10-07 07:37:59 +07:00
|
|
|
void __init udp_table_init(struct udp_table *table, const char *name)
|
|
|
|
{
|
|
|
|
unsigned int i;
|
|
|
|
|
2012-05-23 20:33:35 +07:00
|
|
|
table->hash = alloc_large_system_hash(name,
|
|
|
|
2 * sizeof(struct udp_hslot),
|
|
|
|
uhash_entries,
|
|
|
|
21, /* one slot per 2 MB */
|
|
|
|
0,
|
|
|
|
&table->log,
|
|
|
|
&table->mask,
|
|
|
|
UDP_HTABLE_SIZE_MIN,
|
|
|
|
64 * 1024);
|
|
|
|
|
2009-11-08 17:17:58 +07:00
|
|
|
table->hash2 = table->hash + (table->mask + 1);
|
2009-10-07 07:37:59 +07:00
|
|
|
for (i = 0; i <= table->mask; i++) {
|
2016-04-01 22:52:13 +07:00
|
|
|
INIT_HLIST_HEAD(&table->hash[i].head);
|
2009-11-08 17:17:05 +07:00
|
|
|
table->hash[i].count = 0;
|
2008-10-29 15:41:45 +07:00
|
|
|
spin_lock_init(&table->hash[i].lock);
|
|
|
|
}
|
2009-11-08 17:17:58 +07:00
|
|
|
for (i = 0; i <= table->mask; i++) {
|
2016-04-01 22:52:13 +07:00
|
|
|
INIT_HLIST_HEAD(&table->hash2[i].head);
|
2009-11-08 17:17:58 +07:00
|
|
|
table->hash2[i].count = 0;
|
|
|
|
spin_lock_init(&table->hash2[i].lock);
|
|
|
|
}
|
2008-10-29 15:41:45 +07:00
|
|
|
}
|
|
|
|
|
2015-02-25 00:17:31 +07:00
|
|
|
u32 udp_flow_hashrnd(void)
|
|
|
|
{
|
|
|
|
static u32 hashrnd __read_mostly;
|
|
|
|
|
|
|
|
net_get_random_once(&hashrnd, sizeof(hashrnd));
|
|
|
|
|
|
|
|
return hashrnd;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(udp_flow_hashrnd);
|
|
|
|
|
2018-03-14 11:57:16 +07:00
|
|
|
static void __udp_sysctl_init(struct net *net)
|
|
|
|
{
|
|
|
|
net->ipv4.sysctl_udp_rmem_min = SK_MEM_QUANTUM;
|
|
|
|
net->ipv4.sysctl_udp_wmem_min = SK_MEM_QUANTUM;
|
|
|
|
|
|
|
|
#ifdef CONFIG_NET_L3_MASTER_DEV
|
|
|
|
net->ipv4.sysctl_udp_l3mdev_accept = 0;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __net_init udp_sysctl_init(struct net *net)
|
|
|
|
{
|
|
|
|
__udp_sysctl_init(net);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct pernet_operations __net_initdata udp_sysctl_ops = {
|
2018-03-23 01:34:46 +07:00
|
|
|
.init = udp_sysctl_init,
|
2018-03-14 11:57:16 +07:00
|
|
|
};
|
|
|
|
|
2007-12-31 15:29:24 +07:00
|
|
|
void __init udp_init(void)
|
|
|
|
{
|
2011-07-07 14:27:05 +07:00
|
|
|
unsigned long limit;
|
2016-12-09 02:41:54 +07:00
|
|
|
unsigned int i;
|
2007-12-31 15:29:24 +07:00
|
|
|
|
2009-10-07 07:37:59 +07:00
|
|
|
udp_table_init(&udp_table, "UDP");
|
2011-07-07 14:27:05 +07:00
|
|
|
limit = nr_free_buffer_pages() / 8;
|
2007-12-31 15:29:24 +07:00
|
|
|
limit = max(limit, 128UL);
|
|
|
|
sysctl_udp_mem[0] = limit / 4 * 3;
|
|
|
|
sysctl_udp_mem[1] = limit;
|
|
|
|
sysctl_udp_mem[2] = sysctl_udp_mem[0] * 2;
|
|
|
|
|
2018-03-14 11:57:16 +07:00
|
|
|
__udp_sysctl_init(&init_net);
|
2016-12-09 02:41:54 +07:00
|
|
|
|
|
|
|
/* 16 spinlocks per cpu */
|
|
|
|
udp_busylocks_log = ilog2(nr_cpu_ids) + 4;
|
|
|
|
udp_busylocks = kmalloc(sizeof(spinlock_t) << udp_busylocks_log,
|
|
|
|
GFP_KERNEL);
|
|
|
|
if (!udp_busylocks)
|
|
|
|
panic("UDP: failed to alloc udp_busylocks\n");
|
|
|
|
for (i = 0; i < (1U << udp_busylocks_log); i++)
|
|
|
|
spin_lock_init(udp_busylocks + i);
|
2018-03-14 11:57:16 +07:00
|
|
|
|
|
|
|
if (register_pernet_subsys(&udp_sysctl_ops))
|
|
|
|
panic("UDP: failed to init sysctl parameters.\n");
|
2007-12-31 15:29:24 +07:00
|
|
|
}
|