linux_dsm_epyc7002/net/xfrm/xfrm_device.c

424 lines
9.5 KiB
C
Raw Normal View History

// SPDX-License-Identifier: GPL-2.0-or-later
/*
* xfrm_device.c - IPsec device offloading code.
*
* Copyright (c) 2015 secunet Security Networks AG
*
* Author:
* Steffen Klassert <steffen.klassert@secunet.com>
*/
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/netdevice.h>
#include <linux/skbuff.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <net/dst.h>
#include <net/xfrm.h>
#include <linux/notifier.h>
#ifdef CONFIG_XFRM_OFFLOAD
static void __xfrm_transport_prep(struct xfrm_state *x, struct sk_buff *skb,
unsigned int hsize)
{
struct xfrm_offload *xo = xfrm_offload(skb);
skb_reset_mac_len(skb);
xfrm: do pskb_pull properly in __xfrm_transport_prep For transport mode, when ipv6 nexthdr is set, the packet format might be like: ---------------------------------------------------- | | dest | | | | ESP | ESP | | IP6 hdr| opts.| ESP | TCP | Data | Trailer | ICV | ---------------------------------------------------- and in __xfrm_transport_prep(): pskb_pull(skb, skb->mac_len + sizeof(ip6hdr) + x->props.header_len); it will pull the data pointer to the wrong position, as it missed the nexthdrs/dest opts. This patch is to fix it by using: pskb_pull(skb, skb_transport_offset(skb) + x->props.header_len); as we can be sure transport_header points to ESP header at that moment. It also fixes a panic when packets with ipv6 nexthdr are sent over esp6 transport mode: [ 100.473845] kernel BUG at net/core/skbuff.c:4325! [ 100.478517] RIP: 0010:__skb_to_sgvec+0x252/0x260 [ 100.494355] Call Trace: [ 100.494829] skb_to_sgvec+0x11/0x40 [ 100.495492] esp6_output_tail+0x12e/0x550 [esp6] [ 100.496358] esp6_xmit+0x1d5/0x260 [esp6_offload] [ 100.498029] validate_xmit_xfrm+0x22f/0x2e0 [ 100.499604] __dev_queue_xmit+0x589/0x910 [ 100.502928] ip6_finish_output2+0x2a5/0x5a0 [ 100.503718] ip6_output+0x6c/0x120 [ 100.505198] xfrm_output_resume+0x4bf/0x530 [ 100.508683] xfrm6_output+0x3a/0xc0 [ 100.513446] inet6_csk_xmit+0xa1/0xf0 [ 100.517335] tcp_sendmsg+0x27/0x40 [ 100.517977] sock_sendmsg+0x3e/0x60 [ 100.518648] __sys_sendto+0xee/0x160 Fixes: c35fe4106b92 ("xfrm: Add mode handlers for IPsec on layer 2") Signed-off-by: Xin Long <lucien.xin@gmail.com> Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
2020-04-10 16:06:31 +07:00
if (xo->flags & XFRM_GSO_SEGMENT)
skb->transport_header -= x->props.header_len;
xfrm: do pskb_pull properly in __xfrm_transport_prep For transport mode, when ipv6 nexthdr is set, the packet format might be like: ---------------------------------------------------- | | dest | | | | ESP | ESP | | IP6 hdr| opts.| ESP | TCP | Data | Trailer | ICV | ---------------------------------------------------- and in __xfrm_transport_prep(): pskb_pull(skb, skb->mac_len + sizeof(ip6hdr) + x->props.header_len); it will pull the data pointer to the wrong position, as it missed the nexthdrs/dest opts. This patch is to fix it by using: pskb_pull(skb, skb_transport_offset(skb) + x->props.header_len); as we can be sure transport_header points to ESP header at that moment. It also fixes a panic when packets with ipv6 nexthdr are sent over esp6 transport mode: [ 100.473845] kernel BUG at net/core/skbuff.c:4325! [ 100.478517] RIP: 0010:__skb_to_sgvec+0x252/0x260 [ 100.494355] Call Trace: [ 100.494829] skb_to_sgvec+0x11/0x40 [ 100.495492] esp6_output_tail+0x12e/0x550 [esp6] [ 100.496358] esp6_xmit+0x1d5/0x260 [esp6_offload] [ 100.498029] validate_xmit_xfrm+0x22f/0x2e0 [ 100.499604] __dev_queue_xmit+0x589/0x910 [ 100.502928] ip6_finish_output2+0x2a5/0x5a0 [ 100.503718] ip6_output+0x6c/0x120 [ 100.505198] xfrm_output_resume+0x4bf/0x530 [ 100.508683] xfrm6_output+0x3a/0xc0 [ 100.513446] inet6_csk_xmit+0xa1/0xf0 [ 100.517335] tcp_sendmsg+0x27/0x40 [ 100.517977] sock_sendmsg+0x3e/0x60 [ 100.518648] __sys_sendto+0xee/0x160 Fixes: c35fe4106b92 ("xfrm: Add mode handlers for IPsec on layer 2") Signed-off-by: Xin Long <lucien.xin@gmail.com> Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
2020-04-10 16:06:31 +07:00
pskb_pull(skb, skb_transport_offset(skb) + x->props.header_len);
}
static void __xfrm_mode_tunnel_prep(struct xfrm_state *x, struct sk_buff *skb,
unsigned int hsize)
{
struct xfrm_offload *xo = xfrm_offload(skb);
if (xo->flags & XFRM_GSO_SEGMENT)
skb->transport_header = skb->network_header + hsize;
skb_reset_mac_len(skb);
pskb_pull(skb, skb->mac_len + x->props.header_len);
}
static void __xfrm_mode_beet_prep(struct xfrm_state *x, struct sk_buff *skb,
unsigned int hsize)
{
struct xfrm_offload *xo = xfrm_offload(skb);
int phlen = 0;
if (xo->flags & XFRM_GSO_SEGMENT)
skb->transport_header = skb->network_header + hsize;
skb_reset_mac_len(skb);
if (x->sel.family != AF_INET6) {
phlen = IPV4_BEET_PHMAXLEN;
if (x->outer_mode.family == AF_INET6)
phlen += sizeof(struct ipv6hdr) - sizeof(struct iphdr);
}
pskb_pull(skb, skb->mac_len + hsize + (x->props.header_len - phlen));
}
/* Adjust pointers into the packet when IPsec is done at layer2 */
static void xfrm_outer_mode_prep(struct xfrm_state *x, struct sk_buff *skb)
{
switch (x->outer_mode.encap) {
case XFRM_MODE_TUNNEL:
if (x->outer_mode.family == AF_INET)
return __xfrm_mode_tunnel_prep(x, skb,
sizeof(struct iphdr));
if (x->outer_mode.family == AF_INET6)
return __xfrm_mode_tunnel_prep(x, skb,
sizeof(struct ipv6hdr));
break;
case XFRM_MODE_TRANSPORT:
if (x->outer_mode.family == AF_INET)
return __xfrm_transport_prep(x, skb,
sizeof(struct iphdr));
if (x->outer_mode.family == AF_INET6)
return __xfrm_transport_prep(x, skb,
sizeof(struct ipv6hdr));
break;
case XFRM_MODE_BEET:
if (x->outer_mode.family == AF_INET)
return __xfrm_mode_beet_prep(x, skb,
sizeof(struct iphdr));
if (x->outer_mode.family == AF_INET6)
return __xfrm_mode_beet_prep(x, skb,
sizeof(struct ipv6hdr));
break;
case XFRM_MODE_ROUTEOPTIMIZATION:
case XFRM_MODE_IN_TRIGGER:
break;
}
}
struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again)
{
int err;
unsigned long flags;
struct xfrm_state *x;
struct softnet_data *sd;
esp: remove the skb from the chain when it's enqueued in cryptd_wq Xiumei found a panic in esp offload: BUG: unable to handle kernel NULL pointer dereference at 0000000000000020 RIP: 0010:esp_output_done+0x101/0x160 [esp4] Call Trace: ? esp_output+0x180/0x180 [esp4] cryptd_aead_crypt+0x4c/0x90 cryptd_queue_worker+0x6e/0xa0 process_one_work+0x1a7/0x3b0 worker_thread+0x30/0x390 ? create_worker+0x1a0/0x1a0 kthread+0x112/0x130 ? kthread_flush_work_fn+0x10/0x10 ret_from_fork+0x35/0x40 It was caused by that skb secpath is used in esp_output_done() after it's been released elsewhere. The tx path for esp offload is: __dev_queue_xmit()-> validate_xmit_skb_list()-> validate_xmit_xfrm()-> esp_xmit()-> esp_output_tail()-> aead_request_set_callback(esp_output_done) <--[1] crypto_aead_encrypt() <--[2] In [1], .callback is set, and in [2] it will trigger the worker schedule, later on a kernel thread will call .callback(esp_output_done), as the call trace shows. But in validate_xmit_xfrm(): skb_list_walk_safe(skb, skb2, nskb) { ... err = x->type_offload->xmit(x, skb2, esp_features); [esp_xmit] ... } When the err is -EINPROGRESS, which means this skb2 will be enqueued and later gets encrypted and sent out by .callback later in a kernel thread, skb2 should be removed fromt skb chain. Otherwise, it will get processed again outside validate_xmit_xfrm(), which could release skb secpath, and cause the panic above. This patch is to remove the skb from the chain when it's enqueued in cryptd_wq. While at it, remove the unnecessary 'if (!skb)' check. Fixes: 3dca3f38cfb8 ("xfrm: Separate ESP handling from segmentation for GRO packets.") Reported-by: Xiumei Mu <xmu@redhat.com> Signed-off-by: Xin Long <lucien.xin@gmail.com> Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
2020-03-04 15:51:42 +07:00
struct sk_buff *skb2, *nskb, *pskb = NULL;
netdev_features_t esp_features = features;
struct xfrm_offload *xo = xfrm_offload(skb);
struct sec_path *sp;
if (!xo)
return skb;
if (!(features & NETIF_F_HW_ESP))
esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK);
sp = skb_sec_path(skb);
x = sp->xvec[sp->len - 1];
if (xo->flags & XFRM_GRO || x->xso.flags & XFRM_OFFLOAD_INBOUND)
return skb;
local_irq_save(flags);
sd = this_cpu_ptr(&softnet_data);
err = !skb_queue_empty(&sd->xfrm_backlog);
local_irq_restore(flags);
if (err) {
*again = true;
return skb;
}
if (skb_is_gso(skb)) {
struct net_device *dev = skb->dev;
if (unlikely(x->xso.dev != dev)) {
struct sk_buff *segs;
/* Packet got rerouted, fixup features and segment it. */
esp_features = esp_features & ~(NETIF_F_HW_ESP
| NETIF_F_GSO_ESP);
segs = skb_gso_segment(skb, esp_features);
if (IS_ERR(segs)) {
kfree_skb(skb);
atomic_long_inc(&dev->tx_dropped);
return NULL;
} else {
consume_skb(skb);
skb = segs;
}
}
}
if (!skb->next) {
esp_features |= skb->dev->gso_partial_features;
xfrm_outer_mode_prep(x, skb);
xo->flags |= XFRM_DEV_RESUME;
err = x->type_offload->xmit(x, skb, esp_features);
if (err) {
if (err == -EINPROGRESS)
return NULL;
XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
kfree_skb(skb);
return NULL;
}
skb_push(skb, skb->data - skb_mac_header(skb));
return skb;
}
skb_list_walk_safe(skb, skb2, nskb) {
esp_features |= skb->dev->gso_partial_features;
skb_mark_not_on_list(skb2);
xo = xfrm_offload(skb2);
xo->flags |= XFRM_DEV_RESUME;
xfrm_outer_mode_prep(x, skb2);
err = x->type_offload->xmit(x, skb2, esp_features);
if (!err) {
skb2->next = nskb;
} else if (err != -EINPROGRESS) {
XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
skb2->next = nskb;
kfree_skb_list(skb2);
return NULL;
} else {
if (skb == skb2)
skb = nskb;
esp: remove the skb from the chain when it's enqueued in cryptd_wq Xiumei found a panic in esp offload: BUG: unable to handle kernel NULL pointer dereference at 0000000000000020 RIP: 0010:esp_output_done+0x101/0x160 [esp4] Call Trace: ? esp_output+0x180/0x180 [esp4] cryptd_aead_crypt+0x4c/0x90 cryptd_queue_worker+0x6e/0xa0 process_one_work+0x1a7/0x3b0 worker_thread+0x30/0x390 ? create_worker+0x1a0/0x1a0 kthread+0x112/0x130 ? kthread_flush_work_fn+0x10/0x10 ret_from_fork+0x35/0x40 It was caused by that skb secpath is used in esp_output_done() after it's been released elsewhere. The tx path for esp offload is: __dev_queue_xmit()-> validate_xmit_skb_list()-> validate_xmit_xfrm()-> esp_xmit()-> esp_output_tail()-> aead_request_set_callback(esp_output_done) <--[1] crypto_aead_encrypt() <--[2] In [1], .callback is set, and in [2] it will trigger the worker schedule, later on a kernel thread will call .callback(esp_output_done), as the call trace shows. But in validate_xmit_xfrm(): skb_list_walk_safe(skb, skb2, nskb) { ... err = x->type_offload->xmit(x, skb2, esp_features); [esp_xmit] ... } When the err is -EINPROGRESS, which means this skb2 will be enqueued and later gets encrypted and sent out by .callback later in a kernel thread, skb2 should be removed fromt skb chain. Otherwise, it will get processed again outside validate_xmit_xfrm(), which could release skb secpath, and cause the panic above. This patch is to remove the skb from the chain when it's enqueued in cryptd_wq. While at it, remove the unnecessary 'if (!skb)' check. Fixes: 3dca3f38cfb8 ("xfrm: Separate ESP handling from segmentation for GRO packets.") Reported-by: Xiumei Mu <xmu@redhat.com> Signed-off-by: Xin Long <lucien.xin@gmail.com> Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
2020-03-04 15:51:42 +07:00
else
pskb->next = nskb;
continue;
}
skb_push(skb2, skb2->data - skb_mac_header(skb2));
esp: remove the skb from the chain when it's enqueued in cryptd_wq Xiumei found a panic in esp offload: BUG: unable to handle kernel NULL pointer dereference at 0000000000000020 RIP: 0010:esp_output_done+0x101/0x160 [esp4] Call Trace: ? esp_output+0x180/0x180 [esp4] cryptd_aead_crypt+0x4c/0x90 cryptd_queue_worker+0x6e/0xa0 process_one_work+0x1a7/0x3b0 worker_thread+0x30/0x390 ? create_worker+0x1a0/0x1a0 kthread+0x112/0x130 ? kthread_flush_work_fn+0x10/0x10 ret_from_fork+0x35/0x40 It was caused by that skb secpath is used in esp_output_done() after it's been released elsewhere. The tx path for esp offload is: __dev_queue_xmit()-> validate_xmit_skb_list()-> validate_xmit_xfrm()-> esp_xmit()-> esp_output_tail()-> aead_request_set_callback(esp_output_done) <--[1] crypto_aead_encrypt() <--[2] In [1], .callback is set, and in [2] it will trigger the worker schedule, later on a kernel thread will call .callback(esp_output_done), as the call trace shows. But in validate_xmit_xfrm(): skb_list_walk_safe(skb, skb2, nskb) { ... err = x->type_offload->xmit(x, skb2, esp_features); [esp_xmit] ... } When the err is -EINPROGRESS, which means this skb2 will be enqueued and later gets encrypted and sent out by .callback later in a kernel thread, skb2 should be removed fromt skb chain. Otherwise, it will get processed again outside validate_xmit_xfrm(), which could release skb secpath, and cause the panic above. This patch is to remove the skb from the chain when it's enqueued in cryptd_wq. While at it, remove the unnecessary 'if (!skb)' check. Fixes: 3dca3f38cfb8 ("xfrm: Separate ESP handling from segmentation for GRO packets.") Reported-by: Xiumei Mu <xmu@redhat.com> Signed-off-by: Xin Long <lucien.xin@gmail.com> Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
2020-03-04 15:51:42 +07:00
pskb = skb2;
}
return skb;
}
EXPORT_SYMBOL_GPL(validate_xmit_xfrm);
int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
struct xfrm_user_offload *xuo)
{
int err;
struct dst_entry *dst;
struct net_device *dev;
struct xfrm_state_offload *xso = &x->xso;
xfrm_address_t *saddr;
xfrm_address_t *daddr;
if (!x->type_offload)
return -EINVAL;
/* We don't yet support UDP encapsulation and TFC padding. */
if (x->encap || x->tfcpad)
return -EINVAL;
dev = dev_get_by_index(net, xuo->ifindex);
if (!dev) {
if (!(xuo->flags & XFRM_OFFLOAD_INBOUND)) {
saddr = &x->props.saddr;
daddr = &x->id.daddr;
} else {
saddr = &x->id.daddr;
daddr = &x->props.saddr;
}
net: xfrm: support setting an output mark. On systems that use mark-based routing it may be necessary for routing lookups to use marks in order for packets to be routed correctly. An example of such a system is Android, which uses socket marks to route packets via different networks. Currently, routing lookups in tunnel mode always use a mark of zero, making routing incorrect on such systems. This patch adds a new output_mark element to the xfrm state and a corresponding XFRMA_OUTPUT_MARK netlink attribute. The output mark differs from the existing xfrm mark in two ways: 1. The xfrm mark is used to match xfrm policies and states, while the xfrm output mark is used to set the mark (and influence the routing) of the packets emitted by those states. 2. The existing mark is constrained to be a subset of the bits of the originating socket or transformed packet, but the output mark is arbitrary and depends only on the state. The use of a separate mark provides additional flexibility. For example: - A packet subject to two transforms (e.g., transport mode inside tunnel mode) can have two different output marks applied to it, one for the transport mode SA and one for the tunnel mode SA. - On a system where socket marks determine routing, the packets emitted by an IPsec tunnel can be routed based on a mark that is determined by the tunnel, not by the marks of the unencrypted packets. - Support for setting the output marks can be introduced without breaking any existing setups that employ both mark-based routing and xfrm tunnel mode. Simply changing the code to use the xfrm mark for routing output packets could xfrm mark could change behaviour in a way that breaks these setups. If the output mark is unspecified or set to zero, the mark is not set or changed. Tested: make allyesconfig; make -j64 Tested: https://android-review.googlesource.com/452776 Signed-off-by: Lorenzo Colitti <lorenzo@google.com> Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
2017-08-11 00:11:33 +07:00
dst = __xfrm_dst_lookup(net, 0, 0, saddr, daddr,
x->props.family,
xfrm_smark_get(0, x));
if (IS_ERR(dst))
return 0;
dev = dst->dev;
dev_hold(dev);
dst_release(dst);
}
if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_state_add) {
xso->dev = NULL;
dev_put(dev);
return 0;
}
if (x->props.flags & XFRM_STATE_ESN &&
!dev->xfrmdev_ops->xdo_dev_state_advance_esn) {
xso->dev = NULL;
dev_put(dev);
return -EINVAL;
}
xso->dev = dev;
xso->num_exthdrs = 1;
xso->flags = xuo->flags;
err = dev->xfrmdev_ops->xdo_dev_state_add(x);
if (err) {
xso->num_exthdrs = 0;
xso->flags = 0;
xso->dev = NULL;
dev_put(dev);
if (err != -EOPNOTSUPP)
return err;
}
return 0;
}
EXPORT_SYMBOL_GPL(xfrm_dev_state_add);
bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
{
int mtu;
struct dst_entry *dst = skb_dst(skb);
struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
struct net_device *dev = x->xso.dev;
if (!x->type_offload || x->encap)
return false;
if ((!dev || (dev == xfrm_dst_path(dst)->dev)) &&
(!xdst->child->xfrm)) {
mtu = xfrm_state_mtu(x, xdst->child_mtu_cached);
if (skb->len <= mtu)
goto ok;
if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
goto ok;
}
return false;
ok:
if (dev && dev->xfrmdev_ops && dev->xfrmdev_ops->xdo_dev_offload_ok)
return x->xso.dev->xfrmdev_ops->xdo_dev_offload_ok(skb, x);
return true;
}
EXPORT_SYMBOL_GPL(xfrm_dev_offload_ok);
void xfrm_dev_resume(struct sk_buff *skb)
{
struct net_device *dev = skb->dev;
int ret = NETDEV_TX_BUSY;
struct netdev_queue *txq;
struct softnet_data *sd;
unsigned long flags;
rcu_read_lock();
txq = netdev_core_pick_tx(dev, skb, NULL);
HARD_TX_LOCK(dev, txq, smp_processor_id());
if (!netif_xmit_frozen_or_stopped(txq))
skb = dev_hard_start_xmit(skb, dev, txq, &ret);
HARD_TX_UNLOCK(dev, txq);
if (!dev_xmit_complete(ret)) {
local_irq_save(flags);
sd = this_cpu_ptr(&softnet_data);
skb_queue_tail(&sd->xfrm_backlog, skb);
raise_softirq_irqoff(NET_TX_SOFTIRQ);
local_irq_restore(flags);
}
rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(xfrm_dev_resume);
void xfrm_dev_backlog(struct softnet_data *sd)
{
struct sk_buff_head *xfrm_backlog = &sd->xfrm_backlog;
struct sk_buff_head list;
struct sk_buff *skb;
if (skb_queue_empty(xfrm_backlog))
return;
__skb_queue_head_init(&list);
spin_lock(&xfrm_backlog->lock);
skb_queue_splice_init(xfrm_backlog, &list);
spin_unlock(&xfrm_backlog->lock);
while (!skb_queue_empty(&list)) {
skb = __skb_dequeue(&list);
xfrm_dev_resume(skb);
}
}
#endif
static int xfrm_api_check(struct net_device *dev)
{
#ifdef CONFIG_XFRM_OFFLOAD
if ((dev->features & NETIF_F_HW_ESP_TX_CSUM) &&
!(dev->features & NETIF_F_HW_ESP))
return NOTIFY_BAD;
if ((dev->features & NETIF_F_HW_ESP) &&
(!(dev->xfrmdev_ops &&
dev->xfrmdev_ops->xdo_dev_state_add &&
dev->xfrmdev_ops->xdo_dev_state_delete)))
return NOTIFY_BAD;
#else
if (dev->features & (NETIF_F_HW_ESP | NETIF_F_HW_ESP_TX_CSUM))
return NOTIFY_BAD;
#endif
return NOTIFY_DONE;
}
static int xfrm_dev_register(struct net_device *dev)
{
return xfrm_api_check(dev);
}
static int xfrm_dev_feat_change(struct net_device *dev)
{
return xfrm_api_check(dev);
}
static int xfrm_dev_down(struct net_device *dev)
{
if (dev->features & NETIF_F_HW_ESP)
xfrm_dev_state_flush(dev_net(dev), dev, true);
return NOTIFY_DONE;
}
static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr)
{
struct net_device *dev = netdev_notifier_info_to_dev(ptr);
switch (event) {
case NETDEV_REGISTER:
return xfrm_dev_register(dev);
case NETDEV_FEAT_CHANGE:
return xfrm_dev_feat_change(dev);
case NETDEV_DOWN:
case NETDEV_UNREGISTER:
return xfrm_dev_down(dev);
}
return NOTIFY_DONE;
}
static struct notifier_block xfrm_dev_notifier = {
.notifier_call = xfrm_dev_event,
};
void __init xfrm_dev_init(void)
{
register_netdevice_notifier(&xfrm_dev_notifier);
}