2018-05-18 19:00:21 +07:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0 */
|
|
|
|
/* XDP user-space ring structure
|
2018-05-02 18:01:24 +07:00
|
|
|
* Copyright(c) 2018 Intel Corporation.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef _LINUX_XSK_QUEUE_H
|
|
|
|
#define _LINUX_XSK_QUEUE_H
|
|
|
|
|
|
|
|
#include <linux/types.h>
|
|
|
|
#include <linux/if_xdp.h>
|
2018-06-04 19:05:51 +07:00
|
|
|
#include <net/xdp_sock.h>
|
2018-05-02 18:01:24 +07:00
|
|
|
|
xsk: remove explicit ring structure from uapi
In this commit we remove the explicit ring structure from the the
uapi. It is tricky for an uapi to depend on a certain L1 cache line
size, since it can differ for variants of the same architecture. Now,
we let the user application determine the offsets of the producer,
consumer and descriptors by asking the socket via getsockopt.
A typical flow would be (Rx ring):
struct xdp_mmap_offsets off;
struct xdp_desc *ring;
u32 *prod, *cons;
void *map;
...
getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
map = mmap(NULL, off.rx.desc +
NUM_DESCS * sizeof(struct xdp_desc),
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, sfd,
XDP_PGOFF_RX_RING);
prod = map + off.rx.producer;
cons = map + off.rx.consumer;
ring = map + off.rx.desc;
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-05-22 14:34:59 +07:00
|
|
|
struct xdp_ring {
|
|
|
|
u32 producer ____cacheline_aligned_in_smp;
|
|
|
|
u32 consumer ____cacheline_aligned_in_smp;
|
xsk: add support for need_wakeup flag in AF_XDP rings
This commit adds support for a new flag called need_wakeup in the
AF_XDP Tx and fill rings. When this flag is set, it means that the
application has to explicitly wake up the kernel Rx (for the bit in
the fill ring) or kernel Tx (for bit in the Tx ring) processing by
issuing a syscall. Poll() can wake up both depending on the flags
submitted and sendto() will wake up tx processing only.
The main reason for introducing this new flag is to be able to
efficiently support the case when application and driver is executing
on the same core. Previously, the driver was just busy-spinning on the
fill ring if it ran out of buffers in the HW and there were none on
the fill ring. This approach works when the application is running on
another core as it can replenish the fill ring while the driver is
busy-spinning. Though, this is a lousy approach if both of them are
running on the same core as the probability of the fill ring getting
more entries when the driver is busy-spinning is zero. With this new
feature the driver now sets the need_wakeup flag and returns to the
application. The application can then replenish the fill queue and
then explicitly wake up the Rx processing in the kernel using the
syscall poll(). For Tx, the flag is only set to one if the driver has
no outstanding Tx completion interrupts. If it has some, the flag is
zero as it will be woken up by a completion interrupt anyway.
As a nice side effect, this new flag also improves the performance of
the case where application and driver are running on two different
cores as it reduces the number of syscalls to the kernel. The kernel
tells user space if it needs to be woken up by a syscall, and this
eliminates many of the syscalls.
This flag needs some simple driver support. If the driver does not
support this, the Rx flag is always zero and the Tx flag is always
one. This makes any application relying on this feature default to the
old behaviour of not requiring any syscalls in the Rx path and always
having to call sendto() in the Tx path.
For backwards compatibility reasons, this feature has to be explicitly
turned on using a new bind flag (XDP_USE_NEED_WAKEUP). I recommend
that you always turn it on as it so far always have had a positive
performance impact.
The name and inspiration of the flag has been taken from io_uring by
Jens Axboe. Details about this feature in io_uring can be found in
http://kernel.dk/io_uring.pdf, section 8.3.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-14 14:27:17 +07:00
|
|
|
u32 flags;
|
xsk: remove explicit ring structure from uapi
In this commit we remove the explicit ring structure from the the
uapi. It is tricky for an uapi to depend on a certain L1 cache line
size, since it can differ for variants of the same architecture. Now,
we let the user application determine the offsets of the producer,
consumer and descriptors by asking the socket via getsockopt.
A typical flow would be (Rx ring):
struct xdp_mmap_offsets off;
struct xdp_desc *ring;
u32 *prod, *cons;
void *map;
...
getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
map = mmap(NULL, off.rx.desc +
NUM_DESCS * sizeof(struct xdp_desc),
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, sfd,
XDP_PGOFF_RX_RING);
prod = map + off.rx.producer;
cons = map + off.rx.consumer;
ring = map + off.rx.desc;
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-05-22 14:34:59 +07:00
|
|
|
};
|
|
|
|
|
|
|
|
/* Used for the RX and TX queues for packets */
|
|
|
|
struct xdp_rxtx_ring {
|
|
|
|
struct xdp_ring ptrs;
|
|
|
|
struct xdp_desc desc[0] ____cacheline_aligned_in_smp;
|
|
|
|
};
|
|
|
|
|
|
|
|
/* Used for the fill and completion queues for buffers */
|
|
|
|
struct xdp_umem_ring {
|
|
|
|
struct xdp_ring ptrs;
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 18:57:13 +07:00
|
|
|
u64 desc[0] ____cacheline_aligned_in_smp;
|
xsk: remove explicit ring structure from uapi
In this commit we remove the explicit ring structure from the the
uapi. It is tricky for an uapi to depend on a certain L1 cache line
size, since it can differ for variants of the same architecture. Now,
we let the user application determine the offsets of the producer,
consumer and descriptors by asking the socket via getsockopt.
A typical flow would be (Rx ring):
struct xdp_mmap_offsets off;
struct xdp_desc *ring;
u32 *prod, *cons;
void *map;
...
getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
map = mmap(NULL, off.rx.desc +
NUM_DESCS * sizeof(struct xdp_desc),
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, sfd,
XDP_PGOFF_RX_RING);
prod = map + off.rx.producer;
cons = map + off.rx.consumer;
ring = map + off.rx.desc;
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-05-22 14:34:59 +07:00
|
|
|
};
|
|
|
|
|
2018-05-02 18:01:24 +07:00
|
|
|
struct xsk_queue {
|
2018-08-31 18:40:02 +07:00
|
|
|
u64 chunk_mask;
|
|
|
|
u64 size;
|
2018-05-02 18:01:24 +07:00
|
|
|
u32 ring_mask;
|
|
|
|
u32 nentries;
|
2019-12-19 19:39:22 +07:00
|
|
|
u32 cached_prod;
|
2019-12-19 19:39:26 +07:00
|
|
|
u32 cached_cons;
|
2018-05-02 18:01:24 +07:00
|
|
|
struct xdp_ring *ring;
|
|
|
|
u64 invalid_descs;
|
|
|
|
};
|
|
|
|
|
2019-04-16 19:58:08 +07:00
|
|
|
/* The structure of the shared state of the rings are the same as the
|
|
|
|
* ring buffer in kernel/events/ring_buffer.c. For the Rx and completion
|
|
|
|
* ring, the kernel is the producer and user space is the consumer. For
|
|
|
|
* the Tx and fill rings, the kernel is the consumer and user space is
|
|
|
|
* the producer.
|
|
|
|
*
|
|
|
|
* producer consumer
|
|
|
|
*
|
|
|
|
* if (LOAD ->consumer) { LOAD ->producer
|
|
|
|
* (A) smp_rmb() (C)
|
|
|
|
* STORE $data LOAD $data
|
|
|
|
* smp_wmb() (B) smp_mb() (D)
|
|
|
|
* STORE ->producer STORE ->consumer
|
|
|
|
* }
|
|
|
|
*
|
|
|
|
* (A) pairs with (D), and (B) pairs with (C).
|
|
|
|
*
|
|
|
|
* Starting with (B), it protects the data from being written after
|
|
|
|
* the producer pointer. If this barrier was missing, the consumer
|
|
|
|
* could observe the producer pointer being set and thus load the data
|
|
|
|
* before the producer has written the new data. The consumer would in
|
|
|
|
* this case load the old data.
|
|
|
|
*
|
|
|
|
* (C) protects the consumer from speculatively loading the data before
|
|
|
|
* the producer pointer actually has been read. If we do not have this
|
|
|
|
* barrier, some architectures could load old data as speculative loads
|
|
|
|
* are not discarded as the CPU does not know there is a dependency
|
|
|
|
* between ->producer and data.
|
|
|
|
*
|
|
|
|
* (A) is a control dependency that separates the load of ->consumer
|
|
|
|
* from the stores of $data. In case ->consumer indicates there is no
|
|
|
|
* room in the buffer to store $data we do not. So no barrier is needed.
|
|
|
|
*
|
|
|
|
* (D) protects the load of the data to be observed to happen after the
|
|
|
|
* store of the consumer pointer. If we did not have this memory
|
|
|
|
* barrier, the producer could observe the consumer pointer being set
|
|
|
|
* and overwrite the data with a new value before the consumer got the
|
|
|
|
* chance to read the old value. The consumer would thus miss reading
|
|
|
|
* the old entry and very likely read the new entry twice, once right
|
|
|
|
* now and again after circling through the ring.
|
|
|
|
*/
|
|
|
|
|
2019-12-19 19:39:30 +07:00
|
|
|
/* The operations on the rings are the following:
|
|
|
|
*
|
|
|
|
* producer consumer
|
|
|
|
*
|
|
|
|
* RESERVE entries PEEK in the ring for entries
|
|
|
|
* WRITE data into the ring READ data from the ring
|
|
|
|
* SUBMIT entries RELEASE entries
|
|
|
|
*
|
|
|
|
* The producer reserves one or more entries in the ring. It can then
|
|
|
|
* fill in these entries and finally submit them so that they can be
|
|
|
|
* seen and read by the consumer.
|
|
|
|
*
|
|
|
|
* The consumer peeks into the ring to see if the producer has written
|
|
|
|
* any new entries. If so, the producer can then read these entries
|
|
|
|
* and when it is done reading them release them back to the producer
|
|
|
|
* so that the producer can use these slots to fill in new entries.
|
|
|
|
*
|
|
|
|
* The function names below reflect these operations.
|
|
|
|
*/
|
2019-06-26 21:35:24 +07:00
|
|
|
|
2019-12-19 19:39:30 +07:00
|
|
|
/* Functions that read and validate content from consumer rings. */
|
2018-05-02 18:01:27 +07:00
|
|
|
|
2019-12-19 19:39:27 +07:00
|
|
|
static inline bool xskq_cons_crosses_non_contig_pg(struct xdp_umem *umem,
|
|
|
|
u64 addr,
|
|
|
|
u64 length)
|
xsk: add support to allow unaligned chunk placement
Currently, addresses are chunk size aligned. This means, we are very
restricted in terms of where we can place chunk within the umem. For
example, if we have a chunk size of 2k, then our chunks can only be placed
at 0,2k,4k,6k,8k... and so on (ie. every 2k starting from 0).
This patch introduces the ability to use unaligned chunks. With these
changes, we are no longer bound to having to place chunks at a 2k (or
whatever your chunk size is) interval. Since we are no longer dealing with
aligned chunks, they can now cross page boundaries. Checks for page
contiguity have been added in order to keep track of which pages are
followed by a physically contiguous page.
Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-27 09:25:22 +07:00
|
|
|
{
|
|
|
|
bool cross_pg = (addr & (PAGE_SIZE - 1)) + length > PAGE_SIZE;
|
|
|
|
bool next_pg_contig =
|
|
|
|
(unsigned long)umem->pages[(addr >> PAGE_SHIFT)].addr &
|
|
|
|
XSK_NEXT_PG_CONTIG_MASK;
|
|
|
|
|
|
|
|
return cross_pg && !next_pg_contig;
|
|
|
|
}
|
|
|
|
|
2019-12-19 19:39:27 +07:00
|
|
|
static inline bool xskq_cons_is_valid_unaligned(struct xsk_queue *q,
|
|
|
|
u64 addr,
|
xsk: add support to allow unaligned chunk placement
Currently, addresses are chunk size aligned. This means, we are very
restricted in terms of where we can place chunk within the umem. For
example, if we have a chunk size of 2k, then our chunks can only be placed
at 0,2k,4k,6k,8k... and so on (ie. every 2k starting from 0).
This patch introduces the ability to use unaligned chunks. With these
changes, we are no longer bound to having to place chunks at a 2k (or
whatever your chunk size is) interval. Since we are no longer dealing with
aligned chunks, they can now cross page boundaries. Checks for page
contiguity have been added in order to keep track of which pages are
followed by a physically contiguous page.
Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-27 09:25:22 +07:00
|
|
|
u64 length,
|
|
|
|
struct xdp_umem *umem)
|
|
|
|
{
|
|
|
|
u64 base_addr = xsk_umem_extract_addr(addr);
|
|
|
|
|
|
|
|
addr = xsk_umem_add_offset_to_addr(addr);
|
|
|
|
if (base_addr >= q->size || addr >= q->size ||
|
2019-12-19 19:39:27 +07:00
|
|
|
xskq_cons_crosses_non_contig_pg(umem, addr, length)) {
|
xsk: add support to allow unaligned chunk placement
Currently, addresses are chunk size aligned. This means, we are very
restricted in terms of where we can place chunk within the umem. For
example, if we have a chunk size of 2k, then our chunks can only be placed
at 0,2k,4k,6k,8k... and so on (ie. every 2k starting from 0).
This patch introduces the ability to use unaligned chunks. With these
changes, we are no longer bound to having to place chunks at a 2k (or
whatever your chunk size is) interval. Since we are no longer dealing with
aligned chunks, they can now cross page boundaries. Checks for page
contiguity have been added in order to keep track of which pages are
followed by a physically contiguous page.
Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-27 09:25:22 +07:00
|
|
|
q->invalid_descs++;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2019-12-19 19:39:30 +07:00
|
|
|
static inline bool xskq_cons_is_valid_addr(struct xsk_queue *q, u64 addr)
|
|
|
|
{
|
|
|
|
if (addr >= q->size) {
|
|
|
|
q->invalid_descs++;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2019-12-19 19:39:27 +07:00
|
|
|
static inline bool xskq_cons_read_addr(struct xsk_queue *q, u64 *addr,
|
|
|
|
struct xdp_umem *umem)
|
2018-05-02 18:01:27 +07:00
|
|
|
{
|
2019-12-19 19:39:26 +07:00
|
|
|
struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
|
|
|
|
|
|
|
|
while (q->cached_cons != q->cached_prod) {
|
|
|
|
u32 idx = q->cached_cons & q->ring_mask;
|
2018-05-02 18:01:27 +07:00
|
|
|
|
2019-12-19 19:39:29 +07:00
|
|
|
*addr = ring->desc[idx] & q->chunk_mask;
|
xsk: add support to allow unaligned chunk placement
Currently, addresses are chunk size aligned. This means, we are very
restricted in terms of where we can place chunk within the umem. For
example, if we have a chunk size of 2k, then our chunks can only be placed
at 0,2k,4k,6k,8k... and so on (ie. every 2k starting from 0).
This patch introduces the ability to use unaligned chunks. With these
changes, we are no longer bound to having to place chunks at a 2k (or
whatever your chunk size is) interval. Since we are no longer dealing with
aligned chunks, they can now cross page boundaries. Checks for page
contiguity have been added in order to keep track of which pages are
followed by a physically contiguous page.
Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-27 09:25:22 +07:00
|
|
|
|
|
|
|
if (umem->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG) {
|
2019-12-19 19:39:27 +07:00
|
|
|
if (xskq_cons_is_valid_unaligned(q, *addr,
|
xsk: add support to allow unaligned chunk placement
Currently, addresses are chunk size aligned. This means, we are very
restricted in terms of where we can place chunk within the umem. For
example, if we have a chunk size of 2k, then our chunks can only be placed
at 0,2k,4k,6k,8k... and so on (ie. every 2k starting from 0).
This patch introduces the ability to use unaligned chunks. With these
changes, we are no longer bound to having to place chunks at a 2k (or
whatever your chunk size is) interval. Since we are no longer dealing with
aligned chunks, they can now cross page boundaries. Checks for page
contiguity have been added in order to keep track of which pages are
followed by a physically contiguous page.
Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-27 09:25:22 +07:00
|
|
|
umem->chunk_size_nohr,
|
|
|
|
umem))
|
2019-12-19 19:39:27 +07:00
|
|
|
return true;
|
xsk: add support to allow unaligned chunk placement
Currently, addresses are chunk size aligned. This means, we are very
restricted in terms of where we can place chunk within the umem. For
example, if we have a chunk size of 2k, then our chunks can only be placed
at 0,2k,4k,6k,8k... and so on (ie. every 2k starting from 0).
This patch introduces the ability to use unaligned chunks. With these
changes, we are no longer bound to having to place chunks at a 2k (or
whatever your chunk size is) interval. Since we are no longer dealing with
aligned chunks, they can now cross page boundaries. Checks for page
contiguity have been added in order to keep track of which pages are
followed by a physically contiguous page.
Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-27 09:25:22 +07:00
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2019-12-19 19:39:27 +07:00
|
|
|
if (xskq_cons_is_valid_addr(q, *addr))
|
|
|
|
return true;
|
2018-05-02 18:01:27 +07:00
|
|
|
|
xsk: add support to allow unaligned chunk placement
Currently, addresses are chunk size aligned. This means, we are very
restricted in terms of where we can place chunk within the umem. For
example, if we have a chunk size of 2k, then our chunks can only be placed
at 0,2k,4k,6k,8k... and so on (ie. every 2k starting from 0).
This patch introduces the ability to use unaligned chunks. With these
changes, we are no longer bound to having to place chunks at a 2k (or
whatever your chunk size is) interval. Since we are no longer dealing with
aligned chunks, they can now cross page boundaries. Checks for page
contiguity have been added in order to keep track of which pages are
followed by a physically contiguous page.
Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-27 09:25:22 +07:00
|
|
|
out:
|
2019-12-19 19:39:26 +07:00
|
|
|
q->cached_cons++;
|
2018-05-02 18:01:27 +07:00
|
|
|
}
|
|
|
|
|
2019-12-19 19:39:27 +07:00
|
|
|
return false;
|
2018-05-02 18:01:27 +07:00
|
|
|
}
|
|
|
|
|
2019-12-19 19:39:27 +07:00
|
|
|
static inline bool xskq_cons_is_valid_desc(struct xsk_queue *q,
|
|
|
|
struct xdp_desc *d,
|
|
|
|
struct xdp_umem *umem)
|
2018-05-02 18:01:34 +07:00
|
|
|
{
|
xsk: add support to allow unaligned chunk placement
Currently, addresses are chunk size aligned. This means, we are very
restricted in terms of where we can place chunk within the umem. For
example, if we have a chunk size of 2k, then our chunks can only be placed
at 0,2k,4k,6k,8k... and so on (ie. every 2k starting from 0).
This patch introduces the ability to use unaligned chunks. With these
changes, we are no longer bound to having to place chunks at a 2k (or
whatever your chunk size is) interval. Since we are no longer dealing with
aligned chunks, they can now cross page boundaries. Checks for page
contiguity have been added in order to keep track of which pages are
followed by a physically contiguous page.
Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-27 09:25:22 +07:00
|
|
|
if (umem->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG) {
|
2019-12-19 19:39:27 +07:00
|
|
|
if (!xskq_cons_is_valid_unaligned(q, d->addr, d->len, umem))
|
xsk: add support to allow unaligned chunk placement
Currently, addresses are chunk size aligned. This means, we are very
restricted in terms of where we can place chunk within the umem. For
example, if we have a chunk size of 2k, then our chunks can only be placed
at 0,2k,4k,6k,8k... and so on (ie. every 2k starting from 0).
This patch introduces the ability to use unaligned chunks. With these
changes, we are no longer bound to having to place chunks at a 2k (or
whatever your chunk size is) interval. Since we are no longer dealing with
aligned chunks, they can now cross page boundaries. Checks for page
contiguity have been added in order to keep track of which pages are
followed by a physically contiguous page.
Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-27 09:25:22 +07:00
|
|
|
return false;
|
|
|
|
|
|
|
|
if (d->len > umem->chunk_size_nohr || d->options) {
|
|
|
|
q->invalid_descs++;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2019-12-19 19:39:27 +07:00
|
|
|
if (!xskq_cons_is_valid_addr(q, d->addr))
|
2018-05-02 18:01:34 +07:00
|
|
|
return false;
|
|
|
|
|
2019-03-08 14:57:27 +07:00
|
|
|
if (((d->addr + d->len) & q->chunk_mask) != (d->addr & q->chunk_mask) ||
|
|
|
|
d->options) {
|
2018-05-02 18:01:34 +07:00
|
|
|
q->invalid_descs++;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2019-12-19 19:39:27 +07:00
|
|
|
static inline bool xskq_cons_read_desc(struct xsk_queue *q,
|
|
|
|
struct xdp_desc *desc,
|
|
|
|
struct xdp_umem *umem)
|
2018-05-02 18:01:34 +07:00
|
|
|
{
|
2019-12-19 19:39:26 +07:00
|
|
|
while (q->cached_cons != q->cached_prod) {
|
2018-05-02 18:01:34 +07:00
|
|
|
struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring;
|
2019-12-19 19:39:26 +07:00
|
|
|
u32 idx = q->cached_cons & q->ring_mask;
|
2018-05-02 18:01:34 +07:00
|
|
|
|
2019-12-19 19:39:29 +07:00
|
|
|
*desc = ring->desc[idx];
|
2019-12-19 19:39:27 +07:00
|
|
|
if (xskq_cons_is_valid_desc(q, desc, umem))
|
|
|
|
return true;
|
2018-05-02 18:01:34 +07:00
|
|
|
|
2019-12-19 19:39:26 +07:00
|
|
|
q->cached_cons++;
|
2018-05-02 18:01:34 +07:00
|
|
|
}
|
|
|
|
|
2019-12-19 19:39:27 +07:00
|
|
|
return false;
|
2018-05-02 18:01:34 +07:00
|
|
|
}
|
|
|
|
|
2019-12-19 19:39:30 +07:00
|
|
|
/* Functions for consumers */
|
|
|
|
|
|
|
|
static inline void __xskq_cons_release(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
smp_mb(); /* D, matches A */
|
|
|
|
WRITE_ONCE(q->ring->consumer, q->cached_cons);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void __xskq_cons_peek(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
/* Refresh the local pointer */
|
|
|
|
q->cached_prod = READ_ONCE(q->ring->producer);
|
|
|
|
smp_rmb(); /* C, matches B */
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void xskq_cons_get_entries(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
__xskq_cons_release(q);
|
|
|
|
__xskq_cons_peek(q);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool xskq_cons_has_entries(struct xsk_queue *q, u32 cnt)
|
|
|
|
{
|
|
|
|
u32 entries = q->cached_prod - q->cached_cons;
|
|
|
|
|
|
|
|
if (entries >= cnt)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
__xskq_cons_peek(q);
|
|
|
|
entries = q->cached_prod - q->cached_cons;
|
|
|
|
|
|
|
|
return entries >= cnt;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool xskq_cons_peek_addr(struct xsk_queue *q, u64 *addr,
|
|
|
|
struct xdp_umem *umem)
|
|
|
|
{
|
|
|
|
if (q->cached_prod == q->cached_cons)
|
|
|
|
xskq_cons_get_entries(q);
|
|
|
|
return xskq_cons_read_addr(q, addr, umem);
|
|
|
|
}
|
|
|
|
|
2019-12-19 19:39:27 +07:00
|
|
|
static inline bool xskq_cons_peek_desc(struct xsk_queue *q,
|
|
|
|
struct xdp_desc *desc,
|
|
|
|
struct xdp_umem *umem)
|
2018-05-02 18:01:34 +07:00
|
|
|
{
|
2019-12-19 19:39:26 +07:00
|
|
|
if (q->cached_prod == q->cached_cons)
|
|
|
|
xskq_cons_get_entries(q);
|
2019-12-19 19:39:27 +07:00
|
|
|
return xskq_cons_read_desc(q, desc, umem);
|
2018-05-02 18:01:34 +07:00
|
|
|
}
|
|
|
|
|
2019-12-19 19:39:30 +07:00
|
|
|
static inline void xskq_cons_release(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
/* To improve performance, only update local state here.
|
|
|
|
* Reflect this to global state when we get new entries
|
|
|
|
* from the ring in xskq_cons_get_entries().
|
|
|
|
*/
|
|
|
|
q->cached_cons++;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool xskq_cons_is_full(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
/* No barriers needed since data is not accessed */
|
|
|
|
return READ_ONCE(q->ring->producer) - READ_ONCE(q->ring->consumer) ==
|
|
|
|
q->nentries;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Functions for producers */
|
|
|
|
|
|
|
|
static inline bool xskq_prod_is_full(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
u32 free_entries = q->nentries - (q->cached_prod - q->cached_cons);
|
|
|
|
|
|
|
|
if (free_entries)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/* Refresh the local tail pointer */
|
|
|
|
q->cached_cons = READ_ONCE(q->ring->consumer);
|
|
|
|
free_entries = q->nentries - (q->cached_prod - q->cached_cons);
|
|
|
|
|
|
|
|
return !free_entries;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int xskq_prod_reserve(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
if (xskq_prod_is_full(q))
|
|
|
|
return -ENOSPC;
|
|
|
|
|
|
|
|
/* A, matches D */
|
|
|
|
q->cached_prod++;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int xskq_prod_reserve_addr(struct xsk_queue *q, u64 addr)
|
|
|
|
{
|
|
|
|
struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
|
|
|
|
|
|
|
|
if (xskq_prod_is_full(q))
|
|
|
|
return -ENOSPC;
|
|
|
|
|
|
|
|
/* A, matches D */
|
|
|
|
ring->desc[q->cached_prod++ & q->ring_mask] = addr;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2019-12-19 19:39:23 +07:00
|
|
|
static inline int xskq_prod_reserve_desc(struct xsk_queue *q,
|
|
|
|
u64 addr, u32 len)
|
2018-05-02 18:01:27 +07:00
|
|
|
{
|
|
|
|
struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring;
|
2019-12-19 19:39:23 +07:00
|
|
|
u32 idx;
|
2018-05-02 18:01:27 +07:00
|
|
|
|
2019-12-19 19:39:25 +07:00
|
|
|
if (xskq_prod_is_full(q))
|
2018-05-02 18:01:27 +07:00
|
|
|
return -ENOSPC;
|
|
|
|
|
2019-04-16 19:58:08 +07:00
|
|
|
/* A, matches D */
|
2019-12-19 19:39:22 +07:00
|
|
|
idx = q->cached_prod++ & q->ring_mask;
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 18:57:13 +07:00
|
|
|
ring->desc[idx].addr = addr;
|
2018-05-02 18:01:27 +07:00
|
|
|
ring->desc[idx].len = len;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2019-12-19 19:39:30 +07:00
|
|
|
static inline void __xskq_prod_submit(struct xsk_queue *q, u32 idx)
|
2018-05-02 18:01:34 +07:00
|
|
|
{
|
2019-12-19 19:39:30 +07:00
|
|
|
smp_wmb(); /* B, matches C */
|
|
|
|
|
|
|
|
WRITE_ONCE(q->ring->producer, idx);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void xskq_prod_submit(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
__xskq_prod_submit(q, q->cached_prod);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void xskq_prod_submit_addr(struct xsk_queue *q, u64 addr)
|
|
|
|
{
|
|
|
|
struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
|
|
|
|
u32 idx = q->ring->producer;
|
|
|
|
|
|
|
|
ring->desc[idx++ & q->ring_mask] = addr;
|
|
|
|
|
|
|
|
__xskq_prod_submit(q, idx);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void xskq_prod_submit_n(struct xsk_queue *q, u32 nb_entries)
|
|
|
|
{
|
|
|
|
__xskq_prod_submit(q, q->ring->producer + nb_entries);
|
2018-05-02 18:01:34 +07:00
|
|
|
}
|
|
|
|
|
2019-12-19 19:39:23 +07:00
|
|
|
static inline bool xskq_prod_is_empty(struct xsk_queue *q)
|
2018-05-02 18:01:27 +07:00
|
|
|
{
|
2019-12-19 19:39:21 +07:00
|
|
|
/* No barriers needed since data is not accessed */
|
|
|
|
return READ_ONCE(q->ring->consumer) == READ_ONCE(q->ring->producer);
|
2018-05-02 18:01:27 +07:00
|
|
|
}
|
|
|
|
|
2019-12-19 19:39:30 +07:00
|
|
|
/* For both producers and consumers */
|
|
|
|
|
|
|
|
static inline u64 xskq_nb_invalid_descs(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
return q ? q->invalid_descs : 0;
|
|
|
|
}
|
|
|
|
|
2018-08-31 18:40:02 +07:00
|
|
|
void xskq_set_umem(struct xsk_queue *q, u64 size, u64 chunk_mask);
|
2018-05-02 18:01:25 +07:00
|
|
|
struct xsk_queue *xskq_create(u32 nentries, bool umem_queue);
|
2018-05-02 18:01:27 +07:00
|
|
|
void xskq_destroy(struct xsk_queue *q_ops);
|
2018-05-02 18:01:24 +07:00
|
|
|
|
2018-09-07 15:18:46 +07:00
|
|
|
/* Executed by the core when the entire UMEM gets freed */
|
|
|
|
void xsk_reuseq_destroy(struct xdp_umem *umem);
|
|
|
|
|
2018-05-02 18:01:24 +07:00
|
|
|
#endif /* _LINUX_XSK_QUEUE_H */
|