2011-04-15 05:25:47 +07:00
|
|
|
/******************************************************************************
|
|
|
|
*
|
|
|
|
* Back-end of the driver for virtual block devices. This portion of the
|
|
|
|
* driver exports a 'unified' block-device interface that can be accessed
|
|
|
|
* by any operating system that implements a compatible front end. A
|
|
|
|
* reference front-end implementation can be found in:
|
2011-04-15 04:05:23 +07:00
|
|
|
* drivers/block/xen-blkfront.c
|
2011-04-15 05:25:47 +07:00
|
|
|
*
|
|
|
|
* Copyright (c) 2003-2004, Keir Fraser & Steve Hand
|
|
|
|
* Copyright (c) 2005, Christopher Clark
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU General Public License version 2
|
|
|
|
* as published by the Free Software Foundation; or, when distributed
|
|
|
|
* separately from the Linux kernel or incorporated into other
|
|
|
|
* software packages, subject to the following license:
|
|
|
|
*
|
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
|
|
* of this source file (the "Software"), to deal in the Software without
|
|
|
|
* restriction, including without limitation the rights to use, copy, modify,
|
|
|
|
* merge, publish, distribute, sublicense, and/or sell copies of the Software,
|
|
|
|
* and to permit persons to whom the Software is furnished to do so, subject to
|
|
|
|
* the following conditions:
|
|
|
|
*
|
|
|
|
* The above copyright notice and this permission notice shall be included in
|
|
|
|
* all copies or substantial portions of the Software.
|
|
|
|
*
|
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
|
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
|
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
|
|
* IN THE SOFTWARE.
|
|
|
|
*/
|
|
|
|
|
2015-04-01 22:04:22 +07:00
|
|
|
#define pr_fmt(fmt) "xen-blkback: " fmt
|
|
|
|
|
2011-04-15 05:25:47 +07:00
|
|
|
#include <linux/spinlock.h>
|
|
|
|
#include <linux/kthread.h>
|
|
|
|
#include <linux/list.h>
|
|
|
|
#include <linux/delay.h>
|
2009-02-10 03:05:51 +07:00
|
|
|
#include <linux/freezer.h>
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
#include <linux/bitmap.h>
|
2009-09-16 04:12:37 +07:00
|
|
|
|
2009-02-10 03:05:51 +07:00
|
|
|
#include <xen/events.h>
|
|
|
|
#include <xen/page.h>
|
2012-08-09 00:21:14 +07:00
|
|
|
#include <xen/xen.h>
|
2009-02-10 03:05:51 +07:00
|
|
|
#include <asm/xen/hypervisor.h>
|
|
|
|
#include <asm/xen/hypercall.h>
|
2013-02-14 17:12:09 +07:00
|
|
|
#include <xen/balloon.h>
|
2015-01-05 23:49:22 +07:00
|
|
|
#include <xen/grant_table.h>
|
2011-04-15 05:25:47 +07:00
|
|
|
#include "common.h"
|
|
|
|
|
2013-04-18 01:18:56 +07:00
|
|
|
/*
|
|
|
|
* Maximum number of unused free pages to keep in the internal buffer.
|
|
|
|
* Setting this to a value too low will reduce memory used in each backend,
|
|
|
|
* but can have a performance penalty.
|
|
|
|
*
|
|
|
|
* A sane value is xen_blkif_reqs * BLKIF_MAX_SEGMENTS_PER_REQUEST, but can
|
|
|
|
* be set to a lower value that might degrade performance on some intensive
|
|
|
|
* IO workloads.
|
|
|
|
*/
|
|
|
|
|
2013-04-18 21:06:54 +07:00
|
|
|
static int xen_blkif_max_buffer_pages = 1024;
|
2013-04-18 01:18:56 +07:00
|
|
|
module_param_named(max_buffer_pages, xen_blkif_max_buffer_pages, int, 0644);
|
|
|
|
MODULE_PARM_DESC(max_buffer_pages,
|
|
|
|
"Maximum number of free pages to keep in each block backend buffer");
|
|
|
|
|
2013-04-18 01:18:57 +07:00
|
|
|
/*
|
|
|
|
* Maximum number of grants to map persistently in blkback. For maximum
|
|
|
|
* performance this should be the total numbers of grants that can be used
|
|
|
|
* to fill the ring, but since this might become too high, specially with
|
|
|
|
* the use of indirect descriptors, we set it to a value that provides good
|
|
|
|
* performance without using too much memory.
|
|
|
|
*
|
|
|
|
* When the list of persistent grants is full we clean it up using a LRU
|
|
|
|
* algorithm.
|
|
|
|
*/
|
|
|
|
|
2013-04-18 21:06:54 +07:00
|
|
|
static int xen_blkif_max_pgrants = 1056;
|
2013-04-18 01:18:57 +07:00
|
|
|
module_param_named(max_persistent_grants, xen_blkif_max_pgrants, int, 0644);
|
|
|
|
MODULE_PARM_DESC(max_persistent_grants,
|
|
|
|
"Maximum number of grants to map persistently");
|
|
|
|
|
2015-11-14 10:12:17 +07:00
|
|
|
/*
|
|
|
|
* Maximum number of rings/queues blkback supports, allow as many queues as there
|
|
|
|
* are CPUs if user has not specified a value.
|
|
|
|
*/
|
|
|
|
unsigned int xenblk_max_queues;
|
|
|
|
module_param_named(max_queues, xenblk_max_queues, uint, 0644);
|
|
|
|
MODULE_PARM_DESC(max_queues,
|
|
|
|
"Maximum number of hardware queues per virtual disk." \
|
|
|
|
"By default it is the number of online CPUs.");
|
|
|
|
|
2015-06-03 12:40:03 +07:00
|
|
|
/*
|
|
|
|
* Maximum order of pages to be used for the shared ring between front and
|
|
|
|
* backend, 4KB page granularity is used.
|
|
|
|
*/
|
2015-10-13 23:50:11 +07:00
|
|
|
unsigned int xen_blkif_max_ring_order = XENBUS_MAX_RING_GRANT_ORDER;
|
2015-06-03 12:40:03 +07:00
|
|
|
module_param_named(max_ring_page_order, xen_blkif_max_ring_order, int, S_IRUGO);
|
|
|
|
MODULE_PARM_DESC(max_ring_page_order, "Maximum order of pages to be used for the shared ring");
|
2013-04-18 01:18:57 +07:00
|
|
|
/*
|
|
|
|
* The LRU mechanism to clean the lists of persistent grants needs to
|
|
|
|
* be executed periodically. The time interval between consecutive executions
|
|
|
|
* of the purge mechanism is set in ms.
|
|
|
|
*/
|
|
|
|
#define LRU_INTERVAL 100
|
|
|
|
|
|
|
|
/*
|
|
|
|
* When the persistent grants list is full we will remove unused grants
|
|
|
|
* from the list. The percent number of grants to be removed at each LRU
|
|
|
|
* execution.
|
|
|
|
*/
|
|
|
|
#define LRU_PERCENT_CLEAN 5
|
|
|
|
|
2011-04-15 05:25:47 +07:00
|
|
|
/* Run-time switchable: /sys/module/blkback/parameters/ */
|
2011-04-15 04:42:07 +07:00
|
|
|
static unsigned int log_stats;
|
2011-04-15 05:25:47 +07:00
|
|
|
module_param(log_stats, int, 0644);
|
|
|
|
|
|
|
|
#define BLKBACK_INVALID_HANDLE (~0)
|
|
|
|
|
2015-01-09 01:06:01 +07:00
|
|
|
/* Number of free pages to remove on each call to gnttab_free_pages */
|
2013-04-18 01:18:56 +07:00
|
|
|
#define NUM_BATCH_FREE_PAGES 10
|
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
static inline int get_free_page(struct xen_blkif_ring *ring, struct page **page)
|
2013-04-18 01:18:56 +07:00
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
spin_lock_irqsave(&ring->free_pages_lock, flags);
|
|
|
|
if (list_empty(&ring->free_pages)) {
|
|
|
|
BUG_ON(ring->free_pages_num != 0);
|
|
|
|
spin_unlock_irqrestore(&ring->free_pages_lock, flags);
|
2015-01-09 01:06:01 +07:00
|
|
|
return gnttab_alloc_pages(1, page);
|
2013-04-18 01:18:56 +07:00
|
|
|
}
|
2015-11-14 10:12:19 +07:00
|
|
|
BUG_ON(ring->free_pages_num == 0);
|
|
|
|
page[0] = list_first_entry(&ring->free_pages, struct page, lru);
|
2013-04-18 01:18:56 +07:00
|
|
|
list_del(&page[0]->lru);
|
2015-11-14 10:12:19 +07:00
|
|
|
ring->free_pages_num--;
|
|
|
|
spin_unlock_irqrestore(&ring->free_pages_lock, flags);
|
2010-02-06 02:19:33 +07:00
|
|
|
|
2013-04-18 01:18:56 +07:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
static inline void put_free_pages(struct xen_blkif_ring *ring, struct page **page,
|
2013-04-18 01:18:56 +07:00
|
|
|
int num)
|
2011-04-15 05:25:47 +07:00
|
|
|
{
|
2013-04-18 01:18:56 +07:00
|
|
|
unsigned long flags;
|
|
|
|
int i;
|
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
spin_lock_irqsave(&ring->free_pages_lock, flags);
|
2013-04-18 01:18:56 +07:00
|
|
|
for (i = 0; i < num; i++)
|
2015-11-14 10:12:19 +07:00
|
|
|
list_add(&page[i]->lru, &ring->free_pages);
|
|
|
|
ring->free_pages_num += num;
|
|
|
|
spin_unlock_irqrestore(&ring->free_pages_lock, flags);
|
2013-04-18 01:18:56 +07:00
|
|
|
}
|
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
static inline void shrink_free_pagepool(struct xen_blkif_ring *ring, int num)
|
2013-04-18 01:18:56 +07:00
|
|
|
{
|
|
|
|
/* Remove requested pages in batches of NUM_BATCH_FREE_PAGES */
|
|
|
|
struct page *page[NUM_BATCH_FREE_PAGES];
|
|
|
|
unsigned int num_pages = 0;
|
|
|
|
unsigned long flags;
|
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
spin_lock_irqsave(&ring->free_pages_lock, flags);
|
|
|
|
while (ring->free_pages_num > num) {
|
|
|
|
BUG_ON(list_empty(&ring->free_pages));
|
|
|
|
page[num_pages] = list_first_entry(&ring->free_pages,
|
2013-04-18 01:18:56 +07:00
|
|
|
struct page, lru);
|
|
|
|
list_del(&page[num_pages]->lru);
|
2015-11-14 10:12:19 +07:00
|
|
|
ring->free_pages_num--;
|
2013-04-18 01:18:56 +07:00
|
|
|
if (++num_pages == NUM_BATCH_FREE_PAGES) {
|
2015-11-14 10:12:19 +07:00
|
|
|
spin_unlock_irqrestore(&ring->free_pages_lock, flags);
|
2015-01-09 01:06:01 +07:00
|
|
|
gnttab_free_pages(num_pages, page);
|
2015-11-14 10:12:19 +07:00
|
|
|
spin_lock_irqsave(&ring->free_pages_lock, flags);
|
2013-04-18 01:18:56 +07:00
|
|
|
num_pages = 0;
|
|
|
|
}
|
|
|
|
}
|
2015-11-14 10:12:19 +07:00
|
|
|
spin_unlock_irqrestore(&ring->free_pages_lock, flags);
|
2013-04-18 01:18:56 +07:00
|
|
|
if (num_pages != 0)
|
2015-01-09 01:06:01 +07:00
|
|
|
gnttab_free_pages(num_pages, page);
|
2011-04-15 05:25:47 +07:00
|
|
|
}
|
|
|
|
|
2013-04-18 01:18:56 +07:00
|
|
|
#define vaddr(page) ((unsigned long)pfn_to_kaddr(page_to_pfn(page)))
|
|
|
|
|
2015-11-14 10:12:15 +07:00
|
|
|
static int do_block_io_op(struct xen_blkif_ring *ring);
|
|
|
|
static int dispatch_rw_block_io(struct xen_blkif_ring *ring,
|
2011-05-06 00:37:23 +07:00
|
|
|
struct blkif_request *req,
|
|
|
|
struct pending_req *pending_req);
|
2015-11-14 10:12:15 +07:00
|
|
|
static void make_response(struct xen_blkif_ring *ring, u64 id,
|
2011-04-15 05:25:47 +07:00
|
|
|
unsigned short op, int st);
|
|
|
|
|
2012-12-04 21:21:52 +07:00
|
|
|
#define foreach_grant_safe(pos, n, rbtree, node) \
|
|
|
|
for ((pos) = container_of(rb_first((rbtree)), typeof(*(pos)), node), \
|
2013-03-18 23:49:33 +07:00
|
|
|
(n) = (&(pos)->node != NULL) ? rb_next(&(pos)->node) : NULL; \
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
&(pos)->node != NULL; \
|
2012-12-04 21:21:52 +07:00
|
|
|
(pos) = container_of(n, typeof(*(pos)), node), \
|
|
|
|
(n) = (&(pos)->node != NULL) ? rb_next(&(pos)->node) : NULL)
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
|
|
|
|
|
2013-04-18 01:18:57 +07:00
|
|
|
/*
|
|
|
|
* We don't need locking around the persistent grant helpers
|
2015-11-14 10:12:19 +07:00
|
|
|
* because blkback uses a single-thread for each backend, so we
|
2013-04-18 01:18:57 +07:00
|
|
|
* can be sure that this functions will never be called recursively.
|
|
|
|
*
|
|
|
|
* The only exception to that is put_persistent_grant, that can be called
|
|
|
|
* from interrupt context (by xen_blkbk_unmap), so we have to use atomic
|
|
|
|
* bit operations to modify the flags of a persistent grant and to count
|
|
|
|
* the number of used grants.
|
|
|
|
*/
|
2015-11-14 10:12:19 +07:00
|
|
|
static int add_persistent_gnt(struct xen_blkif_ring *ring,
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
struct persistent_gnt *persistent_gnt)
|
|
|
|
{
|
2013-04-18 01:18:57 +07:00
|
|
|
struct rb_node **new = NULL, *parent = NULL;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
struct persistent_gnt *this;
|
2015-11-14 10:12:19 +07:00
|
|
|
struct xen_blkif *blkif = ring->blkif;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
if (ring->persistent_gnt_c >= xen_blkif_max_pgrants) {
|
2013-04-18 01:18:57 +07:00
|
|
|
if (!blkif->vbd.overflow_max_grants)
|
|
|
|
blkif->vbd.overflow_max_grants = 1;
|
|
|
|
return -EBUSY;
|
|
|
|
}
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
/* Figure out where to put new node */
|
2015-11-14 10:12:19 +07:00
|
|
|
new = &ring->persistent_gnts.rb_node;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
while (*new) {
|
|
|
|
this = container_of(*new, struct persistent_gnt, node);
|
|
|
|
|
|
|
|
parent = *new;
|
|
|
|
if (persistent_gnt->gnt < this->gnt)
|
|
|
|
new = &((*new)->rb_left);
|
|
|
|
else if (persistent_gnt->gnt > this->gnt)
|
|
|
|
new = &((*new)->rb_right);
|
|
|
|
else {
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_alert_ratelimited("trying to add a gref that's already in the tree\n");
|
2013-04-18 01:18:56 +07:00
|
|
|
return -EINVAL;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-04-18 01:18:57 +07:00
|
|
|
bitmap_zero(persistent_gnt->flags, PERSISTENT_GNT_FLAGS_SIZE);
|
|
|
|
set_bit(PERSISTENT_GNT_ACTIVE, persistent_gnt->flags);
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
/* Add new node and rebalance tree. */
|
|
|
|
rb_link_node(&(persistent_gnt->node), parent, new);
|
2015-11-14 10:12:19 +07:00
|
|
|
rb_insert_color(&(persistent_gnt->node), &ring->persistent_gnts);
|
|
|
|
ring->persistent_gnt_c++;
|
|
|
|
atomic_inc(&ring->persistent_gnt_in_use);
|
2013-04-18 01:18:56 +07:00
|
|
|
return 0;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
}
|
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
static struct persistent_gnt *get_persistent_gnt(struct xen_blkif_ring *ring,
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
grant_ref_t gref)
|
|
|
|
{
|
|
|
|
struct persistent_gnt *data;
|
2013-04-18 01:18:57 +07:00
|
|
|
struct rb_node *node = NULL;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
node = ring->persistent_gnts.rb_node;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
while (node) {
|
|
|
|
data = container_of(node, struct persistent_gnt, node);
|
|
|
|
|
|
|
|
if (gref < data->gnt)
|
|
|
|
node = node->rb_left;
|
|
|
|
else if (gref > data->gnt)
|
|
|
|
node = node->rb_right;
|
2013-04-18 01:18:57 +07:00
|
|
|
else {
|
|
|
|
if(test_bit(PERSISTENT_GNT_ACTIVE, data->flags)) {
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_alert_ratelimited("requesting a grant already in use\n");
|
2013-04-18 01:18:57 +07:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
set_bit(PERSISTENT_GNT_ACTIVE, data->flags);
|
2015-11-14 10:12:19 +07:00
|
|
|
atomic_inc(&ring->persistent_gnt_in_use);
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
return data;
|
2013-04-18 01:18:57 +07:00
|
|
|
}
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
}
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
static void put_persistent_gnt(struct xen_blkif_ring *ring,
|
2013-04-18 01:18:57 +07:00
|
|
|
struct persistent_gnt *persistent_gnt)
|
|
|
|
{
|
|
|
|
if(!test_bit(PERSISTENT_GNT_ACTIVE, persistent_gnt->flags))
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_alert_ratelimited("freeing a grant already unused\n");
|
2013-04-18 01:18:57 +07:00
|
|
|
set_bit(PERSISTENT_GNT_WAS_ACTIVE, persistent_gnt->flags);
|
|
|
|
clear_bit(PERSISTENT_GNT_ACTIVE, persistent_gnt->flags);
|
2015-11-14 10:12:19 +07:00
|
|
|
atomic_dec(&ring->persistent_gnt_in_use);
|
2013-04-18 01:18:57 +07:00
|
|
|
}
|
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
static void free_persistent_gnts(struct xen_blkif_ring *ring, struct rb_root *root,
|
2013-04-18 01:18:56 +07:00
|
|
|
unsigned int num)
|
2012-11-17 01:26:48 +07:00
|
|
|
{
|
|
|
|
struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
|
|
|
|
struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
|
|
|
|
struct persistent_gnt *persistent_gnt;
|
2012-12-04 21:21:52 +07:00
|
|
|
struct rb_node *n;
|
2012-11-17 01:26:48 +07:00
|
|
|
int segs_to_unmap = 0;
|
2015-01-05 23:49:22 +07:00
|
|
|
struct gntab_unmap_queue_data unmap_data;
|
|
|
|
|
|
|
|
unmap_data.pages = pages;
|
|
|
|
unmap_data.unmap_ops = unmap;
|
|
|
|
unmap_data.kunmap_ops = NULL;
|
2012-11-17 01:26:48 +07:00
|
|
|
|
2012-12-04 21:21:52 +07:00
|
|
|
foreach_grant_safe(persistent_gnt, n, root, node) {
|
2012-11-17 01:26:48 +07:00
|
|
|
BUG_ON(persistent_gnt->handle ==
|
|
|
|
BLKBACK_INVALID_HANDLE);
|
|
|
|
gnttab_set_unmap_op(&unmap[segs_to_unmap],
|
|
|
|
(unsigned long) pfn_to_kaddr(page_to_pfn(
|
|
|
|
persistent_gnt->page)),
|
|
|
|
GNTMAP_host_map,
|
|
|
|
persistent_gnt->handle);
|
|
|
|
|
|
|
|
pages[segs_to_unmap] = persistent_gnt->page;
|
|
|
|
|
|
|
|
if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST ||
|
|
|
|
!rb_next(&persistent_gnt->node)) {
|
2015-01-05 23:49:22 +07:00
|
|
|
|
|
|
|
unmap_data.count = segs_to_unmap;
|
2015-04-03 13:42:59 +07:00
|
|
|
BUG_ON(gnttab_unmap_refs_sync(&unmap_data));
|
2015-01-05 23:49:22 +07:00
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
put_free_pages(ring, pages, segs_to_unmap);
|
2012-11-17 01:26:48 +07:00
|
|
|
segs_to_unmap = 0;
|
|
|
|
}
|
2012-12-04 21:21:52 +07:00
|
|
|
|
|
|
|
rb_erase(&persistent_gnt->node, root);
|
|
|
|
kfree(persistent_gnt);
|
|
|
|
num--;
|
2012-11-17 01:26:48 +07:00
|
|
|
}
|
|
|
|
BUG_ON(num != 0);
|
|
|
|
}
|
|
|
|
|
2014-02-12 10:34:03 +07:00
|
|
|
void xen_blkbk_unmap_purged_grants(struct work_struct *work)
|
2013-04-18 01:18:57 +07:00
|
|
|
{
|
|
|
|
struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
|
|
|
|
struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
|
|
|
|
struct persistent_gnt *persistent_gnt;
|
2015-04-03 13:42:58 +07:00
|
|
|
int segs_to_unmap = 0;
|
2015-11-14 10:12:19 +07:00
|
|
|
struct xen_blkif_ring *ring = container_of(work, typeof(*ring), persistent_purge_work);
|
2015-04-03 13:42:58 +07:00
|
|
|
struct gntab_unmap_queue_data unmap_data;
|
|
|
|
|
|
|
|
unmap_data.pages = pages;
|
|
|
|
unmap_data.unmap_ops = unmap;
|
|
|
|
unmap_data.kunmap_ops = NULL;
|
2013-04-18 01:18:57 +07:00
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
while(!list_empty(&ring->persistent_purge_list)) {
|
|
|
|
persistent_gnt = list_first_entry(&ring->persistent_purge_list,
|
2013-04-18 01:18:57 +07:00
|
|
|
struct persistent_gnt,
|
|
|
|
remove_node);
|
|
|
|
list_del(&persistent_gnt->remove_node);
|
|
|
|
|
|
|
|
gnttab_set_unmap_op(&unmap[segs_to_unmap],
|
|
|
|
vaddr(persistent_gnt->page),
|
|
|
|
GNTMAP_host_map,
|
|
|
|
persistent_gnt->handle);
|
|
|
|
|
|
|
|
pages[segs_to_unmap] = persistent_gnt->page;
|
|
|
|
|
|
|
|
if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST) {
|
2015-04-03 13:42:58 +07:00
|
|
|
unmap_data.count = segs_to_unmap;
|
2015-04-03 13:42:59 +07:00
|
|
|
BUG_ON(gnttab_unmap_refs_sync(&unmap_data));
|
2015-11-14 10:12:19 +07:00
|
|
|
put_free_pages(ring, pages, segs_to_unmap);
|
2013-04-18 01:18:57 +07:00
|
|
|
segs_to_unmap = 0;
|
|
|
|
}
|
|
|
|
kfree(persistent_gnt);
|
|
|
|
}
|
|
|
|
if (segs_to_unmap > 0) {
|
2015-04-03 13:42:58 +07:00
|
|
|
unmap_data.count = segs_to_unmap;
|
2015-04-03 13:42:59 +07:00
|
|
|
BUG_ON(gnttab_unmap_refs_sync(&unmap_data));
|
2015-11-14 10:12:19 +07:00
|
|
|
put_free_pages(ring, pages, segs_to_unmap);
|
2013-04-18 01:18:57 +07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
static void purge_persistent_gnt(struct xen_blkif_ring *ring)
|
2013-04-18 01:18:57 +07:00
|
|
|
{
|
|
|
|
struct persistent_gnt *persistent_gnt;
|
|
|
|
struct rb_node *n;
|
|
|
|
unsigned int num_clean, total;
|
2013-06-21 17:56:53 +07:00
|
|
|
bool scan_used = false, clean_used = false;
|
2013-04-18 01:18:57 +07:00
|
|
|
struct rb_root *root;
|
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
if (ring->persistent_gnt_c < xen_blkif_max_pgrants ||
|
|
|
|
(ring->persistent_gnt_c == xen_blkif_max_pgrants &&
|
|
|
|
!ring->blkif->vbd.overflow_max_grants)) {
|
2015-11-14 10:12:15 +07:00
|
|
|
goto out;
|
2013-04-18 01:18:57 +07:00
|
|
|
}
|
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
if (work_busy(&ring->persistent_purge_work)) {
|
2015-07-22 13:40:10 +07:00
|
|
|
pr_alert_ratelimited("Scheduled work from previous purge is still busy, cannot purge list\n");
|
2015-11-14 10:12:15 +07:00
|
|
|
goto out;
|
2013-04-18 01:18:57 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
num_clean = (xen_blkif_max_pgrants / 100) * LRU_PERCENT_CLEAN;
|
2015-11-14 10:12:19 +07:00
|
|
|
num_clean = ring->persistent_gnt_c - xen_blkif_max_pgrants + num_clean;
|
|
|
|
num_clean = min(ring->persistent_gnt_c, num_clean);
|
2013-06-21 17:56:53 +07:00
|
|
|
if ((num_clean == 0) ||
|
2015-11-14 10:12:19 +07:00
|
|
|
(num_clean > (ring->persistent_gnt_c - atomic_read(&ring->persistent_gnt_in_use))))
|
2015-11-14 10:12:15 +07:00
|
|
|
goto out;
|
2013-04-18 01:18:57 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* At this point, we can assure that there will be no calls
|
|
|
|
* to get_persistent_grant (because we are executing this code from
|
|
|
|
* xen_blkif_schedule), there can only be calls to put_persistent_gnt,
|
|
|
|
* which means that the number of currently used grants will go down,
|
|
|
|
* but never up, so we will always be able to remove the requested
|
|
|
|
* number of grants.
|
|
|
|
*/
|
|
|
|
|
|
|
|
total = num_clean;
|
|
|
|
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_debug("Going to purge %u persistent grants\n", num_clean);
|
2013-04-18 01:18:57 +07:00
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
BUG_ON(!list_empty(&ring->persistent_purge_list));
|
|
|
|
root = &ring->persistent_gnts;
|
2013-04-18 01:18:57 +07:00
|
|
|
purge_list:
|
|
|
|
foreach_grant_safe(persistent_gnt, n, root, node) {
|
|
|
|
BUG_ON(persistent_gnt->handle ==
|
|
|
|
BLKBACK_INVALID_HANDLE);
|
|
|
|
|
2013-06-21 17:56:53 +07:00
|
|
|
if (clean_used) {
|
|
|
|
clear_bit(PERSISTENT_GNT_WAS_ACTIVE, persistent_gnt->flags);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2013-04-18 01:18:57 +07:00
|
|
|
if (test_bit(PERSISTENT_GNT_ACTIVE, persistent_gnt->flags))
|
|
|
|
continue;
|
|
|
|
if (!scan_used &&
|
|
|
|
(test_bit(PERSISTENT_GNT_WAS_ACTIVE, persistent_gnt->flags)))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
rb_erase(&persistent_gnt->node, root);
|
|
|
|
list_add(&persistent_gnt->remove_node,
|
2015-11-14 10:12:19 +07:00
|
|
|
&ring->persistent_purge_list);
|
2013-04-18 01:18:57 +07:00
|
|
|
if (--num_clean == 0)
|
|
|
|
goto finished;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* If we get here it means we also need to start cleaning
|
|
|
|
* grants that were used since last purge in order to cope
|
|
|
|
* with the requested num
|
|
|
|
*/
|
2013-06-21 17:56:53 +07:00
|
|
|
if (!scan_used && !clean_used) {
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_debug("Still missing %u purged frames\n", num_clean);
|
2013-04-18 01:18:57 +07:00
|
|
|
scan_used = true;
|
|
|
|
goto purge_list;
|
|
|
|
}
|
|
|
|
finished:
|
2013-06-21 17:56:53 +07:00
|
|
|
if (!clean_used) {
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_debug("Finished scanning for grants to clean, removing used flag\n");
|
2013-06-21 17:56:53 +07:00
|
|
|
clean_used = true;
|
|
|
|
goto purge_list;
|
2013-04-18 01:18:57 +07:00
|
|
|
}
|
2013-06-21 17:56:53 +07:00
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
ring->persistent_gnt_c -= (total - num_clean);
|
|
|
|
ring->blkif->vbd.overflow_max_grants = 0;
|
2013-04-18 01:18:57 +07:00
|
|
|
|
|
|
|
/* We can defer this work */
|
2015-11-14 10:12:19 +07:00
|
|
|
schedule_work(&ring->persistent_purge_work);
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_debug("Purged %u/%u\n", (total - num_clean), total);
|
2015-11-14 10:12:15 +07:00
|
|
|
|
|
|
|
out:
|
2013-04-18 01:18:57 +07:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2011-04-15 04:05:23 +07:00
|
|
|
/*
|
|
|
|
* Retrieve from the 'pending_reqs' a free pending_req structure to be used.
|
2011-04-15 05:25:47 +07:00
|
|
|
*/
|
2015-11-14 10:12:15 +07:00
|
|
|
static struct pending_req *alloc_req(struct xen_blkif_ring *ring)
|
2011-04-15 05:25:47 +07:00
|
|
|
{
|
2011-04-15 04:42:07 +07:00
|
|
|
struct pending_req *req = NULL;
|
2011-04-15 05:25:47 +07:00
|
|
|
unsigned long flags;
|
|
|
|
|
2015-11-14 10:12:15 +07:00
|
|
|
spin_lock_irqsave(&ring->pending_free_lock, flags);
|
|
|
|
if (!list_empty(&ring->pending_free)) {
|
|
|
|
req = list_entry(ring->pending_free.next, struct pending_req,
|
2011-04-15 04:42:07 +07:00
|
|
|
free_list);
|
2011-04-15 05:25:47 +07:00
|
|
|
list_del(&req->free_list);
|
|
|
|
}
|
2015-11-14 10:12:15 +07:00
|
|
|
spin_unlock_irqrestore(&ring->pending_free_lock, flags);
|
2011-04-15 05:25:47 +07:00
|
|
|
return req;
|
|
|
|
}
|
|
|
|
|
2011-04-15 04:05:23 +07:00
|
|
|
/*
|
|
|
|
* Return the 'pending_req' structure back to the freepool. We also
|
|
|
|
* wake up the thread if it was waiting for a free page.
|
|
|
|
*/
|
2015-11-14 10:12:15 +07:00
|
|
|
static void free_req(struct xen_blkif_ring *ring, struct pending_req *req)
|
2011-04-15 05:25:47 +07:00
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
int was_empty;
|
|
|
|
|
2015-11-14 10:12:15 +07:00
|
|
|
spin_lock_irqsave(&ring->pending_free_lock, flags);
|
|
|
|
was_empty = list_empty(&ring->pending_free);
|
|
|
|
list_add(&req->free_list, &ring->pending_free);
|
|
|
|
spin_unlock_irqrestore(&ring->pending_free_lock, flags);
|
2011-04-15 05:25:47 +07:00
|
|
|
if (was_empty)
|
2015-11-14 10:12:15 +07:00
|
|
|
wake_up(&ring->pending_free_wq);
|
2011-04-15 05:25:47 +07:00
|
|
|
}
|
|
|
|
|
2011-04-20 21:57:29 +07:00
|
|
|
/*
|
|
|
|
* Routines for managing virtual block devices (vbds).
|
|
|
|
*/
|
2011-05-13 03:53:56 +07:00
|
|
|
static int xen_vbd_translate(struct phys_req *req, struct xen_blkif *blkif,
|
|
|
|
int operation)
|
2011-04-20 21:57:29 +07:00
|
|
|
{
|
2011-05-13 03:53:56 +07:00
|
|
|
struct xen_vbd *vbd = &blkif->vbd;
|
2011-04-20 21:57:29 +07:00
|
|
|
int rc = -EACCES;
|
|
|
|
|
2016-06-06 02:32:09 +07:00
|
|
|
if ((operation != REQ_OP_READ) && vbd->readonly)
|
2011-04-20 21:57:29 +07:00
|
|
|
goto out;
|
|
|
|
|
2011-05-17 17:07:05 +07:00
|
|
|
if (likely(req->nr_sects)) {
|
|
|
|
blkif_sector_t end = req->sector_number + req->nr_sects;
|
|
|
|
|
|
|
|
if (unlikely(end < req->sector_number))
|
|
|
|
goto out;
|
|
|
|
if (unlikely(end > vbd_sz(vbd)))
|
|
|
|
goto out;
|
|
|
|
}
|
2011-04-20 21:57:29 +07:00
|
|
|
|
|
|
|
req->dev = vbd->pdevice;
|
|
|
|
req->bdev = vbd->bdev;
|
|
|
|
rc = 0;
|
|
|
|
|
|
|
|
out:
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
|
2011-05-13 03:53:56 +07:00
|
|
|
static void xen_vbd_resize(struct xen_blkif *blkif)
|
2011-04-20 21:57:29 +07:00
|
|
|
{
|
2011-05-13 03:53:56 +07:00
|
|
|
struct xen_vbd *vbd = &blkif->vbd;
|
2011-04-20 21:57:29 +07:00
|
|
|
struct xenbus_transaction xbt;
|
|
|
|
int err;
|
2011-04-20 22:50:43 +07:00
|
|
|
struct xenbus_device *dev = xen_blkbk_xenbus(blkif->be);
|
2011-04-20 22:21:43 +07:00
|
|
|
unsigned long long new_size = vbd_sz(vbd);
|
2011-04-20 21:57:29 +07:00
|
|
|
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_info("VBD Resize: Domid: %d, Device: (%d, %d)\n",
|
2011-04-20 21:57:29 +07:00
|
|
|
blkif->domid, MAJOR(vbd->pdevice), MINOR(vbd->pdevice));
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_info("VBD Resize: new size %llu\n", new_size);
|
2011-04-20 21:57:29 +07:00
|
|
|
vbd->size = new_size;
|
|
|
|
again:
|
|
|
|
err = xenbus_transaction_start(&xbt);
|
|
|
|
if (err) {
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_warn("Error starting transaction\n");
|
2011-04-20 21:57:29 +07:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
err = xenbus_printf(xbt, dev->nodename, "sectors", "%llu",
|
2011-04-20 22:21:43 +07:00
|
|
|
(unsigned long long)vbd_sz(vbd));
|
2011-04-20 21:57:29 +07:00
|
|
|
if (err) {
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_warn("Error writing new size\n");
|
2011-04-20 21:57:29 +07:00
|
|
|
goto abort;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* Write the current state; we will use this to synchronize
|
|
|
|
* the front-end. If the current state is "connected" the
|
|
|
|
* front-end will get the new size information online.
|
|
|
|
*/
|
|
|
|
err = xenbus_printf(xbt, dev->nodename, "state", "%d", dev->state);
|
|
|
|
if (err) {
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_warn("Error writing the state\n");
|
2011-04-20 21:57:29 +07:00
|
|
|
goto abort;
|
|
|
|
}
|
|
|
|
|
|
|
|
err = xenbus_transaction_end(xbt, 0);
|
|
|
|
if (err == -EAGAIN)
|
|
|
|
goto again;
|
|
|
|
if (err)
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_warn("Error ending transaction\n");
|
2011-05-13 20:45:40 +07:00
|
|
|
return;
|
2011-04-20 21:57:29 +07:00
|
|
|
abort:
|
|
|
|
xenbus_transaction_end(xbt, 1);
|
|
|
|
}
|
|
|
|
|
2011-04-15 04:05:23 +07:00
|
|
|
/*
|
2011-04-15 21:58:05 +07:00
|
|
|
* Notification from the guest OS.
|
|
|
|
*/
|
2015-11-14 10:12:15 +07:00
|
|
|
static void blkif_notify_work(struct xen_blkif_ring *ring)
|
2011-04-15 05:25:47 +07:00
|
|
|
{
|
2015-11-14 10:12:15 +07:00
|
|
|
ring->waiting_reqs = 1;
|
|
|
|
wake_up(&ring->wq);
|
2011-04-15 21:58:05 +07:00
|
|
|
}
|
2011-04-15 05:25:47 +07:00
|
|
|
|
2011-04-20 22:50:43 +07:00
|
|
|
irqreturn_t xen_blkif_be_int(int irq, void *dev_id)
|
2011-04-15 21:58:05 +07:00
|
|
|
{
|
|
|
|
blkif_notify_work(dev_id);
|
|
|
|
return IRQ_HANDLED;
|
2011-04-15 05:25:47 +07:00
|
|
|
}
|
|
|
|
|
2011-04-15 04:42:07 +07:00
|
|
|
/*
|
2011-04-15 05:25:47 +07:00
|
|
|
* SCHEDULER FUNCTIONS
|
|
|
|
*/
|
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
static void print_stats(struct xen_blkif_ring *ring)
|
2011-04-15 05:25:47 +07:00
|
|
|
{
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_info("(%s): oo %3llu | rd %4llu | wr %4llu | f %4llu"
|
2013-04-18 01:18:57 +07:00
|
|
|
" | ds %4llu | pg: %4u/%4d\n",
|
2015-12-09 06:44:02 +07:00
|
|
|
current->comm, ring->st_oo_req,
|
|
|
|
ring->st_rd_req, ring->st_wr_req,
|
|
|
|
ring->st_f_req, ring->st_ds_req,
|
2015-11-14 10:12:19 +07:00
|
|
|
ring->persistent_gnt_c,
|
2013-04-18 01:18:57 +07:00
|
|
|
xen_blkif_max_pgrants);
|
2015-12-09 06:44:02 +07:00
|
|
|
ring->st_print = jiffies + msecs_to_jiffies(10 * 1000);
|
|
|
|
ring->st_rd_req = 0;
|
|
|
|
ring->st_wr_req = 0;
|
|
|
|
ring->st_oo_req = 0;
|
|
|
|
ring->st_ds_req = 0;
|
2011-04-15 05:25:47 +07:00
|
|
|
}
|
|
|
|
|
2011-04-20 22:50:43 +07:00
|
|
|
int xen_blkif_schedule(void *arg)
|
2011-04-15 05:25:47 +07:00
|
|
|
{
|
2015-11-14 10:12:15 +07:00
|
|
|
struct xen_blkif_ring *ring = arg;
|
|
|
|
struct xen_blkif *blkif = ring->blkif;
|
2011-05-13 03:53:56 +07:00
|
|
|
struct xen_vbd *vbd = &blkif->vbd;
|
2013-04-18 01:18:57 +07:00
|
|
|
unsigned long timeout;
|
xen/blkback: Check for insane amounts of request on the ring (v6).
Check that the ring does not have an insane amount of requests
(more than there could fit on the ring).
If we detect this case we will stop processing the requests
and wait until the XenBus disconnects the ring.
The existing check RING_REQUEST_CONS_OVERFLOW which checks for how
many responses we have created in the past (rsp_prod_pvt) vs
requests consumed (req_cons) and whether said difference is greater or
equal to the size of the ring, does not catch this case.
Wha the condition does check if there is a need to process more
as we still have a backlog of responses to finish. Note that both
of those values (rsp_prod_pvt and req_cons) are not exposed on the
shared ring.
To understand this problem a mini crash course in ring protocol
response/request updates is in place.
There are four entries: req_prod and rsp_prod; req_event and rsp_event
to track the ring entries. We are only concerned about the first two -
which set the tone of this bug.
The req_prod is a value incremented by frontend for each request put
on the ring. Conversely the rsp_prod is a value incremented by the backend
for each response put on the ring (rsp_prod gets set by rsp_prod_pvt when
pushing the responses on the ring). Both values can
wrap and are modulo the size of the ring (in block case that is 32).
Please see RING_GET_REQUEST and RING_GET_RESPONSE for the more details.
The culprit here is that if the difference between the
req_prod and req_cons is greater than the ring size we have a problem.
Fortunately for us, the '__do_block_io_op' loop:
rc = blk_rings->common.req_cons;
rp = blk_rings->common.sring->req_prod;
while (rc != rp) {
..
blk_rings->common.req_cons = ++rc; /* before make_response() */
}
will loop up to the point when rc == rp. The macros inside of the
loop (RING_GET_REQUEST) is smart and is indexing based on the modulo
of the ring size. If the frontend has provided a bogus req_prod value
we will loop until the 'rc == rp' - which means we could be processing
already processed requests (or responses) often.
The reason the RING_REQUEST_CONS_OVERFLOW is not helping here is
b/c it only tracks how many responses we have internally produced
and whether we would should process more. The astute reader will
notice that the macro RING_REQUEST_CONS_OVERFLOW provides two
arguments - more on this later.
For example, if we were to enter this function with these values:
blk_rings->common.sring->req_prod = X+31415 (X is the value from
the last time __do_block_io_op was called).
blk_rings->common.req_cons = X
blk_rings->common.rsp_prod_pvt = X
The RING_REQUEST_CONS_OVERFLOW(&blk_rings->common, blk_rings->common.req_cons)
is doing:
req_cons - rsp_prod_pvt >= 32
Which is,
X - X >= 32 or 0 >= 32
And that is false, so we continue on looping (this bug).
If we re-use said macro RING_REQUEST_CONS_OVERFLOW and pass in the rp
instead (sring->req_prod) of rc, the this macro can do the check:
req_prod - rsp_prov_pvt >= 32
Which is,
X + 31415 - X >= 32 , or 31415 >= 32
which is true, so we can error out and break out of the function.
Unfortunatly the difference between rsp_prov_pvt and req_prod can be
at 32 (which would error out in the macro). This condition exists when
the backend is lagging behind with the responses and still has not finished
responding to all of them (so make_response has not been called), and
the rsp_prov_pvt + 32 == req_cons. This ends up with us not being able
to use said macro.
Hence introducing a new macro called RING_REQUEST_PROD_OVERFLOW which does
a simple check of:
req_prod - rsp_prod_pvt > RING_SIZE
And with the X values from above:
X + 31415 - X > 32
Returns true. Also not that if the ring is full (which is where
the RING_REQUEST_CONS_OVERFLOW triggered), we would not hit the
same condition:
X + 32 - X > 32
Which is false.
Lets use that macro.
Note that in v5 of this patchset the macro was different - we used an
earlier version.
Cc: stable@vger.kernel.org
[v1: Move the check outside the loop]
[v2: Add a pr_warn as suggested by David]
[v3: Use RING_REQUEST_CONS_OVERFLOW as suggested by Jan]
[v4: Move wake_up after kthread_stop as suggested by Jan]
[v5: Use RING_REQUEST_PROD_OVERFLOW instead]
[v6: Use RING_REQUEST_PROD_OVERFLOW - Jan's version]
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>
gadsa
2013-01-24 04:54:32 +07:00
|
|
|
int ret;
|
2011-04-15 05:25:47 +07:00
|
|
|
|
2011-04-20 22:50:43 +07:00
|
|
|
xen_blkif_get(blkif);
|
2011-04-15 05:25:47 +07:00
|
|
|
|
2015-10-26 12:47:21 +07:00
|
|
|
set_freezable();
|
2011-04-15 05:25:47 +07:00
|
|
|
while (!kthread_should_stop()) {
|
|
|
|
if (try_to_freeze())
|
|
|
|
continue;
|
2011-04-20 22:21:43 +07:00
|
|
|
if (unlikely(vbd->size != vbd_sz(vbd)))
|
2011-05-13 03:53:56 +07:00
|
|
|
xen_vbd_resize(blkif);
|
2011-04-15 05:25:47 +07:00
|
|
|
|
2013-04-18 01:18:57 +07:00
|
|
|
timeout = msecs_to_jiffies(LRU_INTERVAL);
|
|
|
|
|
|
|
|
timeout = wait_event_interruptible_timeout(
|
2015-11-14 10:12:15 +07:00
|
|
|
ring->wq,
|
|
|
|
ring->waiting_reqs || kthread_should_stop(),
|
2013-04-18 01:18:57 +07:00
|
|
|
timeout);
|
|
|
|
if (timeout == 0)
|
|
|
|
goto purge_gnt_list;
|
|
|
|
timeout = wait_event_interruptible_timeout(
|
2015-11-14 10:12:15 +07:00
|
|
|
ring->pending_free_wq,
|
|
|
|
!list_empty(&ring->pending_free) ||
|
2013-04-18 01:18:57 +07:00
|
|
|
kthread_should_stop(),
|
|
|
|
timeout);
|
|
|
|
if (timeout == 0)
|
|
|
|
goto purge_gnt_list;
|
2011-04-15 05:25:47 +07:00
|
|
|
|
2015-11-14 10:12:15 +07:00
|
|
|
ring->waiting_reqs = 0;
|
2011-04-15 05:25:47 +07:00
|
|
|
smp_mb(); /* clear flag *before* checking for work */
|
|
|
|
|
2015-11-14 10:12:15 +07:00
|
|
|
ret = do_block_io_op(ring);
|
xen/blkback: Check for insane amounts of request on the ring (v6).
Check that the ring does not have an insane amount of requests
(more than there could fit on the ring).
If we detect this case we will stop processing the requests
and wait until the XenBus disconnects the ring.
The existing check RING_REQUEST_CONS_OVERFLOW which checks for how
many responses we have created in the past (rsp_prod_pvt) vs
requests consumed (req_cons) and whether said difference is greater or
equal to the size of the ring, does not catch this case.
Wha the condition does check if there is a need to process more
as we still have a backlog of responses to finish. Note that both
of those values (rsp_prod_pvt and req_cons) are not exposed on the
shared ring.
To understand this problem a mini crash course in ring protocol
response/request updates is in place.
There are four entries: req_prod and rsp_prod; req_event and rsp_event
to track the ring entries. We are only concerned about the first two -
which set the tone of this bug.
The req_prod is a value incremented by frontend for each request put
on the ring. Conversely the rsp_prod is a value incremented by the backend
for each response put on the ring (rsp_prod gets set by rsp_prod_pvt when
pushing the responses on the ring). Both values can
wrap and are modulo the size of the ring (in block case that is 32).
Please see RING_GET_REQUEST and RING_GET_RESPONSE for the more details.
The culprit here is that if the difference between the
req_prod and req_cons is greater than the ring size we have a problem.
Fortunately for us, the '__do_block_io_op' loop:
rc = blk_rings->common.req_cons;
rp = blk_rings->common.sring->req_prod;
while (rc != rp) {
..
blk_rings->common.req_cons = ++rc; /* before make_response() */
}
will loop up to the point when rc == rp. The macros inside of the
loop (RING_GET_REQUEST) is smart and is indexing based on the modulo
of the ring size. If the frontend has provided a bogus req_prod value
we will loop until the 'rc == rp' - which means we could be processing
already processed requests (or responses) often.
The reason the RING_REQUEST_CONS_OVERFLOW is not helping here is
b/c it only tracks how many responses we have internally produced
and whether we would should process more. The astute reader will
notice that the macro RING_REQUEST_CONS_OVERFLOW provides two
arguments - more on this later.
For example, if we were to enter this function with these values:
blk_rings->common.sring->req_prod = X+31415 (X is the value from
the last time __do_block_io_op was called).
blk_rings->common.req_cons = X
blk_rings->common.rsp_prod_pvt = X
The RING_REQUEST_CONS_OVERFLOW(&blk_rings->common, blk_rings->common.req_cons)
is doing:
req_cons - rsp_prod_pvt >= 32
Which is,
X - X >= 32 or 0 >= 32
And that is false, so we continue on looping (this bug).
If we re-use said macro RING_REQUEST_CONS_OVERFLOW and pass in the rp
instead (sring->req_prod) of rc, the this macro can do the check:
req_prod - rsp_prov_pvt >= 32
Which is,
X + 31415 - X >= 32 , or 31415 >= 32
which is true, so we can error out and break out of the function.
Unfortunatly the difference between rsp_prov_pvt and req_prod can be
at 32 (which would error out in the macro). This condition exists when
the backend is lagging behind with the responses and still has not finished
responding to all of them (so make_response has not been called), and
the rsp_prov_pvt + 32 == req_cons. This ends up with us not being able
to use said macro.
Hence introducing a new macro called RING_REQUEST_PROD_OVERFLOW which does
a simple check of:
req_prod - rsp_prod_pvt > RING_SIZE
And with the X values from above:
X + 31415 - X > 32
Returns true. Also not that if the ring is full (which is where
the RING_REQUEST_CONS_OVERFLOW triggered), we would not hit the
same condition:
X + 32 - X > 32
Which is false.
Lets use that macro.
Note that in v5 of this patchset the macro was different - we used an
earlier version.
Cc: stable@vger.kernel.org
[v1: Move the check outside the loop]
[v2: Add a pr_warn as suggested by David]
[v3: Use RING_REQUEST_CONS_OVERFLOW as suggested by Jan]
[v4: Move wake_up after kthread_stop as suggested by Jan]
[v5: Use RING_REQUEST_PROD_OVERFLOW instead]
[v6: Use RING_REQUEST_PROD_OVERFLOW - Jan's version]
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>
gadsa
2013-01-24 04:54:32 +07:00
|
|
|
if (ret > 0)
|
2015-11-14 10:12:15 +07:00
|
|
|
ring->waiting_reqs = 1;
|
xen/blkback: Check for insane amounts of request on the ring (v6).
Check that the ring does not have an insane amount of requests
(more than there could fit on the ring).
If we detect this case we will stop processing the requests
and wait until the XenBus disconnects the ring.
The existing check RING_REQUEST_CONS_OVERFLOW which checks for how
many responses we have created in the past (rsp_prod_pvt) vs
requests consumed (req_cons) and whether said difference is greater or
equal to the size of the ring, does not catch this case.
Wha the condition does check if there is a need to process more
as we still have a backlog of responses to finish. Note that both
of those values (rsp_prod_pvt and req_cons) are not exposed on the
shared ring.
To understand this problem a mini crash course in ring protocol
response/request updates is in place.
There are four entries: req_prod and rsp_prod; req_event and rsp_event
to track the ring entries. We are only concerned about the first two -
which set the tone of this bug.
The req_prod is a value incremented by frontend for each request put
on the ring. Conversely the rsp_prod is a value incremented by the backend
for each response put on the ring (rsp_prod gets set by rsp_prod_pvt when
pushing the responses on the ring). Both values can
wrap and are modulo the size of the ring (in block case that is 32).
Please see RING_GET_REQUEST and RING_GET_RESPONSE for the more details.
The culprit here is that if the difference between the
req_prod and req_cons is greater than the ring size we have a problem.
Fortunately for us, the '__do_block_io_op' loop:
rc = blk_rings->common.req_cons;
rp = blk_rings->common.sring->req_prod;
while (rc != rp) {
..
blk_rings->common.req_cons = ++rc; /* before make_response() */
}
will loop up to the point when rc == rp. The macros inside of the
loop (RING_GET_REQUEST) is smart and is indexing based on the modulo
of the ring size. If the frontend has provided a bogus req_prod value
we will loop until the 'rc == rp' - which means we could be processing
already processed requests (or responses) often.
The reason the RING_REQUEST_CONS_OVERFLOW is not helping here is
b/c it only tracks how many responses we have internally produced
and whether we would should process more. The astute reader will
notice that the macro RING_REQUEST_CONS_OVERFLOW provides two
arguments - more on this later.
For example, if we were to enter this function with these values:
blk_rings->common.sring->req_prod = X+31415 (X is the value from
the last time __do_block_io_op was called).
blk_rings->common.req_cons = X
blk_rings->common.rsp_prod_pvt = X
The RING_REQUEST_CONS_OVERFLOW(&blk_rings->common, blk_rings->common.req_cons)
is doing:
req_cons - rsp_prod_pvt >= 32
Which is,
X - X >= 32 or 0 >= 32
And that is false, so we continue on looping (this bug).
If we re-use said macro RING_REQUEST_CONS_OVERFLOW and pass in the rp
instead (sring->req_prod) of rc, the this macro can do the check:
req_prod - rsp_prov_pvt >= 32
Which is,
X + 31415 - X >= 32 , or 31415 >= 32
which is true, so we can error out and break out of the function.
Unfortunatly the difference between rsp_prov_pvt and req_prod can be
at 32 (which would error out in the macro). This condition exists when
the backend is lagging behind with the responses and still has not finished
responding to all of them (so make_response has not been called), and
the rsp_prov_pvt + 32 == req_cons. This ends up with us not being able
to use said macro.
Hence introducing a new macro called RING_REQUEST_PROD_OVERFLOW which does
a simple check of:
req_prod - rsp_prod_pvt > RING_SIZE
And with the X values from above:
X + 31415 - X > 32
Returns true. Also not that if the ring is full (which is where
the RING_REQUEST_CONS_OVERFLOW triggered), we would not hit the
same condition:
X + 32 - X > 32
Which is false.
Lets use that macro.
Note that in v5 of this patchset the macro was different - we used an
earlier version.
Cc: stable@vger.kernel.org
[v1: Move the check outside the loop]
[v2: Add a pr_warn as suggested by David]
[v3: Use RING_REQUEST_CONS_OVERFLOW as suggested by Jan]
[v4: Move wake_up after kthread_stop as suggested by Jan]
[v5: Use RING_REQUEST_PROD_OVERFLOW instead]
[v6: Use RING_REQUEST_PROD_OVERFLOW - Jan's version]
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>
gadsa
2013-01-24 04:54:32 +07:00
|
|
|
if (ret == -EACCES)
|
2015-11-14 10:12:15 +07:00
|
|
|
wait_event_interruptible(ring->shutdown_wq,
|
xen/blkback: Check for insane amounts of request on the ring (v6).
Check that the ring does not have an insane amount of requests
(more than there could fit on the ring).
If we detect this case we will stop processing the requests
and wait until the XenBus disconnects the ring.
The existing check RING_REQUEST_CONS_OVERFLOW which checks for how
many responses we have created in the past (rsp_prod_pvt) vs
requests consumed (req_cons) and whether said difference is greater or
equal to the size of the ring, does not catch this case.
Wha the condition does check if there is a need to process more
as we still have a backlog of responses to finish. Note that both
of those values (rsp_prod_pvt and req_cons) are not exposed on the
shared ring.
To understand this problem a mini crash course in ring protocol
response/request updates is in place.
There are four entries: req_prod and rsp_prod; req_event and rsp_event
to track the ring entries. We are only concerned about the first two -
which set the tone of this bug.
The req_prod is a value incremented by frontend for each request put
on the ring. Conversely the rsp_prod is a value incremented by the backend
for each response put on the ring (rsp_prod gets set by rsp_prod_pvt when
pushing the responses on the ring). Both values can
wrap and are modulo the size of the ring (in block case that is 32).
Please see RING_GET_REQUEST and RING_GET_RESPONSE for the more details.
The culprit here is that if the difference between the
req_prod and req_cons is greater than the ring size we have a problem.
Fortunately for us, the '__do_block_io_op' loop:
rc = blk_rings->common.req_cons;
rp = blk_rings->common.sring->req_prod;
while (rc != rp) {
..
blk_rings->common.req_cons = ++rc; /* before make_response() */
}
will loop up to the point when rc == rp. The macros inside of the
loop (RING_GET_REQUEST) is smart and is indexing based on the modulo
of the ring size. If the frontend has provided a bogus req_prod value
we will loop until the 'rc == rp' - which means we could be processing
already processed requests (or responses) often.
The reason the RING_REQUEST_CONS_OVERFLOW is not helping here is
b/c it only tracks how many responses we have internally produced
and whether we would should process more. The astute reader will
notice that the macro RING_REQUEST_CONS_OVERFLOW provides two
arguments - more on this later.
For example, if we were to enter this function with these values:
blk_rings->common.sring->req_prod = X+31415 (X is the value from
the last time __do_block_io_op was called).
blk_rings->common.req_cons = X
blk_rings->common.rsp_prod_pvt = X
The RING_REQUEST_CONS_OVERFLOW(&blk_rings->common, blk_rings->common.req_cons)
is doing:
req_cons - rsp_prod_pvt >= 32
Which is,
X - X >= 32 or 0 >= 32
And that is false, so we continue on looping (this bug).
If we re-use said macro RING_REQUEST_CONS_OVERFLOW and pass in the rp
instead (sring->req_prod) of rc, the this macro can do the check:
req_prod - rsp_prov_pvt >= 32
Which is,
X + 31415 - X >= 32 , or 31415 >= 32
which is true, so we can error out and break out of the function.
Unfortunatly the difference between rsp_prov_pvt and req_prod can be
at 32 (which would error out in the macro). This condition exists when
the backend is lagging behind with the responses and still has not finished
responding to all of them (so make_response has not been called), and
the rsp_prov_pvt + 32 == req_cons. This ends up with us not being able
to use said macro.
Hence introducing a new macro called RING_REQUEST_PROD_OVERFLOW which does
a simple check of:
req_prod - rsp_prod_pvt > RING_SIZE
And with the X values from above:
X + 31415 - X > 32
Returns true. Also not that if the ring is full (which is where
the RING_REQUEST_CONS_OVERFLOW triggered), we would not hit the
same condition:
X + 32 - X > 32
Which is false.
Lets use that macro.
Note that in v5 of this patchset the macro was different - we used an
earlier version.
Cc: stable@vger.kernel.org
[v1: Move the check outside the loop]
[v2: Add a pr_warn as suggested by David]
[v3: Use RING_REQUEST_CONS_OVERFLOW as suggested by Jan]
[v4: Move wake_up after kthread_stop as suggested by Jan]
[v5: Use RING_REQUEST_PROD_OVERFLOW instead]
[v6: Use RING_REQUEST_PROD_OVERFLOW - Jan's version]
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>
gadsa
2013-01-24 04:54:32 +07:00
|
|
|
kthread_should_stop());
|
2011-04-15 05:25:47 +07:00
|
|
|
|
2013-04-18 01:18:57 +07:00
|
|
|
purge_gnt_list:
|
|
|
|
if (blkif->vbd.feature_gnt_persistent &&
|
2015-11-14 10:12:19 +07:00
|
|
|
time_after(jiffies, ring->next_lru)) {
|
|
|
|
purge_persistent_gnt(ring);
|
|
|
|
ring->next_lru = jiffies + msecs_to_jiffies(LRU_INTERVAL);
|
2013-04-18 01:18:57 +07:00
|
|
|
}
|
|
|
|
|
2013-04-18 01:18:56 +07:00
|
|
|
/* Shrink if we have more than xen_blkif_max_buffer_pages */
|
2015-11-14 10:12:19 +07:00
|
|
|
shrink_free_pagepool(ring, xen_blkif_max_buffer_pages);
|
2013-04-18 01:18:56 +07:00
|
|
|
|
2015-12-09 06:44:02 +07:00
|
|
|
if (log_stats && time_after(jiffies, ring->st_print))
|
2015-11-14 10:12:19 +07:00
|
|
|
print_stats(ring);
|
2011-04-15 05:25:47 +07:00
|
|
|
}
|
|
|
|
|
2014-02-04 17:26:13 +07:00
|
|
|
/* Drain pending purge work */
|
2015-11-14 10:12:19 +07:00
|
|
|
flush_work(&ring->persistent_purge_work);
|
2013-04-18 01:18:56 +07:00
|
|
|
|
2014-02-04 17:26:13 +07:00
|
|
|
if (log_stats)
|
2015-11-14 10:12:19 +07:00
|
|
|
print_stats(ring);
|
2014-02-04 17:26:13 +07:00
|
|
|
|
2015-11-14 10:12:15 +07:00
|
|
|
ring->xenblkd = NULL;
|
2014-02-04 17:26:13 +07:00
|
|
|
xen_blkif_put(blkif);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Remove persistent grants and empty the pool of free pages
|
|
|
|
*/
|
2015-11-14 10:12:15 +07:00
|
|
|
void xen_blkbk_free_caches(struct xen_blkif_ring *ring)
|
2014-02-04 17:26:13 +07:00
|
|
|
{
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
/* Free all persistent grant pages */
|
2015-11-14 10:12:19 +07:00
|
|
|
if (!RB_EMPTY_ROOT(&ring->persistent_gnts))
|
|
|
|
free_persistent_gnts(ring, &ring->persistent_gnts,
|
|
|
|
ring->persistent_gnt_c);
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
BUG_ON(!RB_EMPTY_ROOT(&ring->persistent_gnts));
|
|
|
|
ring->persistent_gnt_c = 0;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
|
2014-02-04 17:26:12 +07:00
|
|
|
/* Since we are shutting down remove all pages from the buffer */
|
2015-11-14 10:12:19 +07:00
|
|
|
shrink_free_pagepool(ring, 0 /* All */);
|
2011-04-15 05:25:47 +07:00
|
|
|
}
|
|
|
|
|
2015-01-05 23:49:22 +07:00
|
|
|
static unsigned int xen_blkbk_unmap_prepare(
|
2015-11-14 10:12:15 +07:00
|
|
|
struct xen_blkif_ring *ring,
|
2015-01-05 23:49:22 +07:00
|
|
|
struct grant_page **pages,
|
|
|
|
unsigned int num,
|
|
|
|
struct gnttab_unmap_grant_ref *unmap_ops,
|
|
|
|
struct page **unmap_pages)
|
2011-04-15 21:58:05 +07:00
|
|
|
{
|
|
|
|
unsigned int i, invcount = 0;
|
|
|
|
|
2013-04-18 01:19:00 +07:00
|
|
|
for (i = 0; i < num; i++) {
|
2013-05-02 15:21:17 +07:00
|
|
|
if (pages[i]->persistent_gnt != NULL) {
|
2015-11-14 10:12:19 +07:00
|
|
|
put_persistent_gnt(ring, pages[i]->persistent_gnt);
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
continue;
|
2013-04-18 01:18:57 +07:00
|
|
|
}
|
2013-05-02 15:21:17 +07:00
|
|
|
if (pages[i]->handle == BLKBACK_INVALID_HANDLE)
|
2011-04-15 21:58:05 +07:00
|
|
|
continue;
|
2013-05-02 15:21:17 +07:00
|
|
|
unmap_pages[invcount] = pages[i]->page;
|
2015-01-05 23:49:22 +07:00
|
|
|
gnttab_set_unmap_op(&unmap_ops[invcount], vaddr(pages[i]->page),
|
2013-05-02 15:21:17 +07:00
|
|
|
GNTMAP_host_map, pages[i]->handle);
|
|
|
|
pages[i]->handle = BLKBACK_INVALID_HANDLE;
|
2015-01-05 23:49:22 +07:00
|
|
|
invcount++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return invcount;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void xen_blkbk_unmap_and_respond_callback(int result, struct gntab_unmap_queue_data *data)
|
|
|
|
{
|
2015-11-14 10:12:15 +07:00
|
|
|
struct pending_req *pending_req = (struct pending_req *)(data->data);
|
|
|
|
struct xen_blkif_ring *ring = pending_req->ring;
|
|
|
|
struct xen_blkif *blkif = ring->blkif;
|
2015-01-05 23:49:22 +07:00
|
|
|
|
|
|
|
/* BUG_ON used to reproduce existing behaviour,
|
|
|
|
but is this the best way to deal with this? */
|
|
|
|
BUG_ON(result);
|
|
|
|
|
2015-11-14 10:12:19 +07:00
|
|
|
put_free_pages(ring, data->pages, data->count);
|
2015-11-14 10:12:15 +07:00
|
|
|
make_response(ring, pending_req->id,
|
2015-01-05 23:49:22 +07:00
|
|
|
pending_req->operation, pending_req->status);
|
2015-11-14 10:12:15 +07:00
|
|
|
free_req(ring, pending_req);
|
2015-01-05 23:49:22 +07:00
|
|
|
/*
|
|
|
|
* Make sure the request is freed before releasing blkif,
|
|
|
|
* or there could be a race between free_req and the
|
|
|
|
* cleanup done in xen_blkif_free during shutdown.
|
|
|
|
*
|
|
|
|
* NB: The fact that we might try to wake up pending_free_wq
|
|
|
|
* before drain_complete (in case there's a drain going on)
|
|
|
|
* it's not a problem with our current implementation
|
|
|
|
* because we can assure there's no thread waiting on
|
|
|
|
* pending_free_wq if there's a drain going on, but it has
|
|
|
|
* to be taken into account if the current model is changed.
|
|
|
|
*/
|
2015-11-14 10:12:15 +07:00
|
|
|
if (atomic_dec_and_test(&ring->inflight) && atomic_read(&blkif->drain)) {
|
2015-01-05 23:49:22 +07:00
|
|
|
complete(&blkif->drain_complete);
|
|
|
|
}
|
|
|
|
xen_blkif_put(blkif);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void xen_blkbk_unmap_and_respond(struct pending_req *req)
|
|
|
|
{
|
|
|
|
struct gntab_unmap_queue_data* work = &req->gnttab_unmap_data;
|
2015-11-14 10:12:15 +07:00
|
|
|
struct xen_blkif_ring *ring = req->ring;
|
2015-01-05 23:49:22 +07:00
|
|
|
struct grant_page **pages = req->segments;
|
|
|
|
unsigned int invcount;
|
|
|
|
|
2015-11-14 10:12:15 +07:00
|
|
|
invcount = xen_blkbk_unmap_prepare(ring, pages, req->nr_segs,
|
2015-01-05 23:49:22 +07:00
|
|
|
req->unmap, req->unmap_pages);
|
|
|
|
|
|
|
|
work->data = req;
|
|
|
|
work->done = xen_blkbk_unmap_and_respond_callback;
|
|
|
|
work->unmap_ops = req->unmap;
|
|
|
|
work->kunmap_ops = NULL;
|
|
|
|
work->pages = req->unmap_pages;
|
|
|
|
work->count = invcount;
|
|
|
|
|
|
|
|
gnttab_unmap_refs_async(&req->gnttab_unmap_data);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Unmap the grant references.
|
|
|
|
*
|
|
|
|
* This could accumulate ops up to the batch size to reduce the number
|
|
|
|
* of hypercalls, but since this is only used in error paths there's
|
|
|
|
* no real need.
|
|
|
|
*/
|
2015-11-14 10:12:15 +07:00
|
|
|
static void xen_blkbk_unmap(struct xen_blkif_ring *ring,
|
2015-01-05 23:49:22 +07:00
|
|
|
struct grant_page *pages[],
|
|
|
|
int num)
|
|
|
|
{
|
|
|
|
struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
|
|
|
|
struct page *unmap_pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
|
|
|
|
unsigned int invcount = 0;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
while (num) {
|
|
|
|
unsigned int batch = min(num, BLKIF_MAX_SEGMENTS_PER_REQUEST);
|
2015-11-14 10:12:15 +07:00
|
|
|
|
|
|
|
invcount = xen_blkbk_unmap_prepare(ring, pages, batch,
|
2015-01-05 23:49:22 +07:00
|
|
|
unmap, unmap_pages);
|
|
|
|
if (invcount) {
|
|
|
|
ret = gnttab_unmap_refs(unmap, NULL, unmap_pages, invcount);
|
2013-04-18 01:19:00 +07:00
|
|
|
BUG_ON(ret);
|
2015-11-14 10:12:19 +07:00
|
|
|
put_free_pages(ring, unmap_pages, invcount);
|
2013-04-18 01:19:00 +07:00
|
|
|
}
|
2015-01-05 23:49:22 +07:00
|
|
|
pages += batch;
|
|
|
|
num -= batch;
|
2011-04-15 21:58:05 +07:00
|
|
|
}
|
|
|
|
}
|
2011-05-12 02:57:09 +07:00
|
|
|
|
2015-11-14 10:12:15 +07:00
|
|
|
static int xen_blkbk_map(struct xen_blkif_ring *ring,
|
2013-05-02 15:21:17 +07:00
|
|
|
struct grant_page *pages[],
|
2013-04-18 01:19:00 +07:00
|
|
|
int num, bool ro)
|
2011-04-15 22:35:13 +07:00
|
|
|
{
|
|
|
|
struct gnttab_map_grant_ref map[BLKIF_MAX_SEGMENTS_PER_REQUEST];
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
struct page *pages_to_gnt[BLKIF_MAX_SEGMENTS_PER_REQUEST];
|
|
|
|
struct persistent_gnt *persistent_gnt = NULL;
|
|
|
|
phys_addr_t addr = 0;
|
2013-04-18 01:18:56 +07:00
|
|
|
int i, seg_idx, new_map_idx;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
int segs_to_map = 0;
|
2011-04-15 22:35:13 +07:00
|
|
|
int ret = 0;
|
2013-04-18 01:19:00 +07:00
|
|
|
int last_map = 0, map_until = 0;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
int use_persistent_gnts;
|
2015-11-14 10:12:15 +07:00
|
|
|
struct xen_blkif *blkif = ring->blkif;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
|
|
|
|
use_persistent_gnts = (blkif->vbd.feature_gnt_persistent);
|
|
|
|
|
2011-05-12 02:57:09 +07:00
|
|
|
/*
|
|
|
|
* Fill out preq.nr_sects with proper amount of sectors, and setup
|
2011-04-15 22:35:13 +07:00
|
|
|
* assign map[..] with the PFN of the page in our domain with the
|
|
|
|
* corresponding grant reference for each page.
|
|
|
|
*/
|
2013-04-18 01:19:00 +07:00
|
|
|
again:
|
|
|
|
for (i = map_until; i < num; i++) {
|
2011-04-15 22:35:13 +07:00
|
|
|
uint32_t flags;
|
|
|
|
|
2015-11-14 10:12:15 +07:00
|
|
|
if (use_persistent_gnts) {
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
persistent_gnt = get_persistent_gnt(
|
2015-11-14 10:12:19 +07:00
|
|
|
ring,
|
2013-05-02 15:21:17 +07:00
|
|
|
pages[i]->gref);
|
2015-11-14 10:12:15 +07:00
|
|
|
}
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
|
|
|
|
if (persistent_gnt) {
|
|
|
|
/*
|
|
|
|
* We are using persistent grants and
|
|
|
|
* the grant is already mapped
|
|
|
|
*/
|
2013-05-02 15:21:17 +07:00
|
|
|
pages[i]->page = persistent_gnt->page;
|
|
|
|
pages[i]->persistent_gnt = persistent_gnt;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
} else {
|
2015-11-14 10:12:19 +07:00
|
|
|
if (get_free_page(ring, &pages[i]->page))
|
2013-04-18 01:18:56 +07:00
|
|
|
goto out_of_memory;
|
2013-05-02 15:21:17 +07:00
|
|
|
addr = vaddr(pages[i]->page);
|
|
|
|
pages_to_gnt[segs_to_map] = pages[i]->page;
|
|
|
|
pages[i]->persistent_gnt = NULL;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
flags = GNTMAP_host_map;
|
2013-04-18 01:19:00 +07:00
|
|
|
if (!use_persistent_gnts && ro)
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
flags |= GNTMAP_readonly;
|
|
|
|
gnttab_set_map_op(&map[segs_to_map++], addr,
|
2013-05-02 15:21:17 +07:00
|
|
|
flags, pages[i]->gref,
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
blkif->domid);
|
|
|
|
}
|
2013-04-18 01:19:00 +07:00
|
|
|
map_until = i + 1;
|
|
|
|
if (segs_to_map == BLKIF_MAX_SEGMENTS_PER_REQUEST)
|
|
|
|
break;
|
2011-04-15 22:35:13 +07:00
|
|
|
}
|
|
|
|
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
if (segs_to_map) {
|
|
|
|
ret = gnttab_map_refs(map, NULL, pages_to_gnt, segs_to_map);
|
|
|
|
BUG_ON(ret);
|
|
|
|
}
|
2011-04-15 22:35:13 +07:00
|
|
|
|
2011-05-12 02:57:09 +07:00
|
|
|
/*
|
|
|
|
* Now swizzle the MFN in our domain with the MFN from the other domain
|
2011-04-15 22:35:13 +07:00
|
|
|
* so that when we access vaddr(pending_req,i) it has the contents of
|
|
|
|
* the page from the other domain.
|
|
|
|
*/
|
2013-04-18 01:19:00 +07:00
|
|
|
for (seg_idx = last_map, new_map_idx = 0; seg_idx < map_until; seg_idx++) {
|
2013-05-02 15:21:17 +07:00
|
|
|
if (!pages[seg_idx]->persistent_gnt) {
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
/* This is a newly mapped grant */
|
2013-04-18 01:18:56 +07:00
|
|
|
BUG_ON(new_map_idx >= segs_to_map);
|
|
|
|
if (unlikely(map[new_map_idx].status != 0)) {
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_debug("invalid buffer -- could not remap it\n");
|
2015-11-14 10:12:19 +07:00
|
|
|
put_free_pages(ring, &pages[seg_idx]->page, 1);
|
2013-05-02 15:21:17 +07:00
|
|
|
pages[seg_idx]->handle = BLKBACK_INVALID_HANDLE;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
ret |= 1;
|
2013-04-18 01:19:00 +07:00
|
|
|
goto next;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
}
|
2013-05-02 15:21:17 +07:00
|
|
|
pages[seg_idx]->handle = map[new_map_idx].handle;
|
2013-04-18 01:18:56 +07:00
|
|
|
} else {
|
2013-04-18 01:19:00 +07:00
|
|
|
continue;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
}
|
2013-04-18 01:18:56 +07:00
|
|
|
if (use_persistent_gnts &&
|
2015-11-14 10:12:19 +07:00
|
|
|
ring->persistent_gnt_c < xen_blkif_max_pgrants) {
|
2013-04-18 01:18:56 +07:00
|
|
|
/*
|
|
|
|
* We are using persistent grants, the grant is
|
2013-04-18 01:18:57 +07:00
|
|
|
* not mapped but we might have room for it.
|
2013-04-18 01:18:56 +07:00
|
|
|
*/
|
|
|
|
persistent_gnt = kmalloc(sizeof(struct persistent_gnt),
|
|
|
|
GFP_KERNEL);
|
|
|
|
if (!persistent_gnt) {
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
/*
|
2013-04-18 01:18:56 +07:00
|
|
|
* If we don't have enough memory to
|
|
|
|
* allocate the persistent_gnt struct
|
|
|
|
* map this grant non-persistenly
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
*/
|
2013-04-18 01:19:00 +07:00
|
|
|
goto next;
|
xen/blkback: Persistent grant maps for xen blk drivers
This patch implements persistent grants for the xen-blk{front,back}
mechanism. The effect of this change is to reduce the number of unmap
operations performed, since they cause a (costly) TLB shootdown. This
allows the I/O performance to scale better when a large number of VMs
are performing I/O.
Previously, the blkfront driver was supplied a bvec[] from the request
queue. This was granted to dom0; dom0 performed the I/O and wrote
directly into the grant-mapped memory and unmapped it; blkfront then
removed foreign access for that grant. The cost of unmapping scales
badly with the number of CPUs in Dom0. An experiment showed that when
Dom0 has 24 VCPUs, and guests are performing parallel I/O to a
ramdisk, the IPIs from performing unmap's is a bottleneck at 5 guests
(at which point 650,000 IOPS are being performed in total). If more
than 5 guests are used, the performance declines. By 10 guests, only
400,000 IOPS are being performed.
This patch improves performance by only unmapping when the connection
between blkfront and back is broken.
On startup blkfront notifies blkback that it is using persistent
grants, and blkback will do the same. If blkback is not capable of
persistent mapping, blkfront will still use the same grants, since it
is compatible with the previous protocol, and simplifies the code
complexity in blkfront.
To perform a read, in persistent mode, blkfront uses a separate pool
of pages that it maps to dom0. When a request comes in, blkfront
transmutes the request so that blkback will write into one of these
free pages. Blkback keeps note of which grefs it has already
mapped. When a new ring request comes to blkback, it looks to see if
it has already mapped that page. If so, it will not map it again. If
the page hasn't been previously mapped, it is mapped now, and a record
is kept of this mapping. Blkback proceeds as usual. When blkfront is
notified that blkback has completed a request, it memcpy's from the
shared memory, into the bvec supplied. A record that the {gref, page}
tuple is mapped, and not inflight is kept.
Writes are similar, except that the memcpy is peformed from the
supplied bvecs, into the shared pages, before the request is put onto
the ring.
Blkback stores a mapping of grefs=>{page mapped to by gref} in
a red-black tree. As the grefs are not known apriori, and provide no
guarantees on their ordering, we have to perform a search
through this tree to find the page, for every gref we receive. This
operation takes O(log n) time in the worst case. In blkfront grants
are stored using a single linked list.
The maximum number of grants that blkback will persistenly map is
currently set to RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, to
prevent a malicios guest from attempting a DoS, by supplying fresh
grefs, causing the Dom0 kernel to map excessively. If a guest
is using persistent grants and exceeds the maximum number of grants to
map persistenly the newly passed grefs will be mapped and unmaped.
Using this approach, we can have requests that mix persistent and
non-persistent grants, and we need to handle them correctly.
This allows us to set the maximum number of persistent grants to a
lower value than RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST, although
setting it will lead to unpredictable performance.
In writing this patch, the question arrises as to if the additional
cost of performing memcpys in the guest (to/from the pool of granted
pages) outweigh the gains of not performing TLB shootdowns. The answer
to that question is `no'. There appears to be very little, if any
additional cost to the guest of using persistent grants. There is
perhaps a small saving, from the reduced number of hypercalls
performed in granting, and ending foreign access.
Signed-off-by: Oliver Chick <oliver.chick@citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Fixed up the misuse of bool as int]
2012-10-24 23:58:45 +07:00
|
|
|
}
|
2013-04-18 01:18:56 +07:00
|
|
|
persistent_gnt->gnt = map[new_map_idx].ref;
|
|
|
|
persistent_gnt->handle = map[new_map_idx].handle;
|
2013-05-02 15:21:17 +07:00
|
|
|
persistent_gnt->page = pages[seg_idx]->page;
|
2015-11-14 10:12:19 +07:00
|
|
|
if (add_persistent_gnt(ring,
|
2013-04-18 01:18:56 +07:00
|
|
|
persistent_gnt)) {
|
|
|
|
kfree(persistent_gnt);
|
|
|
|
persistent_gnt = NULL;
|
2013-04-18 01:19:00 +07:00
|
|
|
goto next;
|
2013-04-18 01:18:56 +07:00
|
|
|
}
|
2013-05-02 15:21:17 +07:00
|
|
|
pages[seg_idx]->persistent_gnt = persistent_gnt;
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_debug("grant %u added to the tree of persistent grants, using %u/%u\n",
|
2015-11-14 10:12:19 +07:00
|
|
|
persistent_gnt->gnt, ring->persistent_gnt_c,
|
2013-04-18 01:18:57 +07:00
|
|
|
xen_blkif_max_pgrants);
|
2013-04-18 01:18:56 +07:00
|
|
|
goto next;
|
|
|
|
}
|
|
|
|
if (use_persistent_gnts && !blkif->vbd.overflow_max_grants) {
|
|
|
|
blkif->vbd.overflow_max_grants = 1;
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_debug("domain %u, device %#x is using maximum number of persistent grants\n",
|
2013-04-18 01:18:56 +07:00
|
|
|
blkif->domid, blkif->vbd.handle);
|
2011-04-15 22:35:13 +07:00
|
|
|
}
|
2013-04-18 01:18:56 +07:00
|
|
|
/*
|
|
|
|
* We could not map this grant persistently, so use it as
|
|
|
|
* a non-persistent grant.
|
|
|
|
*/
|
|
|
|
next:
|
2013-04-18 01:19:00 +07:00
|
|
|
new_map_idx++;
|
2011-04-15 22:35:13 +07:00
|
|
|
}
|
2013-04-18 01:19:00 +07:00
|
|
|
segs_to_map = 0;
|
|
|
|
last_map = map_until;
|
|
|
|
if (map_until != num)
|
|
|
|
goto again;
|
|
|
|
|
2011-04-15 22:35:13 +07:00
|
|
|
return ret;
|
2013-04-18 01:18:56 +07:00
|
|
|
|
|
|
|
out_of_memory:
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_alert("%s: out of memory\n", __func__);
|
2015-11-14 10:12:19 +07:00
|
|
|
put_free_pages(ring, pages_to_gnt, segs_to_map);
|
2013-04-18 01:18:56 +07:00
|
|
|
return -ENOMEM;
|
2011-04-15 22:35:13 +07:00
|
|
|
}
|
|
|
|
|
2013-05-02 15:21:17 +07:00
|
|
|
static int xen_blkbk_map_seg(struct pending_req *pending_req)
|
2013-04-18 01:19:00 +07:00
|
|
|
{
|
2013-04-18 21:06:54 +07:00
|
|
|
int rc;
|
2013-04-18 01:19:00 +07:00
|
|
|
|
2015-11-14 10:12:15 +07:00
|
|
|
rc = xen_blkbk_map(pending_req->ring, pending_req->segments,
|
2015-06-17 21:28:08 +07:00
|
|
|
pending_req->nr_segs,
|
2013-04-18 01:19:00 +07:00
|
|
|
(pending_req->operation != BLKIF_OP_READ));
|
|
|
|
|
2013-04-18 21:06:54 +07:00
|
|
|
return rc;
|
|
|
|
}
|
2013-04-18 01:19:00 +07:00
|
|
|
|
2013-04-18 21:06:54 +07:00
|
|
|
static int xen_blkbk_parse_indirect(struct blkif_request *req,
|
|
|
|
struct pending_req *pending_req,
|
|
|
|
struct seg_buf seg[],
|
|
|
|
struct phys_req *preq)
|
|
|
|
{
|
2013-05-02 15:21:17 +07:00
|
|
|
struct grant_page **pages = pending_req->indirect_pages;
|
2015-11-14 10:12:15 +07:00
|
|
|
struct xen_blkif_ring *ring = pending_req->ring;
|
2013-04-18 21:06:54 +07:00
|
|
|
int indirect_grefs, rc, n, nseg, i;
|
2014-02-04 17:26:15 +07:00
|
|
|
struct blkif_request_segment *segments = NULL;
|
2013-04-18 21:06:54 +07:00
|
|
|
|
2015-06-17 21:28:08 +07:00
|
|
|
nseg = pending_req->nr_segs;
|
2013-04-18 21:06:54 +07:00
|
|
|
indirect_grefs = INDIRECT_PAGES(nseg);
|
|
|
|
BUG_ON(indirect_grefs > BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST);
|
|
|
|
|
2013-05-02 15:21:17 +07:00
|
|
|
for (i = 0; i < indirect_grefs; i++)
|
|
|
|
pages[i]->gref = req->u.indirect.indirect_grefs[i];
|
|
|
|
|
2015-11-14 10:12:15 +07:00
|
|
|
rc = xen_blkbk_map(ring, pages, indirect_grefs, true);
|
2013-04-18 21:06:54 +07:00
|
|
|
if (rc)
|
|
|
|
goto unmap;
|
|
|
|
|
|
|
|
for (n = 0, i = 0; n < nseg; n++) {
|
2015-11-03 23:40:43 +07:00
|
|
|
uint8_t first_sect, last_sect;
|
|
|
|
|
2013-04-18 21:06:54 +07:00
|
|
|
if ((n % SEGS_PER_INDIRECT_FRAME) == 0) {
|
|
|
|
/* Map indirect segments */
|
|
|
|
if (segments)
|
|
|
|
kunmap_atomic(segments);
|
2013-05-02 15:21:17 +07:00
|
|
|
segments = kmap_atomic(pages[n/SEGS_PER_INDIRECT_FRAME]->page);
|
2013-04-18 21:06:54 +07:00
|
|
|
}
|
|
|
|
i = n % SEGS_PER_INDIRECT_FRAME;
|
2015-11-03 23:40:43 +07:00
|
|
|
|
2013-05-02 15:21:17 +07:00
|
|
|
pending_req->segments[n]->gref = segments[i].gref;
|
2015-11-03 23:40:43 +07:00
|
|
|
|
|
|
|
first_sect = READ_ONCE(segments[i].first_sect);
|
|
|
|
last_sect = READ_ONCE(segments[i].last_sect);
|
|
|
|
if (last_sect >= (XEN_PAGE_SIZE >> 9) || last_sect < first_sect) {
|
2013-04-18 21:06:54 +07:00
|
|
|
rc = -EINVAL;
|
|
|
|
goto unmap;
|
|
|
|
}
|
2015-11-03 23:40:43 +07:00
|
|
|
|
|
|
|
seg[n].nsec = last_sect - first_sect + 1;
|
|
|
|
seg[n].offset = first_sect << 9;
|
2013-04-18 21:06:54 +07:00
|
|
|
preq->nr_sects += seg[n].nsec;
|
|
|
|
}
|
|
|
|
|
|
|
|
unmap:
|
|
|
|
if (segments)
|
|
|
|
kunmap_atomic(segments);
|
2015-11-14 10:12:15 +07:00
|
|
|
xen_blkbk_unmap(ring, pages, indirect_grefs);
|
2013-04-18 21:06:54 +07:00
|
|
|
return rc;
|
2013-04-18 01:19:00 +07:00
|
|
|
}
|
|
|
|
|
2015-11-14 10:12:15 +07:00
|
|
|
static int dispatch_discard_io(struct xen_blkif_ring *ring,
|
2011-10-13 04:26:47 +07:00
|
|
|
struct blkif_request *req)
|
2011-09-01 17:39:10 +07:00
|
|
|
{
|
|
|
|
int err = 0;
|
|
|
|
int status = BLKIF_RSP_OKAY;
|
2015-11-14 10:12:15 +07:00
|
|
|
struct xen_blkif *blkif = ring->blkif;
|
2011-09-01 17:39:10 +07:00
|
|
|
struct block_device *bdev = blkif->vbd.bdev;
|
2012-03-14 05:43:23 +07:00
|
|
|
unsigned long secure;
|
2013-01-16 23:33:52 +07:00
|
|
|
struct phys_req preq;
|
2011-09-01 17:39:10 +07:00
|
|
|
|
2013-09-05 18:00:14 +07:00
|
|
|
xen_blkif_get(blkif);
|
|
|
|
|
2013-01-16 23:33:52 +07:00
|
|
|
preq.sector_number = req->u.discard.sector_number;
|
|
|
|
preq.nr_sects = req->u.discard.nr_sectors;
|
|
|
|
|
2016-06-06 02:32:09 +07:00
|
|
|
err = xen_vbd_translate(&preq, blkif, REQ_OP_WRITE);
|
2013-01-16 23:33:52 +07:00
|
|
|
if (err) {
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_warn("access denied: DISCARD [%llu->%llu] on dev=%04x\n",
|
2013-01-16 23:33:52 +07:00
|
|
|
preq.sector_number,
|
|
|
|
preq.sector_number + preq.nr_sects, blkif->vbd.pdevice);
|
|
|
|
goto fail_response;
|
|
|
|
}
|
2015-12-09 06:44:02 +07:00
|
|
|
ring->st_ds_req++;
|
2011-10-13 04:26:47 +07:00
|
|
|
|
2012-03-14 05:43:23 +07:00
|
|
|
secure = (blkif->vbd.discard_secure &&
|
|
|
|
(req->u.discard.flag & BLKIF_DISCARD_SECURE)) ?
|
|
|
|
BLKDEV_DISCARD_SECURE : 0;
|
|
|
|
|
|
|
|
err = blkdev_issue_discard(bdev, req->u.discard.sector_number,
|
|
|
|
req->u.discard.nr_sectors,
|
|
|
|
GFP_KERNEL, secure);
|
2013-01-16 23:33:52 +07:00
|
|
|
fail_response:
|
2011-09-01 17:39:10 +07:00
|
|
|
if (err == -EOPNOTSUPP) {
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_debug("discard op failed, not supported\n");
|
2011-09-01 17:39:10 +07:00
|
|
|
status = BLKIF_RSP_EOPNOTSUPP;
|
|
|
|
} else if (err)
|
|
|
|
status = BLKIF_RSP_ERROR;
|
|
|
|
|
2015-11-14 10:12:15 +07:00
|
|
|
make_response(ring, req->u.discard.id, req->operation, status);
|
2011-10-13 04:26:47 +07:00
|
|
|
xen_blkif_put(blkif);
|
|
|
|
return err;
|
2011-09-01 17:39:10 +07:00
|
|
|
}
|
|
|
|
|
2015-11-14 10:12:15 +07:00
|
|
|
static int dispatch_other_io(struct xen_blkif_ring *ring,
|
2013-03-08 00:32:01 +07:00
|
|
|
struct blkif_request *req,
|
|
|
|
struct pending_req *pending_req)
|
|
|
|
{
|
2015-11-14 10:12:15 +07:00
|
|
|
free_req(ring, pending_req);
|
|
|
|
make_response(ring, req->u.other.id, req->operation,
|
2013-03-08 00:32:01 +07:00
|
|
|
BLKIF_RSP_EOPNOTSUPP);
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
2015-11-14 10:12:15 +07:00
|
|
|
static void xen_blk_drain_io(struct xen_blkif_ring *ring)
|
2011-10-10 11:42:22 +07:00
|
|
|
{
|
2015-11-14 10:12:15 +07:00
|
|
|
struct xen_blkif *blkif = ring->blkif;
|
|
|
|
|
2011-10-10 11:42:22 +07:00
|
|
|
atomic_set(&blkif->drain, 1);
|
|
|
|
do {
|
2015-11-14 10:12:15 +07:00
|
|
|
if (atomic_read(&ring->inflight) == 0)
|
2011-10-18 01:27:48 +07:00
|
|
|
break;
|
2011-10-10 11:42:22 +07:00
|
|
|
wait_for_completion_interruptible_timeout(
|
|
|
|
&blkif->drain_complete, HZ);
|
|
|
|
|
|
|
|
if (!atomic_read(&blkif->drain))
|
|
|
|
break;
|
|
|
|
} while (!kthread_should_stop());
|
|
|
|
atomic_set(&blkif->drain, 0);
|
|
|
|
}
|
|
|
|
|
2011-04-15 04:05:23 +07:00
|
|
|
/*
|
|
|
|
* Completion callback on the bio's. Called as bh->b_end_io()
|
2011-04-15 05:25:47 +07:00
|
|
|
*/
|
|
|
|
|
2011-04-15 04:42:07 +07:00
|
|
|
static void __end_block_io_op(struct pending_req *pending_req, int error)
|
2011-04-15 05:25:47 +07:00
|
|
|
{
|
|
|
|
/* An error fails the entire request. */
|
2011-05-05 04:07:27 +07:00
|
|
|
if ((pending_req->operation == BLKIF_OP_FLUSH_DISKCACHE) &&
|
2011-04-15 05:25:47 +07:00
|
|
|
(error == -EOPNOTSUPP)) {
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_debug("flush diskcache op failed, not supported\n");
|
2015-11-14 10:12:15 +07:00
|
|
|
xen_blkbk_flush_diskcache(XBT_NIL, pending_req->ring->blkif->be, 0);
|
2011-04-15 05:25:47 +07:00
|
|
|
pending_req->status = BLKIF_RSP_EOPNOTSUPP;
|
2011-10-10 11:42:22 +07:00
|
|
|
} else if ((pending_req->operation == BLKIF_OP_WRITE_BARRIER) &&
|
|
|
|
(error == -EOPNOTSUPP)) {
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_debug("write barrier op failed, not supported\n");
|
2015-11-14 10:12:15 +07:00
|
|
|
xen_blkbk_barrier(XBT_NIL, pending_req->ring->blkif->be, 0);
|
2011-10-10 11:42:22 +07:00
|
|
|
pending_req->status = BLKIF_RSP_EOPNOTSUPP;
|
2011-04-15 05:25:47 +07:00
|
|
|
} else if (error) {
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_debug("Buffer not up-to-date at end of operation,"
|
2011-05-13 03:42:31 +07:00
|
|
|
" error=%d\n", error);
|
2011-04-15 05:25:47 +07:00
|
|
|
pending_req->status = BLKIF_RSP_ERROR;
|
|
|
|
}
|
|
|
|
|
2011-05-12 02:57:09 +07:00
|
|
|
/*
|
|
|
|
* If all of the bio's have completed it is time to unmap
|
2011-04-15 04:05:23 +07:00
|
|
|
* the grant references associated with 'request' and provide
|
2011-04-15 04:42:07 +07:00
|
|
|
* the proper response on the ring.
|
|
|
|
*/
|
2015-01-05 23:49:22 +07:00
|
|
|
if (atomic_dec_and_test(&pending_req->pendcnt))
|
|
|
|
xen_blkbk_unmap_and_respond(pending_req);
|
2011-04-15 05:25:47 +07:00
|
|
|
}
|
|
|
|
|
2011-04-15 04:05:23 +07:00
|
|
|
/*
|
|
|
|
* bio callback.
|
|
|
|
*/
|
2015-07-20 20:29:37 +07:00
|
|
|
static void end_block_io_op(struct bio *bio)
|
2011-04-15 05:25:47 +07:00
|
|
|
{
|
2015-07-20 20:29:37 +07:00
|
|
|
__end_block_io_op(bio->bi_private, bio->bi_error);
|
2011-04-15 05:25:47 +07:00
|
|
|
bio_put(bio);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-04-15 04:05:23 +07:00
|
|
|
/*
|
|
|
|
* Function to copy the from the ring buffer the 'struct blkif_request'
|
|
|
|
* (which has the sectors we want, number of them, grant references, etc),
|
|
|
|
* and transmute it to the block API to hand it over to the proper block disk.
|
2011-04-15 05:25:47 +07:00
|
|
|
*/
|
2011-05-29 03:21:10 +07:00
|
|
|
static int
|
2015-11-14 10:12:15 +07:00
|
|
|
__do_block_io_op(struct xen_blkif_ring *ring)
|
2011-04-15 05:25:47 +07:00
|
|
|
{
|
2015-11-14 10:12:15 +07:00
|
|
|
union blkif_back_rings *blk_rings = &ring->blk_rings;
|
2009-02-10 03:05:51 +07:00
|
|
|
struct blkif_request req;
|
2011-04-15 04:42:07 +07:00
|
|
|
struct pending_req *pending_req;
|
2011-04-15 05:25:47 +07:00
|
|
|
RING_IDX rc, rp;
|
|
|
|
int more_to_do = 0;
|
|
|
|
|
|
|
|
rc = blk_rings->common.req_cons;
|
|
|
|
rp = blk_rings->common.sring->req_prod;
|
|
|
|
rmb(); /* Ensure we see queued requests up to 'rp'. */
|
|
|
|
|
xen/blkback: Check for insane amounts of request on the ring (v6).
Check that the ring does not have an insane amount of requests
(more than there could fit on the ring).
If we detect this case we will stop processing the requests
and wait until the XenBus disconnects the ring.
The existing check RING_REQUEST_CONS_OVERFLOW which checks for how
many responses we have created in the past (rsp_prod_pvt) vs
requests consumed (req_cons) and whether said difference is greater or
equal to the size of the ring, does not catch this case.
Wha the condition does check if there is a need to process more
as we still have a backlog of responses to finish. Note that both
of those values (rsp_prod_pvt and req_cons) are not exposed on the
shared ring.
To understand this problem a mini crash course in ring protocol
response/request updates is in place.
There are four entries: req_prod and rsp_prod; req_event and rsp_event
to track the ring entries. We are only concerned about the first two -
which set the tone of this bug.
The req_prod is a value incremented by frontend for each request put
on the ring. Conversely the rsp_prod is a value incremented by the backend
for each response put on the ring (rsp_prod gets set by rsp_prod_pvt when
pushing the responses on the ring). Both values can
wrap and are modulo the size of the ring (in block case that is 32).
Please see RING_GET_REQUEST and RING_GET_RESPONSE for the more details.
The culprit here is that if the difference between the
req_prod and req_cons is greater than the ring size we have a problem.
Fortunately for us, the '__do_block_io_op' loop:
rc = blk_rings->common.req_cons;
rp = blk_rings->common.sring->req_prod;
while (rc != rp) {
..
blk_rings->common.req_cons = ++rc; /* before make_response() */
}
will loop up to the point when rc == rp. The macros inside of the
loop (RING_GET_REQUEST) is smart and is indexing based on the modulo
of the ring size. If the frontend has provided a bogus req_prod value
we will loop until the 'rc == rp' - which means we could be processing
already processed requests (or responses) often.
The reason the RING_REQUEST_CONS_OVERFLOW is not helping here is
b/c it only tracks how many responses we have internally produced
and whether we would should process more. The astute reader will
notice that the macro RING_REQUEST_CONS_OVERFLOW provides two
arguments - more on this later.
For example, if we were to enter this function with these values:
blk_rings->common.sring->req_prod = X+31415 (X is the value from
the last time __do_block_io_op was called).
blk_rings->common.req_cons = X
blk_rings->common.rsp_prod_pvt = X
The RING_REQUEST_CONS_OVERFLOW(&blk_rings->common, blk_rings->common.req_cons)
is doing:
req_cons - rsp_prod_pvt >= 32
Which is,
X - X >= 32 or 0 >= 32
And that is false, so we continue on looping (this bug).
If we re-use said macro RING_REQUEST_CONS_OVERFLOW and pass in the rp
instead (sring->req_prod) of rc, the this macro can do the check:
req_prod - rsp_prov_pvt >= 32
Which is,
X + 31415 - X >= 32 , or 31415 >= 32
which is true, so we can error out and break out of the function.
Unfortunatly the difference between rsp_prov_pvt and req_prod can be
at 32 (which would error out in the macro). This condition exists when
the backend is lagging behind with the responses and still has not finished
responding to all of them (so make_response has not been called), and
the rsp_prov_pvt + 32 == req_cons. This ends up with us not being able
to use said macro.
Hence introducing a new macro called RING_REQUEST_PROD_OVERFLOW which does
a simple check of:
req_prod - rsp_prod_pvt > RING_SIZE
And with the X values from above:
X + 31415 - X > 32
Returns true. Also not that if the ring is full (which is where
the RING_REQUEST_CONS_OVERFLOW triggered), we would not hit the
same condition:
X + 32 - X > 32
Which is false.
Lets use that macro.
Note that in v5 of this patchset the macro was different - we used an
earlier version.
Cc: stable@vger.kernel.org
[v1: Move the check outside the loop]
[v2: Add a pr_warn as suggested by David]
[v3: Use RING_REQUEST_CONS_OVERFLOW as suggested by Jan]
[v4: Move wake_up after kthread_stop as suggested by Jan]
[v5: Use RING_REQUEST_PROD_OVERFLOW instead]
[v6: Use RING_REQUEST_PROD_OVERFLOW - Jan's version]
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>
gadsa
2013-01-24 04:54:32 +07:00
|
|
|
if (RING_REQUEST_PROD_OVERFLOW(&blk_rings->common, rp)) {
|
|
|
|
rc = blk_rings->common.rsp_prod_pvt;
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_warn("Frontend provided bogus ring requests (%d - %d = %d). Halting ring processing on dev=%04x\n",
|
2015-11-14 10:12:15 +07:00
|
|
|
rp, rc, rp - rc, ring->blkif->vbd.pdevice);
|
xen/blkback: Check for insane amounts of request on the ring (v6).
Check that the ring does not have an insane amount of requests
(more than there could fit on the ring).
If we detect this case we will stop processing the requests
and wait until the XenBus disconnects the ring.
The existing check RING_REQUEST_CONS_OVERFLOW which checks for how
many responses we have created in the past (rsp_prod_pvt) vs
requests consumed (req_cons) and whether said difference is greater or
equal to the size of the ring, does not catch this case.
Wha the condition does check if there is a need to process more
as we still have a backlog of responses to finish. Note that both
of those values (rsp_prod_pvt and req_cons) are not exposed on the
shared ring.
To understand this problem a mini crash course in ring protocol
response/request updates is in place.
There are four entries: req_prod and rsp_prod; req_event and rsp_event
to track the ring entries. We are only concerned about the first two -
which set the tone of this bug.
The req_prod is a value incremented by frontend for each request put
on the ring. Conversely the rsp_prod is a value incremented by the backend
for each response put on the ring (rsp_prod gets set by rsp_prod_pvt when
pushing the responses on the ring). Both values can
wrap and are modulo the size of the ring (in block case that is 32).
Please see RING_GET_REQUEST and RING_GET_RESPONSE for the more details.
The culprit here is that if the difference between the
req_prod and req_cons is greater than the ring size we have a problem.
Fortunately for us, the '__do_block_io_op' loop:
rc = blk_rings->common.req_cons;
rp = blk_rings->common.sring->req_prod;
while (rc != rp) {
..
blk_rings->common.req_cons = ++rc; /* before make_response() */
}
will loop up to the point when rc == rp. The macros inside of the
loop (RING_GET_REQUEST) is smart and is indexing based on the modulo
of the ring size. If the frontend has provided a bogus req_prod value
we will loop until the 'rc == rp' - which means we could be processing
already processed requests (or responses) often.
The reason the RING_REQUEST_CONS_OVERFLOW is not helping here is
b/c it only tracks how many responses we have internally produced
and whether we would should process more. The astute reader will
notice that the macro RING_REQUEST_CONS_OVERFLOW provides two
arguments - more on this later.
For example, if we were to enter this function with these values:
blk_rings->common.sring->req_prod = X+31415 (X is the value from
the last time __do_block_io_op was called).
blk_rings->common.req_cons = X
blk_rings->common.rsp_prod_pvt = X
The RING_REQUEST_CONS_OVERFLOW(&blk_rings->common, blk_rings->common.req_cons)
is doing:
req_cons - rsp_prod_pvt >= 32
Which is,
X - X >= 32 or 0 >= 32
And that is false, so we continue on looping (this bug).
If we re-use said macro RING_REQUEST_CONS_OVERFLOW and pass in the rp
instead (sring->req_prod) of rc, the this macro can do the check:
req_prod - rsp_prov_pvt >= 32
Which is,
X + 31415 - X >= 32 , or 31415 >= 32
which is true, so we can error out and break out of the function.
Unfortunatly the difference between rsp_prov_pvt and req_prod can be
at 32 (which would error out in the macro). This condition exists when
the backend is lagging behind with the responses and still has not finished
responding to all of them (so make_response has not been called), and
the rsp_prov_pvt + 32 == req_cons. This ends up with us not being able
to use said macro.
Hence introducing a new macro called RING_REQUEST_PROD_OVERFLOW which does
a simple check of:
req_prod - rsp_prod_pvt > RING_SIZE
And with the X values from above:
X + 31415 - X > 32
Returns true. Also not that if the ring is full (which is where
the RING_REQUEST_CONS_OVERFLOW triggered), we would not hit the
same condition:
X + 32 - X > 32
Which is false.
Lets use that macro.
Note that in v5 of this patchset the macro was different - we used an
earlier version.
Cc: stable@vger.kernel.org
[v1: Move the check outside the loop]
[v2: Add a pr_warn as suggested by David]
[v3: Use RING_REQUEST_CONS_OVERFLOW as suggested by Jan]
[v4: Move wake_up after kthread_stop as suggested by Jan]
[v5: Use RING_REQUEST_PROD_OVERFLOW instead]
[v6: Use RING_REQUEST_PROD_OVERFLOW - Jan's version]
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>
gadsa
2013-01-24 04:54:32 +07:00
|
|
|
return -EACCES;
|
|
|
|
}
|
2011-04-15 05:25:47 +07:00
|
|
|
while (rc != rp) {
|
|
|
|
|
|
|
|
if (RING_REQUEST_CONS_OVERFLOW(&blk_rings->common, rc))
|
|
|
|
break;
|
|
|
|
|
2009-03-06 15:29:15 +07:00
|
|
|
if (kthread_should_stop()) {
|
2011-04-15 05:25:47 +07:00
|
|
|
more_to_do = 1;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2015-11-14 10:12:15 +07:00
|
|
|
pending_req = alloc_req(ring);
|
2009-03-06 15:29:15 +07:00
|
|
|
if (NULL == pending_req) {
|
2015-12-09 06:44:02 +07:00
|
|
|
ring->st_oo_req++;
|
2011-04-15 05:25:47 +07:00
|
|
|
more_to_do = 1;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2015-11-14 10:12:15 +07:00
|
|
|
switch (ring->blkif->blk_protocol) {
|
2011-04-15 05:25:47 +07:00
|
|
|
case BLKIF_PROTOCOL_NATIVE:
|
|
|
|
memcpy(&req, RING_GET_REQUEST(&blk_rings->native, rc), sizeof(req));
|
|
|
|
break;
|
|
|
|
case BLKIF_PROTOCOL_X86_32:
|
|
|
|
blkif_get_x86_32_req(&req, RING_GET_REQUEST(&blk_rings->x86_32, rc));
|
|
|
|
break;
|
|
|
|
case BLKIF_PROTOCOL_X86_64:
|
|
|
|
blkif_get_x86_64_req(&req, RING_GET_REQUEST(&blk_rings->x86_64, rc));
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
BUG();
|
|
|
|
}
|
|
|
|
blk_rings->common.req_cons = ++rc; /* before make_response() */
|
|
|
|
|
|
|
|
/* Apply all sanity checks to /private copy/ of request. */
|
|
|
|
barrier();
|
2013-03-08 00:32:01 +07:00
|
|
|
|
|
|
|
switch (req.operation) {
|
|
|
|
case BLKIF_OP_READ:
|
|
|
|
case BLKIF_OP_WRITE:
|
|
|
|
case BLKIF_OP_WRITE_BARRIER:
|
|
|
|
case BLKIF_OP_FLUSH_DISKCACHE:
|
2013-04-18 21:06:54 +07:00
|
|
|
case BLKIF_OP_INDIRECT:
|
2015-11-14 10:12:15 +07:00
|
|
|
if (dispatch_rw_block_io(ring, &req, pending_req))
|
2013-03-08 00:32:01 +07:00
|
|
|
goto done;
|
|
|
|
break;
|
|
|
|
case BLKIF_OP_DISCARD:
|
2015-11-14 10:12:15 +07:00
|
|
|
free_req(ring, pending_req);
|
|
|
|
if (dispatch_discard_io(ring, &req))
|
2013-03-08 00:32:01 +07:00
|
|
|
goto done;
|
2011-04-15 05:25:47 +07:00
|
|
|
break;
|
2013-03-08 00:32:01 +07:00
|
|
|
default:
|
2015-11-14 10:12:15 +07:00
|
|
|
if (dispatch_other_io(ring, &req, pending_req))
|
2013-03-08 00:32:01 +07:00
|
|
|
goto done;
|
|
|
|
break;
|
|
|
|
}
|
2011-04-15 05:25:47 +07:00
|
|
|
|
|
|
|
/* Yield point for this unbounded loop. */
|
|
|
|
cond_resched();
|
|
|
|
}
|
2013-03-08 00:32:01 +07:00
|
|
|
done:
|
2011-04-15 05:25:47 +07:00
|
|
|
return more_to_do;
|
|
|
|
}
|
|
|
|
|
2011-05-29 03:21:10 +07:00
|
|
|
static int
|
2015-11-14 10:12:15 +07:00
|
|
|
do_block_io_op(struct xen_blkif_ring *ring)
|
2011-05-29 03:21:10 +07:00
|
|
|
{
|
2015-11-14 10:12:15 +07:00
|
|
|
union blkif_back_rings *blk_rings = &ring->blk_rings;
|
2011-05-29 03:21:10 +07:00
|
|
|
int more_to_do;
|
|
|
|
|
|
|
|
do {
|
2015-11-14 10:12:15 +07:00
|
|
|
more_to_do = __do_block_io_op(ring);
|
2011-05-29 03:21:10 +07:00
|
|
|
if (more_to_do)
|
|
|
|
break;
|
|
|
|
|
|
|
|
RING_FINAL_CHECK_FOR_REQUESTS(&blk_rings->common, more_to_do);
|
|
|
|
} while (more_to_do);
|
|
|
|
|
|
|
|
return more_to_do;
|
|
|
|
}
|
2011-04-15 04:05:23 +07:00
|
|
|
/*
|
2011-05-12 02:57:09 +07:00
|
|
|
* Transmutation of the 'struct blkif_request' to a proper 'struct bio'
|
|
|
|
* and call the 'submit_bio' to pass it to the underlying storage.
|
2011-04-15 04:05:23 +07:00
|
|
|
*/
|
2015-11-14 10:12:15 +07:00
|
|
|
static int dispatch_rw_block_io(struct xen_blkif_ring *ring,
|
2011-05-13 03:47:48 +07:00
|
|
|
struct blkif_request *req,
|
|
|
|
struct pending_req *pending_req)
|
2011-04-15 05:25:47 +07:00
|
|
|
{
|
|
|
|
struct phys_req preq;
|
2013-04-18 21:06:54 +07:00
|
|
|
struct seg_buf *seg = pending_req->seg;
|
2011-04-15 05:25:47 +07:00
|
|
|
unsigned int nseg;
|
|
|
|
struct bio *bio = NULL;
|
2013-04-18 21:06:54 +07:00
|
|
|
struct bio **biolist = pending_req->biolist;
|
2011-04-15 22:35:13 +07:00
|
|
|
int i, nbio = 0;
|
2011-04-15 05:25:47 +07:00
|
|
|
int operation;
|
2016-06-06 02:32:09 +07:00
|
|
|
int operation_flags = 0;
|
2011-04-27 23:40:11 +07:00
|
|
|
struct blk_plug plug;
|
2011-10-10 11:42:22 +07:00
|
|
|
bool drain = false;
|
2013-05-02 15:21:17 +07:00
|
|
|
struct grant_page **pages = pending_req->segments;
|
2013-04-18 21:06:54 +07:00
|
|
|
unsigned short req_operation;
|
|
|
|
|
|
|
|
req_operation = req->operation == BLKIF_OP_INDIRECT ?
|
|
|
|
req->u.indirect.indirect_op : req->operation;
|
2015-05-05 22:25:56 +07:00
|
|
|
|
2013-04-18 21:06:54 +07:00
|
|
|
if ((req->operation == BLKIF_OP_INDIRECT) &&
|
|
|
|
(req_operation != BLKIF_OP_READ) &&
|
|
|
|
(req_operation != BLKIF_OP_WRITE)) {
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_debug("Invalid indirect operation (%u)\n", req_operation);
|
2013-04-18 21:06:54 +07:00
|
|
|
goto fail_response;
|
|
|
|
}
|
2011-04-15 05:25:47 +07:00
|
|
|
|
2013-04-18 21:06:54 +07:00
|
|
|
switch (req_operation) {
|
2011-04-15 05:25:47 +07:00
|
|
|
case BLKIF_OP_READ:
|
2015-12-09 06:44:02 +07:00
|
|
|
ring->st_rd_req++;
|
2016-06-06 02:32:09 +07:00
|
|
|
operation = REQ_OP_READ;
|
2011-04-15 05:25:47 +07:00
|
|
|
break;
|
|
|
|
case BLKIF_OP_WRITE:
|
2015-12-09 06:44:02 +07:00
|
|
|
ring->st_wr_req++;
|
2016-06-06 02:32:09 +07:00
|
|
|
operation = REQ_OP_WRITE;
|
2016-11-01 20:40:10 +07:00
|
|
|
operation_flags = REQ_SYNC | REQ_IDLE;
|
2011-04-15 05:25:47 +07:00
|
|
|
break;
|
2011-10-10 11:42:22 +07:00
|
|
|
case BLKIF_OP_WRITE_BARRIER:
|
|
|
|
drain = true;
|
2011-05-05 04:07:27 +07:00
|
|
|
case BLKIF_OP_FLUSH_DISKCACHE:
|
2015-12-09 06:44:02 +07:00
|
|
|
ring->st_f_req++;
|
2016-06-06 02:32:09 +07:00
|
|
|
operation = REQ_OP_WRITE;
|
2016-11-01 20:40:10 +07:00
|
|
|
operation_flags = REQ_PREFLUSH;
|
2011-04-15 05:25:47 +07:00
|
|
|
break;
|
|
|
|
default:
|
|
|
|
operation = 0; /* make gcc happy */
|
2011-05-06 00:37:23 +07:00
|
|
|
goto fail_response;
|
|
|
|
break;
|
2011-04-15 05:25:47 +07:00
|
|
|
}
|
|
|
|
|
2011-10-13 04:26:47 +07:00
|
|
|
/* Check that the number of segments is sane. */
|
2013-04-18 21:06:54 +07:00
|
|
|
nseg = req->operation == BLKIF_OP_INDIRECT ?
|
|
|
|
req->u.indirect.nr_segments : req->u.rw.nr_segments;
|
2011-10-12 23:12:36 +07:00
|
|
|
|
2016-11-01 20:40:10 +07:00
|
|
|
if (unlikely(nseg == 0 && operation_flags != REQ_PREFLUSH) ||
|
2013-04-18 21:06:54 +07:00
|
|
|
unlikely((req->operation != BLKIF_OP_INDIRECT) &&
|
|
|
|
(nseg > BLKIF_MAX_SEGMENTS_PER_REQUEST)) ||
|
|
|
|
unlikely((req->operation == BLKIF_OP_INDIRECT) &&
|
|
|
|
(nseg > MAX_INDIRECT_SEGMENTS))) {
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_debug("Bad number of segments in request (%d)\n", nseg);
|
2011-04-15 22:35:13 +07:00
|
|
|
/* Haven't submitted any bio's yet. */
|
2011-04-15 05:25:47 +07:00
|
|
|
goto fail_response;
|
|
|
|
}
|
|
|
|
|
|
|
|
preq.nr_sects = 0;
|
|
|
|
|
2015-11-14 10:12:15 +07:00
|
|
|
pending_req->ring = ring;
|
2011-10-12 23:12:36 +07:00
|
|
|
pending_req->id = req->u.rw.id;
|
2013-04-18 21:06:54 +07:00
|
|
|
pending_req->operation = req_operation;
|
2011-04-15 05:25:47 +07:00
|
|
|
pending_req->status = BLKIF_RSP_OKAY;
|
2015-06-17 21:28:08 +07:00
|
|
|
pending_req->nr_segs = nseg;
|
2011-04-18 22:34:55 +07:00
|
|
|
|
2013-04-18 21:06:54 +07:00
|
|
|
if (req->operation != BLKIF_OP_INDIRECT) {
|
|
|
|
preq.dev = req->u.rw.handle;
|
|
|
|
preq.sector_number = req->u.rw.sector_number;
|
|
|
|
for (i = 0; i < nseg; i++) {
|
2013-05-02 15:21:17 +07:00
|
|
|
pages[i]->gref = req->u.rw.seg[i].gref;
|
2013-04-18 21:06:54 +07:00
|
|
|
seg[i].nsec = req->u.rw.seg[i].last_sect -
|
|
|
|
req->u.rw.seg[i].first_sect + 1;
|
|
|
|
seg[i].offset = (req->u.rw.seg[i].first_sect << 9);
|
2015-05-05 22:25:56 +07:00
|
|
|
if ((req->u.rw.seg[i].last_sect >= (XEN_PAGE_SIZE >> 9)) ||
|
2013-04-18 21:06:54 +07:00
|
|
|
(req->u.rw.seg[i].last_sect <
|
|
|
|
req->u.rw.seg[i].first_sect))
|
|
|
|
goto fail_response;
|
|
|
|
preq.nr_sects += seg[i].nsec;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
preq.dev = req->u.indirect.handle;
|
|
|
|
preq.sector_number = req->u.indirect.sector_number;
|
|
|
|
if (xen_blkbk_parse_indirect(req, pending_req, seg, &preq))
|
2011-04-15 05:25:47 +07:00
|
|
|
goto fail_response;
|
|
|
|
}
|
|
|
|
|
2015-11-14 10:12:15 +07:00
|
|
|
if (xen_vbd_translate(&preq, ring->blkif, operation) != 0) {
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_debug("access denied: %s of [%llu,%llu] on dev=%04x\n",
|
2016-06-06 02:32:09 +07:00
|
|
|
operation == REQ_OP_READ ? "read" : "write",
|
2011-05-13 03:42:31 +07:00
|
|
|
preq.sector_number,
|
2013-02-28 09:34:23 +07:00
|
|
|
preq.sector_number + preq.nr_sects,
|
2015-11-14 10:12:15 +07:00
|
|
|
ring->blkif->vbd.pdevice);
|
2011-04-15 22:35:13 +07:00
|
|
|
goto fail_response;
|
2011-04-15 05:25:47 +07:00
|
|
|
}
|
2011-05-12 02:57:09 +07:00
|
|
|
|
|
|
|
/*
|
2011-05-13 03:53:56 +07:00
|
|
|
* This check _MUST_ be done after xen_vbd_translate as the preq.bdev
|
2011-05-12 02:57:09 +07:00
|
|
|
* is set there.
|
|
|
|
*/
|
2011-04-18 22:34:55 +07:00
|
|
|
for (i = 0; i < nseg; i++) {
|
|
|
|
if (((int)preq.sector_number|(int)seg[i].nsec) &
|
|
|
|
((bdev_logical_block_size(preq.bdev) >> 9) - 1)) {
|
2015-04-01 22:04:22 +07:00
|
|
|
pr_debug("Misaligned I/O request from domain %d\n",
|
2015-11-14 10:12:15 +07:00
|
|
|
ring->blkif->domid);
|
2011-04-18 22:34:55 +07:00
|
|
|
goto fail_response;
|
|
|
|
}
|
|
|
|
}
|
2011-05-12 02:57:09 +07:00
|
|
|
|
2011-10-10 11:42:22 +07:00
|
|
|
/* Wait on all outstanding I/O's and once that has been completed
|
2016-11-01 20:40:10 +07:00
|
|
|
* issue the flush.
|
2011-10-10 11:42:22 +07:00
|
|
|
*/
|
|
|
|
if (drain)
|
2015-11-14 10:12:15 +07:00
|
|
|
xen_blk_drain_io(pending_req->ring);
|
2011-10-10 11:42:22 +07:00
|
|
|
|
2011-05-12 02:57:09 +07:00
|
|
|
/*
|
|
|
|
* If we have failed at this point, we need to undo the M2P override,
|
2011-04-15 04:42:07 +07:00
|
|
|
* set gnttab_set_unmap_op on all of the grant references and perform
|
|
|
|
* the hypercall to unmap the grants - that is all done in
|
2011-04-15 22:50:34 +07:00
|
|
|
* xen_blkbk_unmap.
|
2011-04-15 04:42:07 +07:00
|
|
|
*/
|
2013-05-02 15:21:17 +07:00
|
|
|
if (xen_blkbk_map_seg(pending_req))
|
2011-04-15 05:25:47 +07:00
|
|
|
goto fail_flush;
|
|
|
|
|
2011-09-01 17:39:10 +07:00
|
|
|
/*
|
|
|
|
* This corresponding xen_blkif_put is done in __end_block_io_op, or
|
|
|
|
* below (in "!bio") if we are handling a BLKIF_OP_DISCARD.
|
|
|
|
*/
|
2015-11-14 10:12:15 +07:00
|
|
|
xen_blkif_get(ring->blkif);
|
|
|
|
atomic_inc(&ring->inflight);
|
2011-04-15 05:25:47 +07:00
|
|
|
|
|
|
|
for (i = 0; i < nseg; i++) {
|
|
|
|
while ((bio == NULL) ||
|
|
|
|
(bio_add_page(bio,
|
2013-05-02 15:21:17 +07:00
|
|
|
pages[i]->page,
|
2011-04-15 05:25:47 +07:00
|
|
|
seg[i].nsec << 9,
|
2013-03-18 23:49:32 +07:00
|
|
|
seg[i].offset) == 0)) {
|
2011-04-15 04:42:07 +07:00
|
|
|
|
2013-06-22 14:59:17 +07:00
|
|
|
int nr_iovecs = min_t(int, (nseg-i), BIO_MAX_PAGES);
|
|
|
|
bio = bio_alloc(GFP_KERNEL, nr_iovecs);
|
2011-04-15 05:25:47 +07:00
|
|
|
if (unlikely(bio == NULL))
|
|
|
|
goto fail_put_bio;
|
|
|
|
|
2011-05-13 03:19:23 +07:00
|
|
|
biolist[nbio++] = bio;
|
2011-04-15 05:25:47 +07:00
|
|
|
bio->bi_bdev = preq.bdev;
|
|
|
|
bio->bi_private = pending_req;
|
|
|
|
bio->bi_end_io = end_block_io_op;
|
2013-10-12 05:44:27 +07:00
|
|
|
bio->bi_iter.bi_sector = preq.sector_number;
|
2016-06-06 02:32:09 +07:00
|
|
|
bio_set_op_attrs(bio, operation, operation_flags);
|
2011-04-15 05:25:47 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
preq.sector_number += seg[i].nsec;
|
|
|
|
}
|
|
|
|
|
2011-09-01 17:39:10 +07:00
|
|
|
/* This will be hit if the operation was a flush or discard. */
|
2011-04-15 05:25:47 +07:00
|
|
|
if (!bio) {
|
2016-11-01 20:40:10 +07:00
|
|
|
BUG_ON(operation_flags != REQ_PREFLUSH);
|
2011-05-13 03:23:06 +07:00
|
|
|
|
2011-10-13 04:26:47 +07:00
|
|
|
bio = bio_alloc(GFP_KERNEL, 0);
|
|
|
|
if (unlikely(bio == NULL))
|
|
|
|
goto fail_put_bio;
|
2011-04-15 05:25:47 +07:00
|
|
|
|
2011-10-13 04:26:47 +07:00
|
|
|
biolist[nbio++] = bio;
|
|
|
|
bio->bi_bdev = preq.bdev;
|
|
|
|
bio->bi_private = pending_req;
|
|
|
|
bio->bi_end_io = end_block_io_op;
|
2016-06-06 02:32:09 +07:00
|
|
|
bio_set_op_attrs(bio, operation, operation_flags);
|
2011-04-15 05:25:47 +07:00
|
|
|
}
|
|
|
|
|
2011-04-15 21:51:27 +07:00
|
|
|
atomic_set(&pending_req->pendcnt, nbio);
|
2011-04-27 23:40:11 +07:00
|
|
|
blk_start_plug(&plug);
|
|
|
|
|
2011-04-15 21:51:27 +07:00
|
|
|
for (i = 0; i < nbio; i++)
|
2016-06-06 02:31:41 +07:00
|
|
|
submit_bio(biolist[i]);
|
2011-04-15 21:51:27 +07:00
|
|
|
|
2011-04-27 23:40:11 +07:00
|
|
|
/* Let the I/Os go.. */
|
2011-05-06 00:42:10 +07:00
|
|
|
blk_finish_plug(&plug);
|
2011-04-27 23:40:11 +07:00
|
|
|
|
2016-06-06 02:32:09 +07:00
|
|
|
if (operation == REQ_OP_READ)
|
2015-12-09 06:44:02 +07:00
|
|
|
ring->st_rd_sect += preq.nr_sects;
|
2016-06-06 02:32:09 +07:00
|
|
|
else if (operation == REQ_OP_WRITE)
|
2015-12-09 06:44:02 +07:00
|
|
|
ring->st_wr_sect += preq.nr_sects;
|
2011-04-15 05:25:47 +07:00
|
|
|
|
2011-05-06 00:37:23 +07:00
|
|
|
return 0;
|
2011-04-15 05:25:47 +07:00
|
|
|
|
|
|
|
fail_flush:
|
2015-11-14 10:12:15 +07:00
|
|
|
xen_blkbk_unmap(ring, pending_req->segments,
|
2015-06-17 21:28:08 +07:00
|
|
|
pending_req->nr_segs);
|
2011-04-15 05:25:47 +07:00
|
|
|
fail_response:
|
2011-04-15 04:58:19 +07:00
|
|
|
/* Haven't submitted any bio's yet. */
|
2015-11-14 10:12:15 +07:00
|
|
|
make_response(ring, req->u.rw.id, req_operation, BLKIF_RSP_ERROR);
|
|
|
|
free_req(ring, pending_req);
|
2011-04-15 05:25:47 +07:00
|
|
|
msleep(1); /* back off a bit */
|
2011-05-06 00:37:23 +07:00
|
|
|
return -EIO;
|
2011-04-15 05:25:47 +07:00
|
|
|
|
|
|
|
fail_put_bio:
|
2011-05-13 03:19:23 +07:00
|
|
|
for (i = 0; i < nbio; i++)
|
2011-04-15 21:51:27 +07:00
|
|
|
bio_put(biolist[i]);
|
2013-03-11 16:39:55 +07:00
|
|
|
atomic_set(&pending_req->pendcnt, 1);
|
2011-04-15 05:25:47 +07:00
|
|
|
__end_block_io_op(pending_req, -EINVAL);
|
|
|
|
msleep(1); /* back off a bit */
|
2011-05-06 00:37:23 +07:00
|
|
|
return -EIO;
|
2011-04-15 05:25:47 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-04-15 04:05:23 +07:00
|
|
|
/*
|
|
|
|
* Put a response on the ring on how the operation fared.
|
2011-04-15 05:25:47 +07:00
|
|
|
*/
|
2015-11-14 10:12:15 +07:00
|
|
|
static void make_response(struct xen_blkif_ring *ring, u64 id,
|
2011-04-15 05:25:47 +07:00
|
|
|
unsigned short op, int st)
|
|
|
|
{
|
2009-02-10 03:05:51 +07:00
|
|
|
struct blkif_response resp;
|
2011-04-15 05:25:47 +07:00
|
|
|
unsigned long flags;
|
2015-11-14 10:12:15 +07:00
|
|
|
union blkif_back_rings *blk_rings;
|
2011-04-15 05:25:47 +07:00
|
|
|
int notify;
|
|
|
|
|
|
|
|
resp.id = id;
|
|
|
|
resp.operation = op;
|
|
|
|
resp.status = st;
|
|
|
|
|
2015-11-14 10:12:15 +07:00
|
|
|
spin_lock_irqsave(&ring->blk_ring_lock, flags);
|
|
|
|
blk_rings = &ring->blk_rings;
|
2011-04-15 05:25:47 +07:00
|
|
|
/* Place on the response ring for the relevant domain. */
|
2015-11-14 10:12:15 +07:00
|
|
|
switch (ring->blkif->blk_protocol) {
|
2011-04-15 05:25:47 +07:00
|
|
|
case BLKIF_PROTOCOL_NATIVE:
|
|
|
|
memcpy(RING_GET_RESPONSE(&blk_rings->native, blk_rings->native.rsp_prod_pvt),
|
|
|
|
&resp, sizeof(resp));
|
|
|
|
break;
|
|
|
|
case BLKIF_PROTOCOL_X86_32:
|
|
|
|
memcpy(RING_GET_RESPONSE(&blk_rings->x86_32, blk_rings->x86_32.rsp_prod_pvt),
|
|
|
|
&resp, sizeof(resp));
|
|
|
|
break;
|
|
|
|
case BLKIF_PROTOCOL_X86_64:
|
|
|
|
memcpy(RING_GET_RESPONSE(&blk_rings->x86_64, blk_rings->x86_64.rsp_prod_pvt),
|
|
|
|
&resp, sizeof(resp));
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
BUG();
|
|
|
|
}
|
|
|
|
blk_rings->common.rsp_prod_pvt++;
|
|
|
|
RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&blk_rings->common, notify);
|
2015-11-14 10:12:15 +07:00
|
|
|
spin_unlock_irqrestore(&ring->blk_ring_lock, flags);
|
2011-04-15 05:25:47 +07:00
|
|
|
if (notify)
|
2015-11-14 10:12:15 +07:00
|
|
|
notify_remote_via_irq(ring->irq);
|
2011-04-15 05:25:47 +07:00
|
|
|
}
|
|
|
|
|
2011-04-20 22:50:43 +07:00
|
|
|
static int __init xen_blkif_init(void)
|
2011-04-15 05:25:47 +07:00
|
|
|
{
|
2009-10-09 00:23:09 +07:00
|
|
|
int rc = 0;
|
2011-04-15 05:25:47 +07:00
|
|
|
|
2011-11-28 23:49:05 +07:00
|
|
|
if (!xen_domain())
|
2011-04-15 05:25:47 +07:00
|
|
|
return -ENODEV;
|
|
|
|
|
2015-10-13 23:50:11 +07:00
|
|
|
if (xen_blkif_max_ring_order > XENBUS_MAX_RING_GRANT_ORDER) {
|
2015-06-03 12:40:03 +07:00
|
|
|
pr_info("Invalid max_ring_order (%d), will use default max: %d.\n",
|
2015-10-13 23:50:11 +07:00
|
|
|
xen_blkif_max_ring_order, XENBUS_MAX_RING_GRANT_ORDER);
|
|
|
|
xen_blkif_max_ring_order = XENBUS_MAX_RING_GRANT_ORDER;
|
2015-06-03 12:40:03 +07:00
|
|
|
}
|
|
|
|
|
2015-11-14 10:12:17 +07:00
|
|
|
if (xenblk_max_queues == 0)
|
|
|
|
xenblk_max_queues = num_online_cpus();
|
|
|
|
|
2011-04-20 22:50:43 +07:00
|
|
|
rc = xen_blkif_interface_init();
|
2009-10-09 00:23:09 +07:00
|
|
|
if (rc)
|
|
|
|
goto failed_init;
|
2011-04-15 05:25:47 +07:00
|
|
|
|
2011-04-20 22:50:43 +07:00
|
|
|
rc = xen_blkif_xenbus_init();
|
2009-10-09 00:23:09 +07:00
|
|
|
if (rc)
|
|
|
|
goto failed_init;
|
2011-04-15 05:25:47 +07:00
|
|
|
|
2009-10-09 00:23:09 +07:00
|
|
|
failed_init:
|
|
|
|
return rc;
|
2011-04-15 05:25:47 +07:00
|
|
|
}
|
|
|
|
|
2011-04-20 22:50:43 +07:00
|
|
|
module_init(xen_blkif_init);
|
2011-04-15 05:25:47 +07:00
|
|
|
|
|
|
|
MODULE_LICENSE("Dual BSD/GPL");
|
2011-06-29 19:40:50 +07:00
|
|
|
MODULE_ALIAS("xen-backend:vbd");
|