mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-05 19:16:40 +07:00
36869cb93d
Pull block layer updates from Jens Axboe: "This is the main block pull request this series. Contrary to previous release, I've kept the core and driver changes in the same branch. We always ended up having dependencies between the two for obvious reasons, so makes more sense to keep them together. That said, I'll probably try and keep more topical branches going forward, especially for cycles that end up being as busy as this one. The major parts of this pull request is: - Improved support for O_DIRECT on block devices, with a small private implementation instead of using the pig that is fs/direct-io.c. From Christoph. - Request completion tracking in a scalable fashion. This is utilized by two components in this pull, the new hybrid polling and the writeback queue throttling code. - Improved support for polling with O_DIRECT, adding a hybrid mode that combines pure polling with an initial sleep. From me. - Support for automatic throttling of writeback queues on the block side. This uses feedback from the device completion latencies to scale the queue on the block side up or down. From me. - Support from SMR drives in the block layer and for SD. From Hannes and Shaun. - Multi-connection support for nbd. From Josef. - Cleanup of request and bio flags, so we have a clear split between which are bio (or rq) private, and which ones are shared. From Christoph. - A set of patches from Bart, that improve how we handle queue stopping and starting in blk-mq. - Support for WRITE_ZEROES from Chaitanya. - Lightnvm updates from Javier/Matias. - Supoort for FC for the nvme-over-fabrics code. From James Smart. - A bunch of fixes from a whole slew of people, too many to name here" * 'for-4.10/block' of git://git.kernel.dk/linux-block: (182 commits) blk-stat: fix a few cases of missing batch flushing blk-flush: run the queue when inserting blk-mq flush elevator: make the rqhash helpers exported blk-mq: abstract out blk_mq_dispatch_rq_list() helper blk-mq: add blk_mq_start_stopped_hw_queue() block: improve handling of the magic discard payload blk-wbt: don't throttle discard or write zeroes nbd: use dev_err_ratelimited in io path nbd: reset the setup task for NBD_CLEAR_SOCK nvme-fabrics: Add FC LLDD loopback driver to test FC-NVME nvme-fabrics: Add target support for FC transport nvme-fabrics: Add host support for FC transport nvme-fabrics: Add FC transport LLDD api definitions nvme-fabrics: Add FC transport FC-NVME definitions nvme-fabrics: Add FC transport error codes to nvme.h Add type 0x28 NVME type code to scsi fc headers nvme-fabrics: patch target code in prep for FC transport support nvme-fabrics: set sqe.command_id in core not transports parser: add u64 number parser nvme-rdma: align to generic ib_event logging helper ...
254 lines
5.9 KiB
C
254 lines
5.9 KiB
C
/*
|
|
* Functions related to mapping data to requests
|
|
*/
|
|
#include <linux/kernel.h>
|
|
#include <linux/module.h>
|
|
#include <linux/bio.h>
|
|
#include <linux/blkdev.h>
|
|
#include <linux/uio.h>
|
|
|
|
#include "blk.h"
|
|
|
|
/*
|
|
* Append a bio to a passthrough request. Only works can be merged into
|
|
* the request based on the driver constraints.
|
|
*/
|
|
int blk_rq_append_bio(struct request *rq, struct bio *bio)
|
|
{
|
|
if (!rq->bio) {
|
|
rq->cmd_flags &= REQ_OP_MASK;
|
|
rq->cmd_flags |= (bio->bi_opf & REQ_OP_MASK);
|
|
blk_rq_bio_prep(rq->q, rq, bio);
|
|
} else {
|
|
if (!ll_back_merge_fn(rq->q, rq, bio))
|
|
return -EINVAL;
|
|
|
|
rq->biotail->bi_next = bio;
|
|
rq->biotail = bio;
|
|
rq->__data_len += bio->bi_iter.bi_size;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(blk_rq_append_bio);
|
|
|
|
static int __blk_rq_unmap_user(struct bio *bio)
|
|
{
|
|
int ret = 0;
|
|
|
|
if (bio) {
|
|
if (bio_flagged(bio, BIO_USER_MAPPED))
|
|
bio_unmap_user(bio);
|
|
else
|
|
ret = bio_uncopy_user(bio);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int __blk_rq_map_user_iov(struct request *rq,
|
|
struct rq_map_data *map_data, struct iov_iter *iter,
|
|
gfp_t gfp_mask, bool copy)
|
|
{
|
|
struct request_queue *q = rq->q;
|
|
struct bio *bio, *orig_bio;
|
|
int ret;
|
|
|
|
if (copy)
|
|
bio = bio_copy_user_iov(q, map_data, iter, gfp_mask);
|
|
else
|
|
bio = bio_map_user_iov(q, iter, gfp_mask);
|
|
|
|
if (IS_ERR(bio))
|
|
return PTR_ERR(bio);
|
|
|
|
if (map_data && map_data->null_mapped)
|
|
bio_set_flag(bio, BIO_NULL_MAPPED);
|
|
|
|
iov_iter_advance(iter, bio->bi_iter.bi_size);
|
|
if (map_data)
|
|
map_data->offset += bio->bi_iter.bi_size;
|
|
|
|
orig_bio = bio;
|
|
blk_queue_bounce(q, &bio);
|
|
|
|
/*
|
|
* We link the bounce buffer in and could have to traverse it
|
|
* later so we have to get a ref to prevent it from being freed
|
|
*/
|
|
bio_get(bio);
|
|
|
|
ret = blk_rq_append_bio(rq, bio);
|
|
if (ret) {
|
|
bio_endio(bio);
|
|
__blk_rq_unmap_user(orig_bio);
|
|
bio_put(bio);
|
|
return ret;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* blk_rq_map_user_iov - map user data to a request, for REQ_TYPE_BLOCK_PC usage
|
|
* @q: request queue where request should be inserted
|
|
* @rq: request to map data to
|
|
* @map_data: pointer to the rq_map_data holding pages (if necessary)
|
|
* @iter: iovec iterator
|
|
* @gfp_mask: memory allocation flags
|
|
*
|
|
* Description:
|
|
* Data will be mapped directly for zero copy I/O, if possible. Otherwise
|
|
* a kernel bounce buffer is used.
|
|
*
|
|
* A matching blk_rq_unmap_user() must be issued at the end of I/O, while
|
|
* still in process context.
|
|
*
|
|
* Note: The mapped bio may need to be bounced through blk_queue_bounce()
|
|
* before being submitted to the device, as pages mapped may be out of
|
|
* reach. It's the callers responsibility to make sure this happens. The
|
|
* original bio must be passed back in to blk_rq_unmap_user() for proper
|
|
* unmapping.
|
|
*/
|
|
int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
|
|
struct rq_map_data *map_data,
|
|
const struct iov_iter *iter, gfp_t gfp_mask)
|
|
{
|
|
bool copy = false;
|
|
unsigned long align = q->dma_pad_mask | queue_dma_alignment(q);
|
|
struct bio *bio = NULL;
|
|
struct iov_iter i;
|
|
int ret;
|
|
|
|
if (!iter_is_iovec(iter))
|
|
goto fail;
|
|
|
|
if (map_data)
|
|
copy = true;
|
|
else if (iov_iter_alignment(iter) & align)
|
|
copy = true;
|
|
else if (queue_virt_boundary(q))
|
|
copy = queue_virt_boundary(q) & iov_iter_gap_alignment(iter);
|
|
|
|
i = *iter;
|
|
do {
|
|
ret =__blk_rq_map_user_iov(rq, map_data, &i, gfp_mask, copy);
|
|
if (ret)
|
|
goto unmap_rq;
|
|
if (!bio)
|
|
bio = rq->bio;
|
|
} while (iov_iter_count(&i));
|
|
|
|
if (!bio_flagged(bio, BIO_USER_MAPPED))
|
|
rq->rq_flags |= RQF_COPY_USER;
|
|
return 0;
|
|
|
|
unmap_rq:
|
|
__blk_rq_unmap_user(bio);
|
|
fail:
|
|
rq->bio = NULL;
|
|
return -EINVAL;
|
|
}
|
|
EXPORT_SYMBOL(blk_rq_map_user_iov);
|
|
|
|
int blk_rq_map_user(struct request_queue *q, struct request *rq,
|
|
struct rq_map_data *map_data, void __user *ubuf,
|
|
unsigned long len, gfp_t gfp_mask)
|
|
{
|
|
struct iovec iov;
|
|
struct iov_iter i;
|
|
int ret = import_single_range(rq_data_dir(rq), ubuf, len, &iov, &i);
|
|
|
|
if (unlikely(ret < 0))
|
|
return ret;
|
|
|
|
return blk_rq_map_user_iov(q, rq, map_data, &i, gfp_mask);
|
|
}
|
|
EXPORT_SYMBOL(blk_rq_map_user);
|
|
|
|
/**
|
|
* blk_rq_unmap_user - unmap a request with user data
|
|
* @bio: start of bio list
|
|
*
|
|
* Description:
|
|
* Unmap a rq previously mapped by blk_rq_map_user(). The caller must
|
|
* supply the original rq->bio from the blk_rq_map_user() return, since
|
|
* the I/O completion may have changed rq->bio.
|
|
*/
|
|
int blk_rq_unmap_user(struct bio *bio)
|
|
{
|
|
struct bio *mapped_bio;
|
|
int ret = 0, ret2;
|
|
|
|
while (bio) {
|
|
mapped_bio = bio;
|
|
if (unlikely(bio_flagged(bio, BIO_BOUNCED)))
|
|
mapped_bio = bio->bi_private;
|
|
|
|
ret2 = __blk_rq_unmap_user(mapped_bio);
|
|
if (ret2 && !ret)
|
|
ret = ret2;
|
|
|
|
mapped_bio = bio;
|
|
bio = bio->bi_next;
|
|
bio_put(mapped_bio);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL(blk_rq_unmap_user);
|
|
|
|
/**
|
|
* blk_rq_map_kern - map kernel data to a request, for REQ_TYPE_BLOCK_PC usage
|
|
* @q: request queue where request should be inserted
|
|
* @rq: request to fill
|
|
* @kbuf: the kernel buffer
|
|
* @len: length of user data
|
|
* @gfp_mask: memory allocation flags
|
|
*
|
|
* Description:
|
|
* Data will be mapped directly if possible. Otherwise a bounce
|
|
* buffer is used. Can be called multiple times to append multiple
|
|
* buffers.
|
|
*/
|
|
int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,
|
|
unsigned int len, gfp_t gfp_mask)
|
|
{
|
|
int reading = rq_data_dir(rq) == READ;
|
|
unsigned long addr = (unsigned long) kbuf;
|
|
int do_copy = 0;
|
|
struct bio *bio;
|
|
int ret;
|
|
|
|
if (len > (queue_max_hw_sectors(q) << 9))
|
|
return -EINVAL;
|
|
if (!len || !kbuf)
|
|
return -EINVAL;
|
|
|
|
do_copy = !blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf);
|
|
if (do_copy)
|
|
bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading);
|
|
else
|
|
bio = bio_map_kern(q, kbuf, len, gfp_mask);
|
|
|
|
if (IS_ERR(bio))
|
|
return PTR_ERR(bio);
|
|
|
|
if (!reading)
|
|
bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
|
|
|
|
if (do_copy)
|
|
rq->rq_flags |= RQF_COPY_USER;
|
|
|
|
ret = blk_rq_append_bio(rq, bio);
|
|
if (unlikely(ret)) {
|
|
/* request is too big */
|
|
bio_put(bio);
|
|
return ret;
|
|
}
|
|
|
|
blk_queue_bounce(q, &rq->bio);
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(blk_rq_map_kern);
|