2014-02-06 07:11:33 +07:00
|
|
|
#include <linux/export.h>
|
2016-11-01 20:40:13 +07:00
|
|
|
#include <linux/bvec.h>
|
2014-02-06 07:11:33 +07:00
|
|
|
#include <linux/uio.h>
|
|
|
|
#include <linux/pagemap.h>
|
2014-03-21 15:58:33 +07:00
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/vmalloc.h>
|
2016-09-23 03:33:12 +07:00
|
|
|
#include <linux/splice.h>
|
2014-11-24 13:08:00 +07:00
|
|
|
#include <net/checksum.h>
|
2014-02-06 07:11:33 +07:00
|
|
|
|
2016-09-23 03:33:12 +07:00
|
|
|
#define PIPE_PARANOIA /* for now */
|
|
|
|
|
iov_iter.c: macros for iterating over iov_iter
iterate_all_kinds(iter, size, ident, step_iovec, step_bvec)
iterates through the ranges covered by iter (up to size bytes total),
repeating step_iovec or step_bvec for each of those. ident is
declared in expansion of that thing, either as struct iovec or
struct bvec, and it contains the range we are currently looking
at. step_bvec should be a void expression, step_iovec - a size_t
one, with non-zero meaning "stop here, that many bytes from this
range left". In the end, the amount actually handled is stored
in size.
iov_iter_copy_from_user_atomic() and iov_iter_alignment() converted
to it.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2014-11-28 01:51:41 +07:00
|
|
|
#define iterate_iovec(i, n, __v, __p, skip, STEP) { \
|
|
|
|
size_t left; \
|
|
|
|
size_t wanted = n; \
|
|
|
|
__p = i->iov; \
|
|
|
|
__v.iov_len = min(n, __p->iov_len - skip); \
|
|
|
|
if (likely(__v.iov_len)) { \
|
|
|
|
__v.iov_base = __p->iov_base + skip; \
|
|
|
|
left = (STEP); \
|
|
|
|
__v.iov_len -= left; \
|
|
|
|
skip += __v.iov_len; \
|
|
|
|
n -= __v.iov_len; \
|
|
|
|
} else { \
|
|
|
|
left = 0; \
|
|
|
|
} \
|
|
|
|
while (unlikely(!left && n)) { \
|
|
|
|
__p++; \
|
|
|
|
__v.iov_len = min(n, __p->iov_len); \
|
|
|
|
if (unlikely(!__v.iov_len)) \
|
|
|
|
continue; \
|
|
|
|
__v.iov_base = __p->iov_base; \
|
|
|
|
left = (STEP); \
|
|
|
|
__v.iov_len -= left; \
|
|
|
|
skip = __v.iov_len; \
|
|
|
|
n -= __v.iov_len; \
|
|
|
|
} \
|
|
|
|
n = wanted - n; \
|
|
|
|
}
|
|
|
|
|
2014-11-28 02:48:42 +07:00
|
|
|
#define iterate_kvec(i, n, __v, __p, skip, STEP) { \
|
|
|
|
size_t wanted = n; \
|
|
|
|
__p = i->kvec; \
|
|
|
|
__v.iov_len = min(n, __p->iov_len - skip); \
|
|
|
|
if (likely(__v.iov_len)) { \
|
|
|
|
__v.iov_base = __p->iov_base + skip; \
|
|
|
|
(void)(STEP); \
|
|
|
|
skip += __v.iov_len; \
|
|
|
|
n -= __v.iov_len; \
|
|
|
|
} \
|
|
|
|
while (unlikely(n)) { \
|
|
|
|
__p++; \
|
|
|
|
__v.iov_len = min(n, __p->iov_len); \
|
|
|
|
if (unlikely(!__v.iov_len)) \
|
|
|
|
continue; \
|
|
|
|
__v.iov_base = __p->iov_base; \
|
|
|
|
(void)(STEP); \
|
|
|
|
skip = __v.iov_len; \
|
|
|
|
n -= __v.iov_len; \
|
|
|
|
} \
|
|
|
|
n = wanted; \
|
|
|
|
}
|
|
|
|
|
2016-05-30 20:34:32 +07:00
|
|
|
#define iterate_bvec(i, n, __v, __bi, skip, STEP) { \
|
|
|
|
struct bvec_iter __start; \
|
|
|
|
__start.bi_size = n; \
|
|
|
|
__start.bi_bvec_done = skip; \
|
|
|
|
__start.bi_idx = 0; \
|
|
|
|
for_each_bvec(__v, i->bvec, __bi, __start) { \
|
|
|
|
if (!__v.bv_len) \
|
iov_iter.c: macros for iterating over iov_iter
iterate_all_kinds(iter, size, ident, step_iovec, step_bvec)
iterates through the ranges covered by iter (up to size bytes total),
repeating step_iovec or step_bvec for each of those. ident is
declared in expansion of that thing, either as struct iovec or
struct bvec, and it contains the range we are currently looking
at. step_bvec should be a void expression, step_iovec - a size_t
one, with non-zero meaning "stop here, that many bytes from this
range left". In the end, the amount actually handled is stored
in size.
iov_iter_copy_from_user_atomic() and iov_iter_alignment() converted
to it.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2014-11-28 01:51:41 +07:00
|
|
|
continue; \
|
|
|
|
(void)(STEP); \
|
|
|
|
} \
|
|
|
|
}
|
|
|
|
|
2014-11-28 02:48:42 +07:00
|
|
|
#define iterate_all_kinds(i, n, v, I, B, K) { \
|
2016-12-22 09:55:02 +07:00
|
|
|
if (likely(n)) { \
|
|
|
|
size_t skip = i->iov_offset; \
|
|
|
|
if (unlikely(i->type & ITER_BVEC)) { \
|
|
|
|
struct bio_vec v; \
|
|
|
|
struct bvec_iter __bi; \
|
|
|
|
iterate_bvec(i, n, v, __bi, skip, (B)) \
|
|
|
|
} else if (unlikely(i->type & ITER_KVEC)) { \
|
|
|
|
const struct kvec *kvec; \
|
|
|
|
struct kvec v; \
|
|
|
|
iterate_kvec(i, n, v, kvec, skip, (K)) \
|
|
|
|
} else { \
|
|
|
|
const struct iovec *iov; \
|
|
|
|
struct iovec v; \
|
|
|
|
iterate_iovec(i, n, v, iov, skip, (I)) \
|
|
|
|
} \
|
iov_iter.c: macros for iterating over iov_iter
iterate_all_kinds(iter, size, ident, step_iovec, step_bvec)
iterates through the ranges covered by iter (up to size bytes total),
repeating step_iovec or step_bvec for each of those. ident is
declared in expansion of that thing, either as struct iovec or
struct bvec, and it contains the range we are currently looking
at. step_bvec should be a void expression, step_iovec - a size_t
one, with non-zero meaning "stop here, that many bytes from this
range left". In the end, the amount actually handled is stored
in size.
iov_iter_copy_from_user_atomic() and iov_iter_alignment() converted
to it.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2014-11-28 01:51:41 +07:00
|
|
|
} \
|
|
|
|
}
|
|
|
|
|
2014-11-28 02:48:42 +07:00
|
|
|
#define iterate_and_advance(i, n, v, I, B, K) { \
|
2016-05-09 22:54:48 +07:00
|
|
|
if (unlikely(i->count < n)) \
|
|
|
|
n = i->count; \
|
2016-05-26 04:36:19 +07:00
|
|
|
if (i->count) { \
|
2016-05-09 22:54:48 +07:00
|
|
|
size_t skip = i->iov_offset; \
|
|
|
|
if (unlikely(i->type & ITER_BVEC)) { \
|
2016-05-30 20:34:32 +07:00
|
|
|
const struct bio_vec *bvec = i->bvec; \
|
2016-05-09 22:54:48 +07:00
|
|
|
struct bio_vec v; \
|
2016-05-30 20:34:32 +07:00
|
|
|
struct bvec_iter __bi; \
|
|
|
|
iterate_bvec(i, n, v, __bi, skip, (B)) \
|
|
|
|
i->bvec = __bvec_iter_bvec(i->bvec, __bi); \
|
|
|
|
i->nr_segs -= i->bvec - bvec; \
|
|
|
|
skip = __bi.bi_bvec_done; \
|
2016-05-09 22:54:48 +07:00
|
|
|
} else if (unlikely(i->type & ITER_KVEC)) { \
|
|
|
|
const struct kvec *kvec; \
|
|
|
|
struct kvec v; \
|
|
|
|
iterate_kvec(i, n, v, kvec, skip, (K)) \
|
|
|
|
if (skip == kvec->iov_len) { \
|
|
|
|
kvec++; \
|
|
|
|
skip = 0; \
|
|
|
|
} \
|
|
|
|
i->nr_segs -= kvec - i->kvec; \
|
|
|
|
i->kvec = kvec; \
|
|
|
|
} else { \
|
|
|
|
const struct iovec *iov; \
|
|
|
|
struct iovec v; \
|
|
|
|
iterate_iovec(i, n, v, iov, skip, (I)) \
|
|
|
|
if (skip == iov->iov_len) { \
|
|
|
|
iov++; \
|
|
|
|
skip = 0; \
|
|
|
|
} \
|
|
|
|
i->nr_segs -= iov - i->iov; \
|
|
|
|
i->iov = iov; \
|
2014-11-28 01:59:45 +07:00
|
|
|
} \
|
2016-05-09 22:54:48 +07:00
|
|
|
i->count -= n; \
|
|
|
|
i->iov_offset = skip; \
|
2014-11-28 01:59:45 +07:00
|
|
|
} \
|
|
|
|
}
|
|
|
|
|
2017-06-30 09:25:14 +07:00
|
|
|
static int copyout(void __user *to, const void *from, size_t n)
|
|
|
|
{
|
|
|
|
if (access_ok(VERIFY_WRITE, to, n)) {
|
|
|
|
kasan_check_read(from, n);
|
|
|
|
n = raw_copy_to_user(to, from, n);
|
|
|
|
}
|
|
|
|
return n;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int copyin(void *to, const void __user *from, size_t n)
|
|
|
|
{
|
|
|
|
if (access_ok(VERIFY_READ, from, n)) {
|
|
|
|
kasan_check_write(to, n);
|
|
|
|
n = raw_copy_from_user(to, from, n);
|
|
|
|
}
|
|
|
|
return n;
|
|
|
|
}
|
|
|
|
|
2014-04-05 10:12:29 +07:00
|
|
|
static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
|
2014-02-06 07:11:33 +07:00
|
|
|
struct iov_iter *i)
|
|
|
|
{
|
|
|
|
size_t skip, copy, left, wanted;
|
|
|
|
const struct iovec *iov;
|
|
|
|
char __user *buf;
|
|
|
|
void *kaddr, *from;
|
|
|
|
|
|
|
|
if (unlikely(bytes > i->count))
|
|
|
|
bytes = i->count;
|
|
|
|
|
|
|
|
if (unlikely(!bytes))
|
|
|
|
return 0;
|
|
|
|
|
2017-06-30 09:25:14 +07:00
|
|
|
might_fault();
|
2014-02-06 07:11:33 +07:00
|
|
|
wanted = bytes;
|
|
|
|
iov = i->iov;
|
|
|
|
skip = i->iov_offset;
|
|
|
|
buf = iov->iov_base + skip;
|
|
|
|
copy = min(bytes, iov->iov_len - skip);
|
|
|
|
|
2016-07-29 05:48:50 +07:00
|
|
|
if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) {
|
2014-02-06 07:11:33 +07:00
|
|
|
kaddr = kmap_atomic(page);
|
|
|
|
from = kaddr + offset;
|
|
|
|
|
|
|
|
/* first chunk, usually the only one */
|
2017-06-30 09:25:14 +07:00
|
|
|
left = copyout(buf, from, copy);
|
2014-02-06 07:11:33 +07:00
|
|
|
copy -= left;
|
|
|
|
skip += copy;
|
|
|
|
from += copy;
|
|
|
|
bytes -= copy;
|
|
|
|
|
|
|
|
while (unlikely(!left && bytes)) {
|
|
|
|
iov++;
|
|
|
|
buf = iov->iov_base;
|
|
|
|
copy = min(bytes, iov->iov_len);
|
2017-06-30 09:25:14 +07:00
|
|
|
left = copyout(buf, from, copy);
|
2014-02-06 07:11:33 +07:00
|
|
|
copy -= left;
|
|
|
|
skip = copy;
|
|
|
|
from += copy;
|
|
|
|
bytes -= copy;
|
|
|
|
}
|
|
|
|
if (likely(!bytes)) {
|
|
|
|
kunmap_atomic(kaddr);
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
offset = from - kaddr;
|
|
|
|
buf += copy;
|
|
|
|
kunmap_atomic(kaddr);
|
|
|
|
copy = min(bytes, iov->iov_len - skip);
|
|
|
|
}
|
|
|
|
/* Too bad - revert to non-atomic kmap */
|
2016-07-29 05:48:50 +07:00
|
|
|
|
2014-02-06 07:11:33 +07:00
|
|
|
kaddr = kmap(page);
|
|
|
|
from = kaddr + offset;
|
2017-06-30 09:25:14 +07:00
|
|
|
left = copyout(buf, from, copy);
|
2014-02-06 07:11:33 +07:00
|
|
|
copy -= left;
|
|
|
|
skip += copy;
|
|
|
|
from += copy;
|
|
|
|
bytes -= copy;
|
|
|
|
while (unlikely(!left && bytes)) {
|
|
|
|
iov++;
|
|
|
|
buf = iov->iov_base;
|
|
|
|
copy = min(bytes, iov->iov_len);
|
2017-06-30 09:25:14 +07:00
|
|
|
left = copyout(buf, from, copy);
|
2014-02-06 07:11:33 +07:00
|
|
|
copy -= left;
|
|
|
|
skip = copy;
|
|
|
|
from += copy;
|
|
|
|
bytes -= copy;
|
|
|
|
}
|
|
|
|
kunmap(page);
|
2016-07-29 05:48:50 +07:00
|
|
|
|
2014-02-06 07:11:33 +07:00
|
|
|
done:
|
2014-04-05 06:23:46 +07:00
|
|
|
if (skip == iov->iov_len) {
|
|
|
|
iov++;
|
|
|
|
skip = 0;
|
|
|
|
}
|
2014-02-06 07:11:33 +07:00
|
|
|
i->count -= wanted - bytes;
|
|
|
|
i->nr_segs -= iov - i->iov;
|
|
|
|
i->iov = iov;
|
|
|
|
i->iov_offset = skip;
|
|
|
|
return wanted - bytes;
|
|
|
|
}
|
|
|
|
|
2014-04-05 10:12:29 +07:00
|
|
|
static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
|
2014-04-04 02:05:18 +07:00
|
|
|
struct iov_iter *i)
|
|
|
|
{
|
|
|
|
size_t skip, copy, left, wanted;
|
|
|
|
const struct iovec *iov;
|
|
|
|
char __user *buf;
|
|
|
|
void *kaddr, *to;
|
|
|
|
|
|
|
|
if (unlikely(bytes > i->count))
|
|
|
|
bytes = i->count;
|
|
|
|
|
|
|
|
if (unlikely(!bytes))
|
|
|
|
return 0;
|
|
|
|
|
2017-06-30 09:25:14 +07:00
|
|
|
might_fault();
|
2014-04-04 02:05:18 +07:00
|
|
|
wanted = bytes;
|
|
|
|
iov = i->iov;
|
|
|
|
skip = i->iov_offset;
|
|
|
|
buf = iov->iov_base + skip;
|
|
|
|
copy = min(bytes, iov->iov_len - skip);
|
|
|
|
|
2016-07-29 05:48:50 +07:00
|
|
|
if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) {
|
2014-04-04 02:05:18 +07:00
|
|
|
kaddr = kmap_atomic(page);
|
|
|
|
to = kaddr + offset;
|
|
|
|
|
|
|
|
/* first chunk, usually the only one */
|
2017-06-30 09:25:14 +07:00
|
|
|
left = copyin(to, buf, copy);
|
2014-04-04 02:05:18 +07:00
|
|
|
copy -= left;
|
|
|
|
skip += copy;
|
|
|
|
to += copy;
|
|
|
|
bytes -= copy;
|
|
|
|
|
|
|
|
while (unlikely(!left && bytes)) {
|
|
|
|
iov++;
|
|
|
|
buf = iov->iov_base;
|
|
|
|
copy = min(bytes, iov->iov_len);
|
2017-06-30 09:25:14 +07:00
|
|
|
left = copyin(to, buf, copy);
|
2014-04-04 02:05:18 +07:00
|
|
|
copy -= left;
|
|
|
|
skip = copy;
|
|
|
|
to += copy;
|
|
|
|
bytes -= copy;
|
|
|
|
}
|
|
|
|
if (likely(!bytes)) {
|
|
|
|
kunmap_atomic(kaddr);
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
offset = to - kaddr;
|
|
|
|
buf += copy;
|
|
|
|
kunmap_atomic(kaddr);
|
|
|
|
copy = min(bytes, iov->iov_len - skip);
|
|
|
|
}
|
|
|
|
/* Too bad - revert to non-atomic kmap */
|
2016-07-29 05:48:50 +07:00
|
|
|
|
2014-04-04 02:05:18 +07:00
|
|
|
kaddr = kmap(page);
|
|
|
|
to = kaddr + offset;
|
2017-06-30 09:25:14 +07:00
|
|
|
left = copyin(to, buf, copy);
|
2014-04-04 02:05:18 +07:00
|
|
|
copy -= left;
|
|
|
|
skip += copy;
|
|
|
|
to += copy;
|
|
|
|
bytes -= copy;
|
|
|
|
while (unlikely(!left && bytes)) {
|
|
|
|
iov++;
|
|
|
|
buf = iov->iov_base;
|
|
|
|
copy = min(bytes, iov->iov_len);
|
2017-06-30 09:25:14 +07:00
|
|
|
left = copyin(to, buf, copy);
|
2014-04-04 02:05:18 +07:00
|
|
|
copy -= left;
|
|
|
|
skip = copy;
|
|
|
|
to += copy;
|
|
|
|
bytes -= copy;
|
|
|
|
}
|
|
|
|
kunmap(page);
|
2016-07-29 05:48:50 +07:00
|
|
|
|
2014-04-04 02:05:18 +07:00
|
|
|
done:
|
2014-04-05 06:23:46 +07:00
|
|
|
if (skip == iov->iov_len) {
|
|
|
|
iov++;
|
|
|
|
skip = 0;
|
|
|
|
}
|
2014-04-04 02:05:18 +07:00
|
|
|
i->count -= wanted - bytes;
|
|
|
|
i->nr_segs -= iov - i->iov;
|
|
|
|
i->iov = iov;
|
|
|
|
i->iov_offset = skip;
|
|
|
|
return wanted - bytes;
|
|
|
|
}
|
|
|
|
|
2016-09-23 03:33:12 +07:00
|
|
|
#ifdef PIPE_PARANOIA
|
|
|
|
static bool sanity(const struct iov_iter *i)
|
|
|
|
{
|
|
|
|
struct pipe_inode_info *pipe = i->pipe;
|
|
|
|
int idx = i->idx;
|
|
|
|
int next = pipe->curbuf + pipe->nrbufs;
|
|
|
|
if (i->iov_offset) {
|
|
|
|
struct pipe_buffer *p;
|
|
|
|
if (unlikely(!pipe->nrbufs))
|
|
|
|
goto Bad; // pipe must be non-empty
|
|
|
|
if (unlikely(idx != ((next - 1) & (pipe->buffers - 1))))
|
|
|
|
goto Bad; // must be at the last buffer...
|
|
|
|
|
|
|
|
p = &pipe->bufs[idx];
|
|
|
|
if (unlikely(p->offset + p->len != i->iov_offset))
|
|
|
|
goto Bad; // ... at the end of segment
|
|
|
|
} else {
|
|
|
|
if (idx != (next & (pipe->buffers - 1)))
|
|
|
|
goto Bad; // must be right after the last buffer
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
Bad:
|
|
|
|
printk(KERN_ERR "idx = %d, offset = %zd\n", i->idx, i->iov_offset);
|
|
|
|
printk(KERN_ERR "curbuf = %d, nrbufs = %d, buffers = %d\n",
|
|
|
|
pipe->curbuf, pipe->nrbufs, pipe->buffers);
|
|
|
|
for (idx = 0; idx < pipe->buffers; idx++)
|
|
|
|
printk(KERN_ERR "[%p %p %d %d]\n",
|
|
|
|
pipe->bufs[idx].ops,
|
|
|
|
pipe->bufs[idx].page,
|
|
|
|
pipe->bufs[idx].offset,
|
|
|
|
pipe->bufs[idx].len);
|
|
|
|
WARN_ON(1);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
#define sanity(i) true
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static inline int next_idx(int idx, struct pipe_inode_info *pipe)
|
|
|
|
{
|
|
|
|
return (idx + 1) & (pipe->buffers - 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,
|
|
|
|
struct iov_iter *i)
|
|
|
|
{
|
|
|
|
struct pipe_inode_info *pipe = i->pipe;
|
|
|
|
struct pipe_buffer *buf;
|
|
|
|
size_t off;
|
|
|
|
int idx;
|
|
|
|
|
|
|
|
if (unlikely(bytes > i->count))
|
|
|
|
bytes = i->count;
|
|
|
|
|
|
|
|
if (unlikely(!bytes))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (!sanity(i))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
off = i->iov_offset;
|
|
|
|
idx = i->idx;
|
|
|
|
buf = &pipe->bufs[idx];
|
|
|
|
if (off) {
|
|
|
|
if (offset == off && buf->page == page) {
|
|
|
|
/* merge with the last one */
|
|
|
|
buf->len += bytes;
|
|
|
|
i->iov_offset += bytes;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
idx = next_idx(idx, pipe);
|
|
|
|
buf = &pipe->bufs[idx];
|
|
|
|
}
|
|
|
|
if (idx == pipe->curbuf && pipe->nrbufs)
|
|
|
|
return 0;
|
|
|
|
pipe->nrbufs++;
|
|
|
|
buf->ops = &page_cache_pipe_buf_ops;
|
|
|
|
get_page(buf->page = page);
|
|
|
|
buf->offset = offset;
|
|
|
|
buf->len = bytes;
|
|
|
|
i->iov_offset = offset + bytes;
|
|
|
|
i->idx = idx;
|
|
|
|
out:
|
|
|
|
i->count -= bytes;
|
|
|
|
return bytes;
|
|
|
|
}
|
|
|
|
|
2015-03-11 21:43:31 +07:00
|
|
|
/*
|
|
|
|
* Fault in one or more iovecs of the given iov_iter, to a maximum length of
|
|
|
|
* bytes. For each iovec, fault in each page that constitutes the iovec.
|
|
|
|
*
|
|
|
|
* Return 0 on success, or non-zero if the memory could not be accessed (i.e.
|
|
|
|
* because it is an invalid address).
|
|
|
|
*/
|
2016-09-16 06:11:45 +07:00
|
|
|
int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
|
2015-03-11 21:43:31 +07:00
|
|
|
{
|
|
|
|
size_t skip = i->iov_offset;
|
|
|
|
const struct iovec *iov;
|
|
|
|
int err;
|
|
|
|
struct iovec v;
|
|
|
|
|
|
|
|
if (!(i->type & (ITER_BVEC|ITER_KVEC))) {
|
|
|
|
iterate_iovec(i, bytes, v, iov, skip, ({
|
2016-09-18 05:02:44 +07:00
|
|
|
err = fault_in_pages_readable(v.iov_base, v.iov_len);
|
2015-03-11 21:43:31 +07:00
|
|
|
if (unlikely(err))
|
|
|
|
return err;
|
|
|
|
0;}))
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
2016-09-16 06:11:45 +07:00
|
|
|
EXPORT_SYMBOL(iov_iter_fault_in_readable);
|
2015-03-11 21:43:31 +07:00
|
|
|
|
2014-03-06 07:28:09 +07:00
|
|
|
void iov_iter_init(struct iov_iter *i, int direction,
|
|
|
|
const struct iovec *iov, unsigned long nr_segs,
|
|
|
|
size_t count)
|
|
|
|
{
|
|
|
|
/* It will get better. Eventually... */
|
2017-03-21 08:08:07 +07:00
|
|
|
if (uaccess_kernel()) {
|
2014-04-05 10:12:29 +07:00
|
|
|
direction |= ITER_KVEC;
|
2014-11-28 02:48:42 +07:00
|
|
|
i->type = direction;
|
|
|
|
i->kvec = (struct kvec *)iov;
|
|
|
|
} else {
|
|
|
|
i->type = direction;
|
|
|
|
i->iov = iov;
|
|
|
|
}
|
2014-03-06 07:28:09 +07:00
|
|
|
i->nr_segs = nr_segs;
|
|
|
|
i->iov_offset = 0;
|
|
|
|
i->count = count;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_init);
|
2014-03-15 15:05:57 +07:00
|
|
|
|
2014-04-05 10:12:29 +07:00
|
|
|
static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
|
|
|
|
{
|
|
|
|
char *from = kmap_atomic(page);
|
|
|
|
memcpy(to, from + offset, len);
|
|
|
|
kunmap_atomic(from);
|
|
|
|
}
|
|
|
|
|
2015-12-07 04:49:22 +07:00
|
|
|
static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len)
|
2014-04-05 10:12:29 +07:00
|
|
|
{
|
|
|
|
char *to = kmap_atomic(page);
|
|
|
|
memcpy(to + offset, from, len);
|
|
|
|
kunmap_atomic(to);
|
|
|
|
}
|
|
|
|
|
2014-08-01 20:27:22 +07:00
|
|
|
static void memzero_page(struct page *page, size_t offset, size_t len)
|
|
|
|
{
|
|
|
|
char *addr = kmap_atomic(page);
|
|
|
|
memset(addr + offset, 0, len);
|
|
|
|
kunmap_atomic(addr);
|
|
|
|
}
|
|
|
|
|
2016-09-23 03:33:12 +07:00
|
|
|
static inline bool allocated(struct pipe_buffer *buf)
|
|
|
|
{
|
|
|
|
return buf->ops == &default_pipe_buf_ops;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void data_start(const struct iov_iter *i, int *idxp, size_t *offp)
|
|
|
|
{
|
|
|
|
size_t off = i->iov_offset;
|
|
|
|
int idx = i->idx;
|
|
|
|
if (off && (!allocated(&i->pipe->bufs[idx]) || off == PAGE_SIZE)) {
|
|
|
|
idx = next_idx(idx, i->pipe);
|
|
|
|
off = 0;
|
|
|
|
}
|
|
|
|
*idxp = idx;
|
|
|
|
*offp = off;
|
|
|
|
}
|
|
|
|
|
|
|
|
static size_t push_pipe(struct iov_iter *i, size_t size,
|
|
|
|
int *idxp, size_t *offp)
|
|
|
|
{
|
|
|
|
struct pipe_inode_info *pipe = i->pipe;
|
|
|
|
size_t off;
|
|
|
|
int idx;
|
|
|
|
ssize_t left;
|
|
|
|
|
|
|
|
if (unlikely(size > i->count))
|
|
|
|
size = i->count;
|
|
|
|
if (unlikely(!size))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
left = size;
|
|
|
|
data_start(i, &idx, &off);
|
|
|
|
*idxp = idx;
|
|
|
|
*offp = off;
|
|
|
|
if (off) {
|
|
|
|
left -= PAGE_SIZE - off;
|
|
|
|
if (left <= 0) {
|
|
|
|
pipe->bufs[idx].len += size;
|
|
|
|
return size;
|
|
|
|
}
|
|
|
|
pipe->bufs[idx].len = PAGE_SIZE;
|
|
|
|
idx = next_idx(idx, pipe);
|
|
|
|
}
|
|
|
|
while (idx != pipe->curbuf || !pipe->nrbufs) {
|
|
|
|
struct page *page = alloc_page(GFP_USER);
|
|
|
|
if (!page)
|
|
|
|
break;
|
|
|
|
pipe->nrbufs++;
|
|
|
|
pipe->bufs[idx].ops = &default_pipe_buf_ops;
|
|
|
|
pipe->bufs[idx].page = page;
|
|
|
|
pipe->bufs[idx].offset = 0;
|
|
|
|
if (left <= PAGE_SIZE) {
|
|
|
|
pipe->bufs[idx].len = left;
|
|
|
|
return size;
|
|
|
|
}
|
|
|
|
pipe->bufs[idx].len = PAGE_SIZE;
|
|
|
|
left -= PAGE_SIZE;
|
|
|
|
idx = next_idx(idx, pipe);
|
|
|
|
}
|
|
|
|
return size - left;
|
|
|
|
}
|
|
|
|
|
|
|
|
static size_t copy_pipe_to_iter(const void *addr, size_t bytes,
|
|
|
|
struct iov_iter *i)
|
|
|
|
{
|
|
|
|
struct pipe_inode_info *pipe = i->pipe;
|
|
|
|
size_t n, off;
|
|
|
|
int idx;
|
|
|
|
|
|
|
|
if (!sanity(i))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
bytes = n = push_pipe(i, bytes, &idx, &off);
|
|
|
|
if (unlikely(!n))
|
|
|
|
return 0;
|
|
|
|
for ( ; n; idx = next_idx(idx, pipe), off = 0) {
|
|
|
|
size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
|
|
|
|
memcpy_to_page(pipe->bufs[idx].page, off, addr, chunk);
|
|
|
|
i->idx = idx;
|
|
|
|
i->iov_offset = off + chunk;
|
|
|
|
n -= chunk;
|
|
|
|
addr += chunk;
|
|
|
|
}
|
|
|
|
i->count -= bytes;
|
|
|
|
return bytes;
|
|
|
|
}
|
|
|
|
|
2017-06-30 08:45:10 +07:00
|
|
|
size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
|
2014-04-05 10:12:29 +07:00
|
|
|
{
|
2015-12-07 04:49:22 +07:00
|
|
|
const char *from = addr;
|
2016-09-23 03:33:12 +07:00
|
|
|
if (unlikely(i->type & ITER_PIPE))
|
|
|
|
return copy_pipe_to_iter(addr, bytes, i);
|
2017-06-30 09:25:14 +07:00
|
|
|
if (iter_is_iovec(i))
|
|
|
|
might_fault();
|
2014-11-28 02:28:06 +07:00
|
|
|
iterate_and_advance(i, bytes, v,
|
2017-06-30 09:25:14 +07:00
|
|
|
copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
|
2014-11-28 02:28:06 +07:00
|
|
|
memcpy_to_page(v.bv_page, v.bv_offset,
|
2014-11-28 02:48:42 +07:00
|
|
|
(from += v.bv_len) - v.bv_len, v.bv_len),
|
|
|
|
memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
|
2014-11-28 02:28:06 +07:00
|
|
|
)
|
2014-04-05 10:12:29 +07:00
|
|
|
|
2014-11-28 02:28:06 +07:00
|
|
|
return bytes;
|
2014-08-01 20:27:22 +07:00
|
|
|
}
|
2017-06-30 08:45:10 +07:00
|
|
|
EXPORT_SYMBOL(_copy_to_iter);
|
2014-08-01 20:27:22 +07:00
|
|
|
|
2017-06-30 08:45:10 +07:00
|
|
|
size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
|
2014-08-01 20:27:22 +07:00
|
|
|
{
|
2014-11-28 02:26:43 +07:00
|
|
|
char *to = addr;
|
2016-09-23 03:33:12 +07:00
|
|
|
if (unlikely(i->type & ITER_PIPE)) {
|
|
|
|
WARN_ON(1);
|
|
|
|
return 0;
|
|
|
|
}
|
2017-06-30 09:25:14 +07:00
|
|
|
if (iter_is_iovec(i))
|
|
|
|
might_fault();
|
2014-11-28 02:26:43 +07:00
|
|
|
iterate_and_advance(i, bytes, v,
|
2017-06-30 09:25:14 +07:00
|
|
|
copyin((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
|
2014-11-28 02:26:43 +07:00
|
|
|
memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
|
2014-11-28 02:48:42 +07:00
|
|
|
v.bv_offset, v.bv_len),
|
|
|
|
memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
|
2014-11-28 02:26:43 +07:00
|
|
|
)
|
|
|
|
|
|
|
|
return bytes;
|
2014-08-01 20:27:22 +07:00
|
|
|
}
|
2017-06-30 08:45:10 +07:00
|
|
|
EXPORT_SYMBOL(_copy_from_iter);
|
2014-08-01 20:27:22 +07:00
|
|
|
|
2017-06-30 08:45:10 +07:00
|
|
|
bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
|
2016-11-02 09:09:04 +07:00
|
|
|
{
|
|
|
|
char *to = addr;
|
|
|
|
if (unlikely(i->type & ITER_PIPE)) {
|
|
|
|
WARN_ON(1);
|
|
|
|
return false;
|
|
|
|
}
|
2016-12-22 09:55:02 +07:00
|
|
|
if (unlikely(i->count < bytes))
|
2016-11-02 09:09:04 +07:00
|
|
|
return false;
|
|
|
|
|
2017-06-30 09:25:14 +07:00
|
|
|
if (iter_is_iovec(i))
|
|
|
|
might_fault();
|
2016-11-02 09:09:04 +07:00
|
|
|
iterate_all_kinds(i, bytes, v, ({
|
2017-06-30 09:25:14 +07:00
|
|
|
if (copyin((to += v.iov_len) - v.iov_len,
|
2016-11-02 09:09:04 +07:00
|
|
|
v.iov_base, v.iov_len))
|
|
|
|
return false;
|
|
|
|
0;}),
|
|
|
|
memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
|
|
|
|
v.bv_offset, v.bv_len),
|
|
|
|
memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
|
|
|
|
)
|
|
|
|
|
|
|
|
iov_iter_advance(i, bytes);
|
|
|
|
return true;
|
|
|
|
}
|
2017-06-30 08:45:10 +07:00
|
|
|
EXPORT_SYMBOL(_copy_from_iter_full);
|
2016-11-02 09:09:04 +07:00
|
|
|
|
2017-06-30 08:45:10 +07:00
|
|
|
size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
|
2014-11-28 08:27:08 +07:00
|
|
|
{
|
|
|
|
char *to = addr;
|
2016-09-23 03:33:12 +07:00
|
|
|
if (unlikely(i->type & ITER_PIPE)) {
|
|
|
|
WARN_ON(1);
|
|
|
|
return 0;
|
|
|
|
}
|
2014-11-28 08:27:08 +07:00
|
|
|
iterate_and_advance(i, bytes, v,
|
2017-03-26 05:47:28 +07:00
|
|
|
__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
|
2014-11-28 08:27:08 +07:00
|
|
|
v.iov_base, v.iov_len),
|
|
|
|
memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
|
|
|
|
v.bv_offset, v.bv_len),
|
|
|
|
memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
|
|
|
|
)
|
|
|
|
|
|
|
|
return bytes;
|
|
|
|
}
|
2017-06-30 08:45:10 +07:00
|
|
|
EXPORT_SYMBOL(_copy_from_iter_nocache);
|
2014-11-28 08:27:08 +07:00
|
|
|
|
2017-05-30 02:22:50 +07:00
|
|
|
#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
|
Merge branch 'uaccess-work.iov_iter' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs
Pull iov_iter hardening from Al Viro:
"This is the iov_iter/uaccess/hardening pile.
For one thing, it trims the inline part of copy_to_user/copy_from_user
to the minimum that *does* need to be inlined - object size checks,
basically. For another, it sanitizes the checks for iov_iter
primitives. There are 4 groups of checks: access_ok(), might_fault(),
object size and KASAN.
- access_ok() had been verified by whoever had set the iov_iter up.
However, that has happened in a function far away, so proving that
there's no path to actual copying bypassing those checks is hard
and proving that iov_iter has not been buggered in the meanwhile is
also not pleasant. So we want those redone in actual
copyin/copyout.
- might_fault() is better off consolidated - we know whether it needs
to be checked as soon as we enter iov_iter primitive and observe
the iov_iter flavour. No need to wait until the copyin/copyout. The
call chains are short enough to make sure we won't miss anything -
in fact, it's more robust that way, since there are cases where we
do e.g. forced fault-in before getting to copyin/copyout. It's not
quite what we need to check (in particular, combination of
iovec-backed and set_fs(KERNEL_DS) is almost certainly a bug, not a
cause to skip checks), but that's for later series. For now let's
keep might_fault().
- KASAN checks belong in copyin/copyout - at the same level where
other iov_iter flavours would've hit them in memcpy().
- object size checks should apply to *all* iov_iter flavours, not
just iovec-backed ones.
There are two groups of primitives - one gets the kernel object
described as pointer + size (copy_to_iter(), etc.) while another gets
it as page + offset + size (copy_page_to_iter(), etc.)
For the first group the checks are best done where we actually have a
chance to find the object size. In other words, those belong in inline
wrappers in uio.h, before calling into iov_iter.c. Same kind as we
have for inlined part of copy_to_user().
For the second group there is no object to look at - offset in page is
just a number, it bears no type information. So we do them in the
common helper called by iov_iter.c primitives of that kind. All it
currently does is checking that we are not trying to access outside of
the compound page; eventually we might want to add some sanity checks
on the page involved.
So the things we need in copyin/copyout part of iov_iter.c do not
quite match anything in uaccess.h (we want no zeroing, we *do* want
access_ok() and KASAN and we want no might_fault() or object size
checks done on that level). OTOH, these needs are simple enough to
provide a couple of helpers (static in iov_iter.c) doing just what we
need..."
* 'uaccess-work.iov_iter' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs:
iov_iter: saner checks on copyin/copyout
iov_iter: sanity checks for copy to/from page primitives
iov_iter/hardening: move object size checks to inlined part
copy_{to,from}_user(): consolidate object size checks
copy_{from,to}_user(): move kasan checks and might_fault() out-of-line
2017-07-08 10:39:20 +07:00
|
|
|
size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
|
2017-05-30 02:22:50 +07:00
|
|
|
{
|
|
|
|
char *to = addr;
|
|
|
|
if (unlikely(i->type & ITER_PIPE)) {
|
|
|
|
WARN_ON(1);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
iterate_and_advance(i, bytes, v,
|
|
|
|
__copy_from_user_flushcache((to += v.iov_len) - v.iov_len,
|
|
|
|
v.iov_base, v.iov_len),
|
|
|
|
memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page,
|
|
|
|
v.bv_offset, v.bv_len),
|
|
|
|
memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base,
|
|
|
|
v.iov_len)
|
|
|
|
)
|
|
|
|
|
|
|
|
return bytes;
|
|
|
|
}
|
Merge branch 'uaccess-work.iov_iter' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs
Pull iov_iter hardening from Al Viro:
"This is the iov_iter/uaccess/hardening pile.
For one thing, it trims the inline part of copy_to_user/copy_from_user
to the minimum that *does* need to be inlined - object size checks,
basically. For another, it sanitizes the checks for iov_iter
primitives. There are 4 groups of checks: access_ok(), might_fault(),
object size and KASAN.
- access_ok() had been verified by whoever had set the iov_iter up.
However, that has happened in a function far away, so proving that
there's no path to actual copying bypassing those checks is hard
and proving that iov_iter has not been buggered in the meanwhile is
also not pleasant. So we want those redone in actual
copyin/copyout.
- might_fault() is better off consolidated - we know whether it needs
to be checked as soon as we enter iov_iter primitive and observe
the iov_iter flavour. No need to wait until the copyin/copyout. The
call chains are short enough to make sure we won't miss anything -
in fact, it's more robust that way, since there are cases where we
do e.g. forced fault-in before getting to copyin/copyout. It's not
quite what we need to check (in particular, combination of
iovec-backed and set_fs(KERNEL_DS) is almost certainly a bug, not a
cause to skip checks), but that's for later series. For now let's
keep might_fault().
- KASAN checks belong in copyin/copyout - at the same level where
other iov_iter flavours would've hit them in memcpy().
- object size checks should apply to *all* iov_iter flavours, not
just iovec-backed ones.
There are two groups of primitives - one gets the kernel object
described as pointer + size (copy_to_iter(), etc.) while another gets
it as page + offset + size (copy_page_to_iter(), etc.)
For the first group the checks are best done where we actually have a
chance to find the object size. In other words, those belong in inline
wrappers in uio.h, before calling into iov_iter.c. Same kind as we
have for inlined part of copy_to_user().
For the second group there is no object to look at - offset in page is
just a number, it bears no type information. So we do them in the
common helper called by iov_iter.c primitives of that kind. All it
currently does is checking that we are not trying to access outside of
the compound page; eventually we might want to add some sanity checks
on the page involved.
So the things we need in copyin/copyout part of iov_iter.c do not
quite match anything in uaccess.h (we want no zeroing, we *do* want
access_ok() and KASAN and we want no might_fault() or object size
checks done on that level). OTOH, these needs are simple enough to
provide a couple of helpers (static in iov_iter.c) doing just what we
need..."
* 'uaccess-work.iov_iter' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs:
iov_iter: saner checks on copyin/copyout
iov_iter: sanity checks for copy to/from page primitives
iov_iter/hardening: move object size checks to inlined part
copy_{to,from}_user(): consolidate object size checks
copy_{from,to}_user(): move kasan checks and might_fault() out-of-line
2017-07-08 10:39:20 +07:00
|
|
|
EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache);
|
2017-05-30 02:22:50 +07:00
|
|
|
#endif
|
|
|
|
|
2017-06-30 08:45:10 +07:00
|
|
|
bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
|
2016-11-02 09:09:04 +07:00
|
|
|
{
|
|
|
|
char *to = addr;
|
|
|
|
if (unlikely(i->type & ITER_PIPE)) {
|
|
|
|
WARN_ON(1);
|
|
|
|
return false;
|
|
|
|
}
|
2016-12-22 09:55:02 +07:00
|
|
|
if (unlikely(i->count < bytes))
|
2016-11-02 09:09:04 +07:00
|
|
|
return false;
|
|
|
|
iterate_all_kinds(i, bytes, v, ({
|
2017-03-26 05:47:28 +07:00
|
|
|
if (__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
|
2016-11-02 09:09:04 +07:00
|
|
|
v.iov_base, v.iov_len))
|
|
|
|
return false;
|
|
|
|
0;}),
|
|
|
|
memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
|
|
|
|
v.bv_offset, v.bv_len),
|
|
|
|
memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
|
|
|
|
)
|
|
|
|
|
|
|
|
iov_iter_advance(i, bytes);
|
|
|
|
return true;
|
|
|
|
}
|
2017-06-30 08:45:10 +07:00
|
|
|
EXPORT_SYMBOL(_copy_from_iter_full_nocache);
|
2016-11-02 09:09:04 +07:00
|
|
|
|
2017-06-30 08:52:57 +07:00
|
|
|
static inline bool page_copy_sane(struct page *page, size_t offset, size_t n)
|
|
|
|
{
|
2017-08-30 01:20:32 +07:00
|
|
|
struct page *head = compound_head(page);
|
|
|
|
size_t v = n + offset + page_address(page) - page_address(head);
|
|
|
|
|
|
|
|
if (likely(n <= v && v <= (PAGE_SIZE << compound_order(head))))
|
2017-06-30 08:52:57 +07:00
|
|
|
return true;
|
|
|
|
WARN_ON(1);
|
|
|
|
return false;
|
|
|
|
}
|
2016-11-02 09:09:04 +07:00
|
|
|
|
2014-04-05 10:12:29 +07:00
|
|
|
size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
|
|
|
|
struct iov_iter *i)
|
|
|
|
{
|
2017-06-30 08:52:57 +07:00
|
|
|
if (unlikely(!page_copy_sane(page, offset, bytes)))
|
|
|
|
return 0;
|
2014-11-28 02:22:37 +07:00
|
|
|
if (i->type & (ITER_BVEC|ITER_KVEC)) {
|
|
|
|
void *kaddr = kmap_atomic(page);
|
|
|
|
size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
|
|
|
|
kunmap_atomic(kaddr);
|
|
|
|
return wanted;
|
2016-09-23 03:33:12 +07:00
|
|
|
} else if (likely(!(i->type & ITER_PIPE)))
|
2014-04-05 10:12:29 +07:00
|
|
|
return copy_page_to_iter_iovec(page, offset, bytes, i);
|
2016-09-23 03:33:12 +07:00
|
|
|
else
|
|
|
|
return copy_page_to_iter_pipe(page, offset, bytes, i);
|
2014-04-05 10:12:29 +07:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(copy_page_to_iter);
|
|
|
|
|
|
|
|
size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
|
|
|
|
struct iov_iter *i)
|
|
|
|
{
|
2017-06-30 08:52:57 +07:00
|
|
|
if (unlikely(!page_copy_sane(page, offset, bytes)))
|
|
|
|
return 0;
|
2016-09-23 03:33:12 +07:00
|
|
|
if (unlikely(i->type & ITER_PIPE)) {
|
|
|
|
WARN_ON(1);
|
|
|
|
return 0;
|
|
|
|
}
|
2014-11-28 02:48:42 +07:00
|
|
|
if (i->type & (ITER_BVEC|ITER_KVEC)) {
|
2014-11-28 02:22:37 +07:00
|
|
|
void *kaddr = kmap_atomic(page);
|
2017-06-30 08:45:10 +07:00
|
|
|
size_t wanted = _copy_from_iter(kaddr + offset, bytes, i);
|
2014-11-28 02:22:37 +07:00
|
|
|
kunmap_atomic(kaddr);
|
|
|
|
return wanted;
|
|
|
|
} else
|
2014-04-05 10:12:29 +07:00
|
|
|
return copy_page_from_iter_iovec(page, offset, bytes, i);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(copy_page_from_iter);
|
|
|
|
|
2016-09-23 03:33:12 +07:00
|
|
|
static size_t pipe_zero(size_t bytes, struct iov_iter *i)
|
|
|
|
{
|
|
|
|
struct pipe_inode_info *pipe = i->pipe;
|
|
|
|
size_t n, off;
|
|
|
|
int idx;
|
|
|
|
|
|
|
|
if (!sanity(i))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
bytes = n = push_pipe(i, bytes, &idx, &off);
|
|
|
|
if (unlikely(!n))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
for ( ; n; idx = next_idx(idx, pipe), off = 0) {
|
|
|
|
size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
|
|
|
|
memzero_page(pipe->bufs[idx].page, off, chunk);
|
|
|
|
i->idx = idx;
|
|
|
|
i->iov_offset = off + chunk;
|
|
|
|
n -= chunk;
|
|
|
|
}
|
|
|
|
i->count -= bytes;
|
|
|
|
return bytes;
|
|
|
|
}
|
|
|
|
|
2014-08-01 20:27:22 +07:00
|
|
|
size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
|
|
|
|
{
|
2016-09-23 03:33:12 +07:00
|
|
|
if (unlikely(i->type & ITER_PIPE))
|
|
|
|
return pipe_zero(bytes, i);
|
2014-11-28 02:18:54 +07:00
|
|
|
iterate_and_advance(i, bytes, v,
|
2017-06-30 09:25:14 +07:00
|
|
|
clear_user(v.iov_base, v.iov_len),
|
2014-11-28 02:48:42 +07:00
|
|
|
memzero_page(v.bv_page, v.bv_offset, v.bv_len),
|
|
|
|
memset(v.iov_base, 0, v.iov_len)
|
2014-11-28 02:18:54 +07:00
|
|
|
)
|
|
|
|
|
|
|
|
return bytes;
|
2014-08-01 20:27:22 +07:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_zero);
|
|
|
|
|
2014-04-05 10:12:29 +07:00
|
|
|
size_t iov_iter_copy_from_user_atomic(struct page *page,
|
|
|
|
struct iov_iter *i, unsigned long offset, size_t bytes)
|
|
|
|
{
|
iov_iter.c: macros for iterating over iov_iter
iterate_all_kinds(iter, size, ident, step_iovec, step_bvec)
iterates through the ranges covered by iter (up to size bytes total),
repeating step_iovec or step_bvec for each of those. ident is
declared in expansion of that thing, either as struct iovec or
struct bvec, and it contains the range we are currently looking
at. step_bvec should be a void expression, step_iovec - a size_t
one, with non-zero meaning "stop here, that many bytes from this
range left". In the end, the amount actually handled is stored
in size.
iov_iter_copy_from_user_atomic() and iov_iter_alignment() converted
to it.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2014-11-28 01:51:41 +07:00
|
|
|
char *kaddr = kmap_atomic(page), *p = kaddr + offset;
|
2017-06-30 08:52:57 +07:00
|
|
|
if (unlikely(!page_copy_sane(page, offset, bytes))) {
|
|
|
|
kunmap_atomic(kaddr);
|
|
|
|
return 0;
|
|
|
|
}
|
2016-09-23 03:33:12 +07:00
|
|
|
if (unlikely(i->type & ITER_PIPE)) {
|
|
|
|
kunmap_atomic(kaddr);
|
|
|
|
WARN_ON(1);
|
|
|
|
return 0;
|
|
|
|
}
|
iov_iter.c: macros for iterating over iov_iter
iterate_all_kinds(iter, size, ident, step_iovec, step_bvec)
iterates through the ranges covered by iter (up to size bytes total),
repeating step_iovec or step_bvec for each of those. ident is
declared in expansion of that thing, either as struct iovec or
struct bvec, and it contains the range we are currently looking
at. step_bvec should be a void expression, step_iovec - a size_t
one, with non-zero meaning "stop here, that many bytes from this
range left". In the end, the amount actually handled is stored
in size.
iov_iter_copy_from_user_atomic() and iov_iter_alignment() converted
to it.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2014-11-28 01:51:41 +07:00
|
|
|
iterate_all_kinds(i, bytes, v,
|
2017-06-30 09:25:14 +07:00
|
|
|
copyin((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
|
iov_iter.c: macros for iterating over iov_iter
iterate_all_kinds(iter, size, ident, step_iovec, step_bvec)
iterates through the ranges covered by iter (up to size bytes total),
repeating step_iovec or step_bvec for each of those. ident is
declared in expansion of that thing, either as struct iovec or
struct bvec, and it contains the range we are currently looking
at. step_bvec should be a void expression, step_iovec - a size_t
one, with non-zero meaning "stop here, that many bytes from this
range left". In the end, the amount actually handled is stored
in size.
iov_iter_copy_from_user_atomic() and iov_iter_alignment() converted
to it.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2014-11-28 01:51:41 +07:00
|
|
|
memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
|
2014-11-28 02:48:42 +07:00
|
|
|
v.bv_offset, v.bv_len),
|
|
|
|
memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
|
iov_iter.c: macros for iterating over iov_iter
iterate_all_kinds(iter, size, ident, step_iovec, step_bvec)
iterates through the ranges covered by iter (up to size bytes total),
repeating step_iovec or step_bvec for each of those. ident is
declared in expansion of that thing, either as struct iovec or
struct bvec, and it contains the range we are currently looking
at. step_bvec should be a void expression, step_iovec - a size_t
one, with non-zero meaning "stop here, that many bytes from this
range left". In the end, the amount actually handled is stored
in size.
iov_iter_copy_from_user_atomic() and iov_iter_alignment() converted
to it.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2014-11-28 01:51:41 +07:00
|
|
|
)
|
|
|
|
kunmap_atomic(kaddr);
|
|
|
|
return bytes;
|
2014-04-05 10:12:29 +07:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
|
|
|
|
|
2017-01-15 07:33:08 +07:00
|
|
|
static inline void pipe_truncate(struct iov_iter *i)
|
|
|
|
{
|
|
|
|
struct pipe_inode_info *pipe = i->pipe;
|
|
|
|
if (pipe->nrbufs) {
|
|
|
|
size_t off = i->iov_offset;
|
|
|
|
int idx = i->idx;
|
|
|
|
int nrbufs = (idx - pipe->curbuf) & (pipe->buffers - 1);
|
|
|
|
if (off) {
|
|
|
|
pipe->bufs[idx].len = off - pipe->bufs[idx].offset;
|
|
|
|
idx = next_idx(idx, pipe);
|
|
|
|
nrbufs++;
|
|
|
|
}
|
|
|
|
while (pipe->nrbufs > nrbufs) {
|
|
|
|
pipe_buf_release(pipe, &pipe->bufs[idx]);
|
|
|
|
idx = next_idx(idx, pipe);
|
|
|
|
pipe->nrbufs--;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-09-23 03:33:12 +07:00
|
|
|
static void pipe_advance(struct iov_iter *i, size_t size)
|
|
|
|
{
|
|
|
|
struct pipe_inode_info *pipe = i->pipe;
|
|
|
|
if (unlikely(i->count < size))
|
|
|
|
size = i->count;
|
|
|
|
if (size) {
|
2017-01-15 07:33:08 +07:00
|
|
|
struct pipe_buffer *buf;
|
|
|
|
size_t off = i->iov_offset, left = size;
|
|
|
|
int idx = i->idx;
|
2016-09-23 03:33:12 +07:00
|
|
|
if (off) /* make it relative to the beginning of buffer */
|
2017-01-15 07:33:08 +07:00
|
|
|
left += off - pipe->bufs[idx].offset;
|
2016-09-23 03:33:12 +07:00
|
|
|
while (1) {
|
|
|
|
buf = &pipe->bufs[idx];
|
2017-01-15 07:33:08 +07:00
|
|
|
if (left <= buf->len)
|
2016-09-23 03:33:12 +07:00
|
|
|
break;
|
2017-01-15 07:33:08 +07:00
|
|
|
left -= buf->len;
|
2016-09-23 03:33:12 +07:00
|
|
|
idx = next_idx(idx, pipe);
|
|
|
|
}
|
|
|
|
i->idx = idx;
|
2017-01-15 07:33:08 +07:00
|
|
|
i->iov_offset = buf->offset + left;
|
2016-09-23 03:33:12 +07:00
|
|
|
}
|
2017-01-15 07:33:08 +07:00
|
|
|
i->count -= size;
|
|
|
|
/* ... and discard everything past that point */
|
|
|
|
pipe_truncate(i);
|
2016-09-23 03:33:12 +07:00
|
|
|
}
|
|
|
|
|
2014-04-05 10:12:29 +07:00
|
|
|
void iov_iter_advance(struct iov_iter *i, size_t size)
|
|
|
|
{
|
2016-09-23 03:33:12 +07:00
|
|
|
if (unlikely(i->type & ITER_PIPE)) {
|
|
|
|
pipe_advance(i, size);
|
|
|
|
return;
|
|
|
|
}
|
2014-11-28 02:48:42 +07:00
|
|
|
iterate_and_advance(i, size, v, 0, 0, 0)
|
2014-04-05 10:12:29 +07:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_advance);
|
|
|
|
|
2017-02-18 06:42:24 +07:00
|
|
|
void iov_iter_revert(struct iov_iter *i, size_t unroll)
|
|
|
|
{
|
|
|
|
if (!unroll)
|
|
|
|
return;
|
2017-05-09 00:54:47 +07:00
|
|
|
if (WARN_ON(unroll > MAX_RW_COUNT))
|
|
|
|
return;
|
2017-02-18 06:42:24 +07:00
|
|
|
i->count += unroll;
|
|
|
|
if (unlikely(i->type & ITER_PIPE)) {
|
|
|
|
struct pipe_inode_info *pipe = i->pipe;
|
|
|
|
int idx = i->idx;
|
|
|
|
size_t off = i->iov_offset;
|
|
|
|
while (1) {
|
|
|
|
size_t n = off - pipe->bufs[idx].offset;
|
|
|
|
if (unroll < n) {
|
2017-04-30 03:42:30 +07:00
|
|
|
off -= unroll;
|
2017-02-18 06:42:24 +07:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
unroll -= n;
|
|
|
|
if (!unroll && idx == i->start_idx) {
|
|
|
|
off = 0;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (!idx--)
|
|
|
|
idx = pipe->buffers - 1;
|
|
|
|
off = pipe->bufs[idx].offset + pipe->bufs[idx].len;
|
|
|
|
}
|
|
|
|
i->iov_offset = off;
|
|
|
|
i->idx = idx;
|
|
|
|
pipe_truncate(i);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
if (unroll <= i->iov_offset) {
|
|
|
|
i->iov_offset -= unroll;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
unroll -= i->iov_offset;
|
|
|
|
if (i->type & ITER_BVEC) {
|
|
|
|
const struct bio_vec *bvec = i->bvec;
|
|
|
|
while (1) {
|
|
|
|
size_t n = (--bvec)->bv_len;
|
|
|
|
i->nr_segs++;
|
|
|
|
if (unroll <= n) {
|
|
|
|
i->bvec = bvec;
|
|
|
|
i->iov_offset = n - unroll;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
unroll -= n;
|
|
|
|
}
|
|
|
|
} else { /* same logics for iovec and kvec */
|
|
|
|
const struct iovec *iov = i->iov;
|
|
|
|
while (1) {
|
|
|
|
size_t n = (--iov)->iov_len;
|
|
|
|
i->nr_segs++;
|
|
|
|
if (unroll <= n) {
|
|
|
|
i->iov = iov;
|
|
|
|
i->iov_offset = n - unroll;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
unroll -= n;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_revert);
|
|
|
|
|
2014-04-05 10:12:29 +07:00
|
|
|
/*
|
|
|
|
* Return the count of just the current iov_iter segment.
|
|
|
|
*/
|
|
|
|
size_t iov_iter_single_seg_count(const struct iov_iter *i)
|
|
|
|
{
|
2016-09-23 03:33:12 +07:00
|
|
|
if (unlikely(i->type & ITER_PIPE))
|
|
|
|
return i->count; // it is a silly place, anyway
|
2014-04-05 10:12:29 +07:00
|
|
|
if (i->nr_segs == 1)
|
|
|
|
return i->count;
|
|
|
|
else if (i->type & ITER_BVEC)
|
|
|
|
return min(i->count, i->bvec->bv_len - i->iov_offset);
|
2014-11-13 16:15:23 +07:00
|
|
|
else
|
|
|
|
return min(i->count, i->iov->iov_len - i->iov_offset);
|
2014-04-05 10:12:29 +07:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_single_seg_count);
|
|
|
|
|
2014-11-25 02:46:11 +07:00
|
|
|
void iov_iter_kvec(struct iov_iter *i, int direction,
|
2015-01-23 13:08:07 +07:00
|
|
|
const struct kvec *kvec, unsigned long nr_segs,
|
2014-11-25 02:46:11 +07:00
|
|
|
size_t count)
|
|
|
|
{
|
|
|
|
BUG_ON(!(direction & ITER_KVEC));
|
|
|
|
i->type = direction;
|
2015-01-23 13:08:07 +07:00
|
|
|
i->kvec = kvec;
|
2014-11-25 02:46:11 +07:00
|
|
|
i->nr_segs = nr_segs;
|
|
|
|
i->iov_offset = 0;
|
|
|
|
i->count = count;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_kvec);
|
|
|
|
|
2015-01-23 13:08:07 +07:00
|
|
|
void iov_iter_bvec(struct iov_iter *i, int direction,
|
|
|
|
const struct bio_vec *bvec, unsigned long nr_segs,
|
|
|
|
size_t count)
|
|
|
|
{
|
|
|
|
BUG_ON(!(direction & ITER_BVEC));
|
|
|
|
i->type = direction;
|
|
|
|
i->bvec = bvec;
|
|
|
|
i->nr_segs = nr_segs;
|
|
|
|
i->iov_offset = 0;
|
|
|
|
i->count = count;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_bvec);
|
|
|
|
|
2016-09-23 03:33:12 +07:00
|
|
|
void iov_iter_pipe(struct iov_iter *i, int direction,
|
|
|
|
struct pipe_inode_info *pipe,
|
|
|
|
size_t count)
|
|
|
|
{
|
|
|
|
BUG_ON(direction != ITER_PIPE);
|
2017-01-15 07:33:08 +07:00
|
|
|
WARN_ON(pipe->nrbufs == pipe->buffers);
|
2016-09-23 03:33:12 +07:00
|
|
|
i->type = direction;
|
|
|
|
i->pipe = pipe;
|
|
|
|
i->idx = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
|
|
|
|
i->iov_offset = 0;
|
|
|
|
i->count = count;
|
2017-02-18 06:42:24 +07:00
|
|
|
i->start_idx = i->idx;
|
2016-09-23 03:33:12 +07:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_pipe);
|
|
|
|
|
2014-04-05 10:12:29 +07:00
|
|
|
unsigned long iov_iter_alignment(const struct iov_iter *i)
|
|
|
|
{
|
iov_iter.c: macros for iterating over iov_iter
iterate_all_kinds(iter, size, ident, step_iovec, step_bvec)
iterates through the ranges covered by iter (up to size bytes total),
repeating step_iovec or step_bvec for each of those. ident is
declared in expansion of that thing, either as struct iovec or
struct bvec, and it contains the range we are currently looking
at. step_bvec should be a void expression, step_iovec - a size_t
one, with non-zero meaning "stop here, that many bytes from this
range left". In the end, the amount actually handled is stored
in size.
iov_iter_copy_from_user_atomic() and iov_iter_alignment() converted
to it.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2014-11-28 01:51:41 +07:00
|
|
|
unsigned long res = 0;
|
|
|
|
size_t size = i->count;
|
|
|
|
|
2016-09-23 03:33:12 +07:00
|
|
|
if (unlikely(i->type & ITER_PIPE)) {
|
2016-12-22 09:55:02 +07:00
|
|
|
if (size && i->iov_offset && allocated(&i->pipe->bufs[i->idx]))
|
2016-09-23 03:33:12 +07:00
|
|
|
return size | i->iov_offset;
|
|
|
|
return size;
|
|
|
|
}
|
iov_iter.c: macros for iterating over iov_iter
iterate_all_kinds(iter, size, ident, step_iovec, step_bvec)
iterates through the ranges covered by iter (up to size bytes total),
repeating step_iovec or step_bvec for each of those. ident is
declared in expansion of that thing, either as struct iovec or
struct bvec, and it contains the range we are currently looking
at. step_bvec should be a void expression, step_iovec - a size_t
one, with non-zero meaning "stop here, that many bytes from this
range left". In the end, the amount actually handled is stored
in size.
iov_iter_copy_from_user_atomic() and iov_iter_alignment() converted
to it.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2014-11-28 01:51:41 +07:00
|
|
|
iterate_all_kinds(i, size, v,
|
|
|
|
(res |= (unsigned long)v.iov_base | v.iov_len, 0),
|
2014-11-28 02:48:42 +07:00
|
|
|
res |= v.bv_offset | v.bv_len,
|
|
|
|
res |= (unsigned long)v.iov_base | v.iov_len
|
iov_iter.c: macros for iterating over iov_iter
iterate_all_kinds(iter, size, ident, step_iovec, step_bvec)
iterates through the ranges covered by iter (up to size bytes total),
repeating step_iovec or step_bvec for each of those. ident is
declared in expansion of that thing, either as struct iovec or
struct bvec, and it contains the range we are currently looking
at. step_bvec should be a void expression, step_iovec - a size_t
one, with non-zero meaning "stop here, that many bytes from this
range left". In the end, the amount actually handled is stored
in size.
iov_iter_copy_from_user_atomic() and iov_iter_alignment() converted
to it.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2014-11-28 01:51:41 +07:00
|
|
|
)
|
|
|
|
return res;
|
2014-04-05 10:12:29 +07:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_alignment);
|
|
|
|
|
2016-04-09 06:05:19 +07:00
|
|
|
unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
|
|
|
|
{
|
2016-12-22 09:55:02 +07:00
|
|
|
unsigned long res = 0;
|
2016-04-09 06:05:19 +07:00
|
|
|
size_t size = i->count;
|
|
|
|
|
2016-09-23 03:33:12 +07:00
|
|
|
if (unlikely(i->type & ITER_PIPE)) {
|
|
|
|
WARN_ON(1);
|
|
|
|
return ~0U;
|
|
|
|
}
|
|
|
|
|
2016-04-09 06:05:19 +07:00
|
|
|
iterate_all_kinds(i, size, v,
|
|
|
|
(res |= (!res ? 0 : (unsigned long)v.iov_base) |
|
|
|
|
(size != v.iov_len ? size : 0), 0),
|
|
|
|
(res |= (!res ? 0 : (unsigned long)v.bv_offset) |
|
|
|
|
(size != v.bv_len ? size : 0)),
|
|
|
|
(res |= (!res ? 0 : (unsigned long)v.iov_base) |
|
|
|
|
(size != v.iov_len ? size : 0))
|
|
|
|
);
|
2016-12-22 09:55:02 +07:00
|
|
|
return res;
|
2016-04-09 06:05:19 +07:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_gap_alignment);
|
|
|
|
|
2016-09-23 03:33:12 +07:00
|
|
|
static inline size_t __pipe_get_pages(struct iov_iter *i,
|
|
|
|
size_t maxsize,
|
|
|
|
struct page **pages,
|
|
|
|
int idx,
|
|
|
|
size_t *start)
|
|
|
|
{
|
|
|
|
struct pipe_inode_info *pipe = i->pipe;
|
2016-10-12 00:21:14 +07:00
|
|
|
ssize_t n = push_pipe(i, maxsize, &idx, start);
|
2016-09-23 03:33:12 +07:00
|
|
|
if (!n)
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
maxsize = n;
|
|
|
|
n += *start;
|
2016-10-12 00:21:14 +07:00
|
|
|
while (n > 0) {
|
2016-09-23 03:33:12 +07:00
|
|
|
get_page(*pages++ = pipe->bufs[idx].page);
|
|
|
|
idx = next_idx(idx, pipe);
|
|
|
|
n -= PAGE_SIZE;
|
|
|
|
}
|
|
|
|
|
|
|
|
return maxsize;
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t pipe_get_pages(struct iov_iter *i,
|
|
|
|
struct page **pages, size_t maxsize, unsigned maxpages,
|
|
|
|
size_t *start)
|
|
|
|
{
|
|
|
|
unsigned npages;
|
|
|
|
size_t capacity;
|
|
|
|
int idx;
|
|
|
|
|
2016-12-22 09:55:02 +07:00
|
|
|
if (!maxsize)
|
|
|
|
return 0;
|
|
|
|
|
2016-09-23 03:33:12 +07:00
|
|
|
if (!sanity(i))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
data_start(i, &idx, start);
|
|
|
|
/* some of this one + all after this one */
|
|
|
|
npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1;
|
|
|
|
capacity = min(npages,maxpages) * PAGE_SIZE - *start;
|
|
|
|
|
|
|
|
return __pipe_get_pages(i, min(maxsize, capacity), pages, idx, start);
|
|
|
|
}
|
|
|
|
|
2014-04-05 10:12:29 +07:00
|
|
|
ssize_t iov_iter_get_pages(struct iov_iter *i,
|
2014-09-24 22:09:11 +07:00
|
|
|
struct page **pages, size_t maxsize, unsigned maxpages,
|
2014-04-05 10:12:29 +07:00
|
|
|
size_t *start)
|
|
|
|
{
|
2014-11-28 02:12:09 +07:00
|
|
|
if (maxsize > i->count)
|
|
|
|
maxsize = i->count;
|
|
|
|
|
2016-09-23 03:33:12 +07:00
|
|
|
if (unlikely(i->type & ITER_PIPE))
|
|
|
|
return pipe_get_pages(i, pages, maxsize, maxpages, start);
|
2014-11-28 02:12:09 +07:00
|
|
|
iterate_all_kinds(i, maxsize, v, ({
|
|
|
|
unsigned long addr = (unsigned long)v.iov_base;
|
|
|
|
size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
|
|
|
|
int n;
|
|
|
|
int res;
|
|
|
|
|
|
|
|
if (len > maxpages * PAGE_SIZE)
|
|
|
|
len = maxpages * PAGE_SIZE;
|
|
|
|
addr &= ~(PAGE_SIZE - 1);
|
|
|
|
n = DIV_ROUND_UP(len, PAGE_SIZE);
|
|
|
|
res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
|
|
|
|
if (unlikely(res < 0))
|
|
|
|
return res;
|
|
|
|
return (res == n ? len : res * PAGE_SIZE) - *start;
|
|
|
|
0;}),({
|
|
|
|
/* can't be more than PAGE_SIZE */
|
|
|
|
*start = v.bv_offset;
|
|
|
|
get_page(*pages = v.bv_page);
|
|
|
|
return v.bv_len;
|
2014-11-28 02:48:42 +07:00
|
|
|
}),({
|
|
|
|
return -EFAULT;
|
2014-11-28 02:12:09 +07:00
|
|
|
})
|
|
|
|
)
|
|
|
|
return 0;
|
2014-04-05 10:12:29 +07:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_get_pages);
|
|
|
|
|
2014-11-28 02:14:31 +07:00
|
|
|
static struct page **get_pages_array(size_t n)
|
|
|
|
{
|
2017-05-09 05:57:27 +07:00
|
|
|
return kvmalloc_array(n, sizeof(struct page *), GFP_KERNEL);
|
2014-11-28 02:14:31 +07:00
|
|
|
}
|
|
|
|
|
2016-09-23 03:33:12 +07:00
|
|
|
static ssize_t pipe_get_pages_alloc(struct iov_iter *i,
|
|
|
|
struct page ***pages, size_t maxsize,
|
|
|
|
size_t *start)
|
|
|
|
{
|
|
|
|
struct page **p;
|
|
|
|
size_t n;
|
|
|
|
int idx;
|
|
|
|
int npages;
|
|
|
|
|
2016-12-22 09:55:02 +07:00
|
|
|
if (!maxsize)
|
|
|
|
return 0;
|
|
|
|
|
2016-09-23 03:33:12 +07:00
|
|
|
if (!sanity(i))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
data_start(i, &idx, start);
|
|
|
|
/* some of this one + all after this one */
|
|
|
|
npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1;
|
|
|
|
n = npages * PAGE_SIZE - *start;
|
|
|
|
if (maxsize > n)
|
|
|
|
maxsize = n;
|
|
|
|
else
|
|
|
|
npages = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE);
|
|
|
|
p = get_pages_array(npages);
|
|
|
|
if (!p)
|
|
|
|
return -ENOMEM;
|
|
|
|
n = __pipe_get_pages(i, maxsize, p, idx, start);
|
|
|
|
if (n > 0)
|
|
|
|
*pages = p;
|
|
|
|
else
|
|
|
|
kvfree(p);
|
|
|
|
return n;
|
|
|
|
}
|
|
|
|
|
2014-04-05 10:12:29 +07:00
|
|
|
ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
|
|
|
|
struct page ***pages, size_t maxsize,
|
|
|
|
size_t *start)
|
|
|
|
{
|
2014-11-28 02:14:31 +07:00
|
|
|
struct page **p;
|
|
|
|
|
|
|
|
if (maxsize > i->count)
|
|
|
|
maxsize = i->count;
|
|
|
|
|
2016-09-23 03:33:12 +07:00
|
|
|
if (unlikely(i->type & ITER_PIPE))
|
|
|
|
return pipe_get_pages_alloc(i, pages, maxsize, start);
|
2014-11-28 02:14:31 +07:00
|
|
|
iterate_all_kinds(i, maxsize, v, ({
|
|
|
|
unsigned long addr = (unsigned long)v.iov_base;
|
|
|
|
size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
|
|
|
|
int n;
|
|
|
|
int res;
|
|
|
|
|
|
|
|
addr &= ~(PAGE_SIZE - 1);
|
|
|
|
n = DIV_ROUND_UP(len, PAGE_SIZE);
|
|
|
|
p = get_pages_array(n);
|
|
|
|
if (!p)
|
|
|
|
return -ENOMEM;
|
|
|
|
res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
|
|
|
|
if (unlikely(res < 0)) {
|
|
|
|
kvfree(p);
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
*pages = p;
|
|
|
|
return (res == n ? len : res * PAGE_SIZE) - *start;
|
|
|
|
0;}),({
|
|
|
|
/* can't be more than PAGE_SIZE */
|
|
|
|
*start = v.bv_offset;
|
|
|
|
*pages = p = get_pages_array(1);
|
|
|
|
if (!p)
|
|
|
|
return -ENOMEM;
|
|
|
|
get_page(*p = v.bv_page);
|
|
|
|
return v.bv_len;
|
2014-11-28 02:48:42 +07:00
|
|
|
}),({
|
|
|
|
return -EFAULT;
|
2014-11-28 02:14:31 +07:00
|
|
|
})
|
|
|
|
)
|
|
|
|
return 0;
|
2014-04-05 10:12:29 +07:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_get_pages_alloc);
|
|
|
|
|
2014-11-24 13:08:00 +07:00
|
|
|
size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
|
|
|
|
struct iov_iter *i)
|
|
|
|
{
|
|
|
|
char *to = addr;
|
|
|
|
__wsum sum, next;
|
|
|
|
size_t off = 0;
|
|
|
|
sum = *csum;
|
2016-09-23 03:33:12 +07:00
|
|
|
if (unlikely(i->type & ITER_PIPE)) {
|
|
|
|
WARN_ON(1);
|
|
|
|
return 0;
|
|
|
|
}
|
2014-11-24 13:08:00 +07:00
|
|
|
iterate_and_advance(i, bytes, v, ({
|
|
|
|
int err = 0;
|
2016-11-02 09:09:04 +07:00
|
|
|
next = csum_and_copy_from_user(v.iov_base,
|
2014-11-24 13:08:00 +07:00
|
|
|
(to += v.iov_len) - v.iov_len,
|
|
|
|
v.iov_len, 0, &err);
|
|
|
|
if (!err) {
|
|
|
|
sum = csum_block_add(sum, next, off);
|
|
|
|
off += v.iov_len;
|
|
|
|
}
|
|
|
|
err ? v.iov_len : 0;
|
|
|
|
}), ({
|
|
|
|
char *p = kmap_atomic(v.bv_page);
|
|
|
|
next = csum_partial_copy_nocheck(p + v.bv_offset,
|
|
|
|
(to += v.bv_len) - v.bv_len,
|
|
|
|
v.bv_len, 0);
|
|
|
|
kunmap_atomic(p);
|
|
|
|
sum = csum_block_add(sum, next, off);
|
|
|
|
off += v.bv_len;
|
|
|
|
}),({
|
|
|
|
next = csum_partial_copy_nocheck(v.iov_base,
|
|
|
|
(to += v.iov_len) - v.iov_len,
|
|
|
|
v.iov_len, 0);
|
|
|
|
sum = csum_block_add(sum, next, off);
|
|
|
|
off += v.iov_len;
|
|
|
|
})
|
|
|
|
)
|
|
|
|
*csum = sum;
|
|
|
|
return bytes;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(csum_and_copy_from_iter);
|
|
|
|
|
2016-11-02 09:09:04 +07:00
|
|
|
bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum,
|
|
|
|
struct iov_iter *i)
|
|
|
|
{
|
|
|
|
char *to = addr;
|
|
|
|
__wsum sum, next;
|
|
|
|
size_t off = 0;
|
|
|
|
sum = *csum;
|
|
|
|
if (unlikely(i->type & ITER_PIPE)) {
|
|
|
|
WARN_ON(1);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
if (unlikely(i->count < bytes))
|
|
|
|
return false;
|
|
|
|
iterate_all_kinds(i, bytes, v, ({
|
|
|
|
int err = 0;
|
|
|
|
next = csum_and_copy_from_user(v.iov_base,
|
|
|
|
(to += v.iov_len) - v.iov_len,
|
|
|
|
v.iov_len, 0, &err);
|
|
|
|
if (err)
|
|
|
|
return false;
|
|
|
|
sum = csum_block_add(sum, next, off);
|
|
|
|
off += v.iov_len;
|
|
|
|
0;
|
|
|
|
}), ({
|
|
|
|
char *p = kmap_atomic(v.bv_page);
|
|
|
|
next = csum_partial_copy_nocheck(p + v.bv_offset,
|
|
|
|
(to += v.bv_len) - v.bv_len,
|
|
|
|
v.bv_len, 0);
|
|
|
|
kunmap_atomic(p);
|
|
|
|
sum = csum_block_add(sum, next, off);
|
|
|
|
off += v.bv_len;
|
|
|
|
}),({
|
|
|
|
next = csum_partial_copy_nocheck(v.iov_base,
|
|
|
|
(to += v.iov_len) - v.iov_len,
|
|
|
|
v.iov_len, 0);
|
|
|
|
sum = csum_block_add(sum, next, off);
|
|
|
|
off += v.iov_len;
|
|
|
|
})
|
|
|
|
)
|
|
|
|
*csum = sum;
|
|
|
|
iov_iter_advance(i, bytes);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(csum_and_copy_from_iter_full);
|
|
|
|
|
2015-12-07 04:49:22 +07:00
|
|
|
size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
|
2014-11-24 13:08:00 +07:00
|
|
|
struct iov_iter *i)
|
|
|
|
{
|
2015-12-07 04:49:22 +07:00
|
|
|
const char *from = addr;
|
2014-11-24 13:08:00 +07:00
|
|
|
__wsum sum, next;
|
|
|
|
size_t off = 0;
|
|
|
|
sum = *csum;
|
2016-09-23 03:33:12 +07:00
|
|
|
if (unlikely(i->type & ITER_PIPE)) {
|
|
|
|
WARN_ON(1); /* for now */
|
|
|
|
return 0;
|
|
|
|
}
|
2014-11-24 13:08:00 +07:00
|
|
|
iterate_and_advance(i, bytes, v, ({
|
|
|
|
int err = 0;
|
|
|
|
next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
|
2016-11-02 09:09:04 +07:00
|
|
|
v.iov_base,
|
2014-11-24 13:08:00 +07:00
|
|
|
v.iov_len, 0, &err);
|
|
|
|
if (!err) {
|
|
|
|
sum = csum_block_add(sum, next, off);
|
|
|
|
off += v.iov_len;
|
|
|
|
}
|
|
|
|
err ? v.iov_len : 0;
|
|
|
|
}), ({
|
|
|
|
char *p = kmap_atomic(v.bv_page);
|
|
|
|
next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len,
|
|
|
|
p + v.bv_offset,
|
|
|
|
v.bv_len, 0);
|
|
|
|
kunmap_atomic(p);
|
|
|
|
sum = csum_block_add(sum, next, off);
|
|
|
|
off += v.bv_len;
|
|
|
|
}),({
|
|
|
|
next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len,
|
|
|
|
v.iov_base,
|
|
|
|
v.iov_len, 0);
|
|
|
|
sum = csum_block_add(sum, next, off);
|
|
|
|
off += v.iov_len;
|
|
|
|
})
|
|
|
|
)
|
|
|
|
*csum = sum;
|
|
|
|
return bytes;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(csum_and_copy_to_iter);
|
|
|
|
|
2014-04-05 10:12:29 +07:00
|
|
|
int iov_iter_npages(const struct iov_iter *i, int maxpages)
|
|
|
|
{
|
2014-11-28 02:09:46 +07:00
|
|
|
size_t size = i->count;
|
|
|
|
int npages = 0;
|
|
|
|
|
|
|
|
if (!size)
|
|
|
|
return 0;
|
|
|
|
|
2016-09-23 03:33:12 +07:00
|
|
|
if (unlikely(i->type & ITER_PIPE)) {
|
|
|
|
struct pipe_inode_info *pipe = i->pipe;
|
|
|
|
size_t off;
|
|
|
|
int idx;
|
|
|
|
|
|
|
|
if (!sanity(i))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
data_start(i, &idx, &off);
|
|
|
|
/* some of this one + all after this one */
|
|
|
|
npages = ((pipe->curbuf - idx - 1) & (pipe->buffers - 1)) + 1;
|
|
|
|
if (npages >= maxpages)
|
|
|
|
return maxpages;
|
|
|
|
} else iterate_all_kinds(i, size, v, ({
|
2014-11-28 02:09:46 +07:00
|
|
|
unsigned long p = (unsigned long)v.iov_base;
|
|
|
|
npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
|
|
|
|
- p / PAGE_SIZE;
|
|
|
|
if (npages >= maxpages)
|
|
|
|
return maxpages;
|
|
|
|
0;}),({
|
|
|
|
npages++;
|
|
|
|
if (npages >= maxpages)
|
|
|
|
return maxpages;
|
2014-11-28 02:48:42 +07:00
|
|
|
}),({
|
|
|
|
unsigned long p = (unsigned long)v.iov_base;
|
|
|
|
npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
|
|
|
|
- p / PAGE_SIZE;
|
|
|
|
if (npages >= maxpages)
|
|
|
|
return maxpages;
|
2014-11-28 02:09:46 +07:00
|
|
|
})
|
|
|
|
)
|
|
|
|
return npages;
|
2014-04-05 10:12:29 +07:00
|
|
|
}
|
2014-03-19 12:16:16 +07:00
|
|
|
EXPORT_SYMBOL(iov_iter_npages);
|
2015-02-01 08:08:47 +07:00
|
|
|
|
|
|
|
const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
|
|
|
|
{
|
|
|
|
*new = *old;
|
2016-09-23 03:33:12 +07:00
|
|
|
if (unlikely(new->type & ITER_PIPE)) {
|
|
|
|
WARN_ON(1);
|
|
|
|
return NULL;
|
|
|
|
}
|
2015-02-01 08:08:47 +07:00
|
|
|
if (new->type & ITER_BVEC)
|
|
|
|
return new->bvec = kmemdup(new->bvec,
|
|
|
|
new->nr_segs * sizeof(struct bio_vec),
|
|
|
|
flags);
|
|
|
|
else
|
|
|
|
/* iovec and kvec have identical layout */
|
|
|
|
return new->iov = kmemdup(new->iov,
|
|
|
|
new->nr_segs * sizeof(struct iovec),
|
|
|
|
flags);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(dup_iter);
|
saner iov_iter initialization primitives
iovec-backed iov_iter instances are assumed to satisfy several properties:
* no more than UIO_MAXIOV elements in iovec array
* total size of all ranges is no more than MAX_RW_COUNT
* all ranges pass access_ok().
The problem is, invariants of data structures should be established in the
primitives creating those data structures, not in the code using those
primitives. And iov_iter_init() violates that principle. For a while we
managed to get away with that, but once the use of iov_iter started to
spread, it didn't take long for shit to hit the fan - missed check in
sys_sendto() had introduced a roothole.
We _do_ have primitives for importing and validating iovecs (both native and
compat ones) and those primitives are almost always followed by shoving the
resulting iovec into iov_iter. Life would be considerably simpler (and safer)
if we combined those primitives with initializing iov_iter.
That gives us two new primitives - import_iovec() and compat_import_iovec().
Calling conventions:
iovec = iov_array;
err = import_iovec(direction, uvec, nr_segs,
ARRAY_SIZE(iov_array), &iovec,
&iter);
imports user vector into kernel space (into iov_array if it fits, allocated
if it doesn't fit or if iovec was NULL), validates it and sets iter up to
refer to it. On success 0 is returned and allocated kernel copy (or NULL
if the array had fit into caller-supplied one) is returned via iovec.
On failure all allocations are undone and -E... is returned. If the total
size of ranges exceeds MAX_RW_COUNT, the excess is silently truncated.
compat_import_iovec() expects uvec to be a pointer to user array of compat_iovec;
otherwise it's identical to import_iovec().
Finally, import_single_range() sets iov_iter backed by single-element iovec
covering a user-supplied range -
err = import_single_range(direction, address, size, iovec, &iter);
does validation and sets iter up. Again, size in excess of MAX_RW_COUNT gets
silently truncated.
Next commits will be switching the things up to use of those and reducing
the amount of iov_iter_init() instances.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2015-03-22 04:45:43 +07:00
|
|
|
|
2016-10-08 16:18:07 +07:00
|
|
|
/**
|
|
|
|
* import_iovec() - Copy an array of &struct iovec from userspace
|
|
|
|
* into the kernel, check that it is valid, and initialize a new
|
|
|
|
* &struct iov_iter iterator to access it.
|
|
|
|
*
|
|
|
|
* @type: One of %READ or %WRITE.
|
|
|
|
* @uvector: Pointer to the userspace array.
|
|
|
|
* @nr_segs: Number of elements in userspace array.
|
|
|
|
* @fast_segs: Number of elements in @iov.
|
|
|
|
* @iov: (input and output parameter) Pointer to pointer to (usually small
|
|
|
|
* on-stack) kernel array.
|
|
|
|
* @i: Pointer to iterator that will be initialized on success.
|
|
|
|
*
|
|
|
|
* If the array pointed to by *@iov is large enough to hold all @nr_segs,
|
|
|
|
* then this function places %NULL in *@iov on return. Otherwise, a new
|
|
|
|
* array will be allocated and the result placed in *@iov. This means that
|
|
|
|
* the caller may call kfree() on *@iov regardless of whether the small
|
|
|
|
* on-stack array was used or not (and regardless of whether this function
|
|
|
|
* returns an error or not).
|
|
|
|
*
|
|
|
|
* Return: 0 on success or negative error code on error.
|
|
|
|
*/
|
saner iov_iter initialization primitives
iovec-backed iov_iter instances are assumed to satisfy several properties:
* no more than UIO_MAXIOV elements in iovec array
* total size of all ranges is no more than MAX_RW_COUNT
* all ranges pass access_ok().
The problem is, invariants of data structures should be established in the
primitives creating those data structures, not in the code using those
primitives. And iov_iter_init() violates that principle. For a while we
managed to get away with that, but once the use of iov_iter started to
spread, it didn't take long for shit to hit the fan - missed check in
sys_sendto() had introduced a roothole.
We _do_ have primitives for importing and validating iovecs (both native and
compat ones) and those primitives are almost always followed by shoving the
resulting iovec into iov_iter. Life would be considerably simpler (and safer)
if we combined those primitives with initializing iov_iter.
That gives us two new primitives - import_iovec() and compat_import_iovec().
Calling conventions:
iovec = iov_array;
err = import_iovec(direction, uvec, nr_segs,
ARRAY_SIZE(iov_array), &iovec,
&iter);
imports user vector into kernel space (into iov_array if it fits, allocated
if it doesn't fit or if iovec was NULL), validates it and sets iter up to
refer to it. On success 0 is returned and allocated kernel copy (or NULL
if the array had fit into caller-supplied one) is returned via iovec.
On failure all allocations are undone and -E... is returned. If the total
size of ranges exceeds MAX_RW_COUNT, the excess is silently truncated.
compat_import_iovec() expects uvec to be a pointer to user array of compat_iovec;
otherwise it's identical to import_iovec().
Finally, import_single_range() sets iov_iter backed by single-element iovec
covering a user-supplied range -
err = import_single_range(direction, address, size, iovec, &iter);
does validation and sets iter up. Again, size in excess of MAX_RW_COUNT gets
silently truncated.
Next commits will be switching the things up to use of those and reducing
the amount of iov_iter_init() instances.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2015-03-22 04:45:43 +07:00
|
|
|
int import_iovec(int type, const struct iovec __user * uvector,
|
|
|
|
unsigned nr_segs, unsigned fast_segs,
|
|
|
|
struct iovec **iov, struct iov_iter *i)
|
|
|
|
{
|
|
|
|
ssize_t n;
|
|
|
|
struct iovec *p;
|
|
|
|
n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
|
|
|
|
*iov, &p);
|
|
|
|
if (n < 0) {
|
|
|
|
if (p != *iov)
|
|
|
|
kfree(p);
|
|
|
|
*iov = NULL;
|
|
|
|
return n;
|
|
|
|
}
|
|
|
|
iov_iter_init(i, type, p, nr_segs, n);
|
|
|
|
*iov = p == *iov ? NULL : p;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(import_iovec);
|
|
|
|
|
|
|
|
#ifdef CONFIG_COMPAT
|
|
|
|
#include <linux/compat.h>
|
|
|
|
|
|
|
|
int compat_import_iovec(int type, const struct compat_iovec __user * uvector,
|
|
|
|
unsigned nr_segs, unsigned fast_segs,
|
|
|
|
struct iovec **iov, struct iov_iter *i)
|
|
|
|
{
|
|
|
|
ssize_t n;
|
|
|
|
struct iovec *p;
|
|
|
|
n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
|
|
|
|
*iov, &p);
|
|
|
|
if (n < 0) {
|
|
|
|
if (p != *iov)
|
|
|
|
kfree(p);
|
|
|
|
*iov = NULL;
|
|
|
|
return n;
|
|
|
|
}
|
|
|
|
iov_iter_init(i, type, p, nr_segs, n);
|
|
|
|
*iov = p == *iov ? NULL : p;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
int import_single_range(int rw, void __user *buf, size_t len,
|
|
|
|
struct iovec *iov, struct iov_iter *i)
|
|
|
|
{
|
|
|
|
if (len > MAX_RW_COUNT)
|
|
|
|
len = MAX_RW_COUNT;
|
|
|
|
if (unlikely(!access_ok(!rw, buf, len)))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
iov->iov_base = buf;
|
|
|
|
iov->iov_len = len;
|
|
|
|
iov_iter_init(i, rw, iov, 1, len);
|
|
|
|
return 0;
|
|
|
|
}
|
2015-12-07 08:38:56 +07:00
|
|
|
EXPORT_SYMBOL(import_single_range);
|