2018-11-29 01:53:33 +07:00
|
|
|
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
|
2013-07-07 21:25:49 +07:00
|
|
|
/*
|
2018-11-29 01:53:33 +07:00
|
|
|
* Copyright (c) 2013-2018, Mellanox Technologies inc. All rights reserved.
|
2013-07-07 21:25:49 +07:00
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/mlx5/qp.h>
|
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <rdma/ib_umem.h>
|
IB/mlx5: add missing padding at end of struct mlx5_ib_create_srq
The i386 ABI disagrees with most other ABIs regarding alignment of
data type larger than 4 bytes: on most ABIs a padding must be added at
end of the structures, while it is not required on i386.
So for most ABIs struct mlx5_ib_create_srq gets implicitly padded to be
aligned on a 8 bytes multiple, while for i386, such padding is not
added.
Tool pahole could be used to find such implicit padding:
$ pahole --anon_include \
--nested_anon_include \
--recursive \
--class_name mlx5_ib_create_srq \
drivers/infiniband/hw/mlx5/mlx5_ib.o
Then, structure layout can be compared between i386 and x86_64:
+++ obj-i386/drivers/infiniband/hw/mlx5/mlx5_ib.o.pahole.txt 2014-03-28 11:43:07.386413682 +0100
--- obj-x86_64/drivers/infiniband/hw/mlx5/mlx5_ib.o.pahole.txt 2014-03-27 13:06:17.788472721 +0100
@@ -69,7 +68,6 @@ struct mlx5_ib_create_srq {
__u64 db_addr; /* 8 8 */
__u32 flags; /* 16 4 */
- /* size: 20, cachelines: 1, members: 3 */
- /* last cacheline: 20 bytes */
+ /* size: 24, cachelines: 1, members: 3 */
+ /* padding: 4 */
+ /* last cacheline: 24 bytes */
};
ABI disagreement will make an x86_64 kernel try to read past
the buffer provided by an i386 binary.
When boundary check will be implemented, the x86_64 kernel will
refuse to read past the i386 userspace provided buffer and the
uverb will fail.
Anyway, if the structure lay in memory on a page boundary and
next page is not mapped, ib_copy_from_udata() will fail and the
uverb will fail.
This patch makes create_srq_user() takes care of the input
data size to handle the case where no padding was provided.
This way, x86_64 kernel will be able to handle struct mlx5_ib_create_srq
as sent by unpatched and patched i386 libmlx5.
Link: http://marc.info/?i=cover.1399309513.git.ydroneaud@opteya.com
Cc: <stable@vger.kernel.org>
Fixes: e126ba97dba9e ("mlx5: Add driver for Mellanox Connect-IB adapter")
Signed-off-by: Yann Droneaud <ydroneaud@opteya.com>
Signed-off-by: Roland Dreier <roland@purestorage.com>
2014-05-06 00:33:22 +07:00
|
|
|
#include <rdma/ib_user_verbs.h>
|
2013-07-07 21:25:49 +07:00
|
|
|
#include "mlx5_ib.h"
|
2018-11-29 01:53:37 +07:00
|
|
|
#include "srq.h"
|
2013-07-07 21:25:49 +07:00
|
|
|
|
|
|
|
static void *get_wqe(struct mlx5_ib_srq *srq, int n)
|
|
|
|
{
|
|
|
|
return mlx5_buf_offset(&srq->buf, n << srq->msrq.wqe_shift);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void mlx5_ib_srq_event(struct mlx5_core_srq *srq, enum mlx5_event type)
|
|
|
|
{
|
|
|
|
struct ib_event event;
|
|
|
|
struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq;
|
|
|
|
|
|
|
|
if (ibsrq->event_handler) {
|
|
|
|
event.device = ibsrq->device;
|
|
|
|
event.element.srq = ibsrq;
|
|
|
|
switch (type) {
|
|
|
|
case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
|
|
|
|
event.event = IB_EVENT_SRQ_LIMIT_REACHED;
|
|
|
|
break;
|
|
|
|
case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
|
|
|
|
event.event = IB_EVENT_SRQ_ERR;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
pr_warn("mlx5_ib: Unexpected event type %d on SRQ %06x\n",
|
|
|
|
type, srq->srqn);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
ibsrq->event_handler(&event, ibsrq->srq_context);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,
|
2016-06-17 19:33:32 +07:00
|
|
|
struct mlx5_srq_attr *in,
|
|
|
|
struct ib_udata *udata, int buf_size)
|
2013-07-07 21:25:49 +07:00
|
|
|
{
|
|
|
|
struct mlx5_ib_dev *dev = to_mdev(pd->device);
|
2016-01-15 00:12:57 +07:00
|
|
|
struct mlx5_ib_create_srq ucmd = {};
|
IB/mlx5: add missing padding at end of struct mlx5_ib_create_srq
The i386 ABI disagrees with most other ABIs regarding alignment of
data type larger than 4 bytes: on most ABIs a padding must be added at
end of the structures, while it is not required on i386.
So for most ABIs struct mlx5_ib_create_srq gets implicitly padded to be
aligned on a 8 bytes multiple, while for i386, such padding is not
added.
Tool pahole could be used to find such implicit padding:
$ pahole --anon_include \
--nested_anon_include \
--recursive \
--class_name mlx5_ib_create_srq \
drivers/infiniband/hw/mlx5/mlx5_ib.o
Then, structure layout can be compared between i386 and x86_64:
+++ obj-i386/drivers/infiniband/hw/mlx5/mlx5_ib.o.pahole.txt 2014-03-28 11:43:07.386413682 +0100
--- obj-x86_64/drivers/infiniband/hw/mlx5/mlx5_ib.o.pahole.txt 2014-03-27 13:06:17.788472721 +0100
@@ -69,7 +68,6 @@ struct mlx5_ib_create_srq {
__u64 db_addr; /* 8 8 */
__u32 flags; /* 16 4 */
- /* size: 20, cachelines: 1, members: 3 */
- /* last cacheline: 20 bytes */
+ /* size: 24, cachelines: 1, members: 3 */
+ /* padding: 4 */
+ /* last cacheline: 24 bytes */
};
ABI disagreement will make an x86_64 kernel try to read past
the buffer provided by an i386 binary.
When boundary check will be implemented, the x86_64 kernel will
refuse to read past the i386 userspace provided buffer and the
uverb will fail.
Anyway, if the structure lay in memory on a page boundary and
next page is not mapped, ib_copy_from_udata() will fail and the
uverb will fail.
This patch makes create_srq_user() takes care of the input
data size to handle the case where no padding was provided.
This way, x86_64 kernel will be able to handle struct mlx5_ib_create_srq
as sent by unpatched and patched i386 libmlx5.
Link: http://marc.info/?i=cover.1399309513.git.ydroneaud@opteya.com
Cc: <stable@vger.kernel.org>
Fixes: e126ba97dba9e ("mlx5: Add driver for Mellanox Connect-IB adapter")
Signed-off-by: Yann Droneaud <ydroneaud@opteya.com>
Signed-off-by: Roland Dreier <roland@purestorage.com>
2014-05-06 00:33:22 +07:00
|
|
|
size_t ucmdlen;
|
2013-07-07 21:25:49 +07:00
|
|
|
int err;
|
|
|
|
int npages;
|
|
|
|
int page_shift;
|
|
|
|
int ncont;
|
|
|
|
u32 offset;
|
2016-01-15 00:12:57 +07:00
|
|
|
u32 uidx = MLX5_IB_DEFAULT_UIDX;
|
2013-07-07 21:25:49 +07:00
|
|
|
|
2016-02-14 23:35:52 +07:00
|
|
|
ucmdlen = min(udata->inlen, sizeof(ucmd));
|
IB/mlx5: add missing padding at end of struct mlx5_ib_create_srq
The i386 ABI disagrees with most other ABIs regarding alignment of
data type larger than 4 bytes: on most ABIs a padding must be added at
end of the structures, while it is not required on i386.
So for most ABIs struct mlx5_ib_create_srq gets implicitly padded to be
aligned on a 8 bytes multiple, while for i386, such padding is not
added.
Tool pahole could be used to find such implicit padding:
$ pahole --anon_include \
--nested_anon_include \
--recursive \
--class_name mlx5_ib_create_srq \
drivers/infiniband/hw/mlx5/mlx5_ib.o
Then, structure layout can be compared between i386 and x86_64:
+++ obj-i386/drivers/infiniband/hw/mlx5/mlx5_ib.o.pahole.txt 2014-03-28 11:43:07.386413682 +0100
--- obj-x86_64/drivers/infiniband/hw/mlx5/mlx5_ib.o.pahole.txt 2014-03-27 13:06:17.788472721 +0100
@@ -69,7 +68,6 @@ struct mlx5_ib_create_srq {
__u64 db_addr; /* 8 8 */
__u32 flags; /* 16 4 */
- /* size: 20, cachelines: 1, members: 3 */
- /* last cacheline: 20 bytes */
+ /* size: 24, cachelines: 1, members: 3 */
+ /* padding: 4 */
+ /* last cacheline: 24 bytes */
};
ABI disagreement will make an x86_64 kernel try to read past
the buffer provided by an i386 binary.
When boundary check will be implemented, the x86_64 kernel will
refuse to read past the i386 userspace provided buffer and the
uverb will fail.
Anyway, if the structure lay in memory on a page boundary and
next page is not mapped, ib_copy_from_udata() will fail and the
uverb will fail.
This patch makes create_srq_user() takes care of the input
data size to handle the case where no padding was provided.
This way, x86_64 kernel will be able to handle struct mlx5_ib_create_srq
as sent by unpatched and patched i386 libmlx5.
Link: http://marc.info/?i=cover.1399309513.git.ydroneaud@opteya.com
Cc: <stable@vger.kernel.org>
Fixes: e126ba97dba9e ("mlx5: Add driver for Mellanox Connect-IB adapter")
Signed-off-by: Yann Droneaud <ydroneaud@opteya.com>
Signed-off-by: Roland Dreier <roland@purestorage.com>
2014-05-06 00:33:22 +07:00
|
|
|
|
|
|
|
if (ib_copy_from_udata(&ucmd, udata, ucmdlen)) {
|
2013-07-07 21:25:49 +07:00
|
|
|
mlx5_ib_dbg(dev, "failed copy udata\n");
|
|
|
|
return -EFAULT;
|
|
|
|
}
|
IB/mlx5: add missing padding at end of struct mlx5_ib_create_srq
The i386 ABI disagrees with most other ABIs regarding alignment of
data type larger than 4 bytes: on most ABIs a padding must be added at
end of the structures, while it is not required on i386.
So for most ABIs struct mlx5_ib_create_srq gets implicitly padded to be
aligned on a 8 bytes multiple, while for i386, such padding is not
added.
Tool pahole could be used to find such implicit padding:
$ pahole --anon_include \
--nested_anon_include \
--recursive \
--class_name mlx5_ib_create_srq \
drivers/infiniband/hw/mlx5/mlx5_ib.o
Then, structure layout can be compared between i386 and x86_64:
+++ obj-i386/drivers/infiniband/hw/mlx5/mlx5_ib.o.pahole.txt 2014-03-28 11:43:07.386413682 +0100
--- obj-x86_64/drivers/infiniband/hw/mlx5/mlx5_ib.o.pahole.txt 2014-03-27 13:06:17.788472721 +0100
@@ -69,7 +68,6 @@ struct mlx5_ib_create_srq {
__u64 db_addr; /* 8 8 */
__u32 flags; /* 16 4 */
- /* size: 20, cachelines: 1, members: 3 */
- /* last cacheline: 20 bytes */
+ /* size: 24, cachelines: 1, members: 3 */
+ /* padding: 4 */
+ /* last cacheline: 24 bytes */
};
ABI disagreement will make an x86_64 kernel try to read past
the buffer provided by an i386 binary.
When boundary check will be implemented, the x86_64 kernel will
refuse to read past the i386 userspace provided buffer and the
uverb will fail.
Anyway, if the structure lay in memory on a page boundary and
next page is not mapped, ib_copy_from_udata() will fail and the
uverb will fail.
This patch makes create_srq_user() takes care of the input
data size to handle the case where no padding was provided.
This way, x86_64 kernel will be able to handle struct mlx5_ib_create_srq
as sent by unpatched and patched i386 libmlx5.
Link: http://marc.info/?i=cover.1399309513.git.ydroneaud@opteya.com
Cc: <stable@vger.kernel.org>
Fixes: e126ba97dba9e ("mlx5: Add driver for Mellanox Connect-IB adapter")
Signed-off-by: Yann Droneaud <ydroneaud@opteya.com>
Signed-off-by: Roland Dreier <roland@purestorage.com>
2014-05-06 00:33:22 +07:00
|
|
|
|
2016-01-15 00:12:57 +07:00
|
|
|
if (ucmd.reserved0 || ucmd.reserved1)
|
IB/mlx5: add missing padding at end of struct mlx5_ib_create_srq
The i386 ABI disagrees with most other ABIs regarding alignment of
data type larger than 4 bytes: on most ABIs a padding must be added at
end of the structures, while it is not required on i386.
So for most ABIs struct mlx5_ib_create_srq gets implicitly padded to be
aligned on a 8 bytes multiple, while for i386, such padding is not
added.
Tool pahole could be used to find such implicit padding:
$ pahole --anon_include \
--nested_anon_include \
--recursive \
--class_name mlx5_ib_create_srq \
drivers/infiniband/hw/mlx5/mlx5_ib.o
Then, structure layout can be compared between i386 and x86_64:
+++ obj-i386/drivers/infiniband/hw/mlx5/mlx5_ib.o.pahole.txt 2014-03-28 11:43:07.386413682 +0100
--- obj-x86_64/drivers/infiniband/hw/mlx5/mlx5_ib.o.pahole.txt 2014-03-27 13:06:17.788472721 +0100
@@ -69,7 +68,6 @@ struct mlx5_ib_create_srq {
__u64 db_addr; /* 8 8 */
__u32 flags; /* 16 4 */
- /* size: 20, cachelines: 1, members: 3 */
- /* last cacheline: 20 bytes */
+ /* size: 24, cachelines: 1, members: 3 */
+ /* padding: 4 */
+ /* last cacheline: 24 bytes */
};
ABI disagreement will make an x86_64 kernel try to read past
the buffer provided by an i386 binary.
When boundary check will be implemented, the x86_64 kernel will
refuse to read past the i386 userspace provided buffer and the
uverb will fail.
Anyway, if the structure lay in memory on a page boundary and
next page is not mapped, ib_copy_from_udata() will fail and the
uverb will fail.
This patch makes create_srq_user() takes care of the input
data size to handle the case where no padding was provided.
This way, x86_64 kernel will be able to handle struct mlx5_ib_create_srq
as sent by unpatched and patched i386 libmlx5.
Link: http://marc.info/?i=cover.1399309513.git.ydroneaud@opteya.com
Cc: <stable@vger.kernel.org>
Fixes: e126ba97dba9e ("mlx5: Add driver for Mellanox Connect-IB adapter")
Signed-off-by: Yann Droneaud <ydroneaud@opteya.com>
Signed-off-by: Roland Dreier <roland@purestorage.com>
2014-05-06 00:33:22 +07:00
|
|
|
return -EINVAL;
|
|
|
|
|
2016-02-14 23:35:52 +07:00
|
|
|
if (udata->inlen > sizeof(ucmd) &&
|
2016-01-15 00:12:57 +07:00
|
|
|
!ib_is_udata_cleared(udata, sizeof(ucmd),
|
2016-02-14 23:35:52 +07:00
|
|
|
udata->inlen - sizeof(ucmd)))
|
2016-01-15 00:12:57 +07:00
|
|
|
return -EINVAL;
|
|
|
|
|
2017-08-17 19:52:11 +07:00
|
|
|
if (in->type != IB_SRQT_BASIC) {
|
2016-02-14 23:35:51 +07:00
|
|
|
err = get_srq_user_index(to_mucontext(pd->uobject->context),
|
|
|
|
&ucmd, udata->inlen, &uidx);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
}
|
2016-01-15 00:12:57 +07:00
|
|
|
|
2013-07-07 21:25:49 +07:00
|
|
|
srq->wq_sig = !!(ucmd.flags & MLX5_SRQ_FLAG_SIGNATURE);
|
|
|
|
|
|
|
|
srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr, buf_size,
|
|
|
|
0, 0);
|
|
|
|
if (IS_ERR(srq->umem)) {
|
|
|
|
mlx5_ib_dbg(dev, "failed umem get, size %d\n", buf_size);
|
|
|
|
err = PTR_ERR(srq->umem);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2016-10-27 20:36:47 +07:00
|
|
|
mlx5_ib_cont_pages(srq->umem, ucmd.buf_addr, 0, &npages,
|
2013-07-07 21:25:49 +07:00
|
|
|
&page_shift, &ncont, NULL);
|
|
|
|
err = mlx5_ib_get_buf_offset(ucmd.buf_addr, page_shift,
|
|
|
|
&offset);
|
|
|
|
if (err) {
|
|
|
|
mlx5_ib_warn(dev, "bad offset\n");
|
|
|
|
goto err_umem;
|
|
|
|
}
|
|
|
|
|
treewide: kvzalloc() -> kvcalloc()
The kvzalloc() function has a 2-factor argument form, kvcalloc(). This
patch replaces cases of:
kvzalloc(a * b, gfp)
with:
kvcalloc(a * b, gfp)
as well as handling cases of:
kvzalloc(a * b * c, gfp)
with:
kvzalloc(array3_size(a, b, c), gfp)
as it's slightly less ugly than:
kvcalloc(array_size(a, b), c, gfp)
This does, however, attempt to ignore constant size factors like:
kvzalloc(4 * 1024, gfp)
though any constants defined via macros get caught up in the conversion.
Any factors with a sizeof() of "unsigned char", "char", and "u8" were
dropped, since they're redundant.
The Coccinelle script used for this was:
// Fix redundant parens around sizeof().
@@
type TYPE;
expression THING, E;
@@
(
kvzalloc(
- (sizeof(TYPE)) * E
+ sizeof(TYPE) * E
, ...)
|
kvzalloc(
- (sizeof(THING)) * E
+ sizeof(THING) * E
, ...)
)
// Drop single-byte sizes and redundant parens.
@@
expression COUNT;
typedef u8;
typedef __u8;
@@
(
kvzalloc(
- sizeof(u8) * (COUNT)
+ COUNT
, ...)
|
kvzalloc(
- sizeof(__u8) * (COUNT)
+ COUNT
, ...)
|
kvzalloc(
- sizeof(char) * (COUNT)
+ COUNT
, ...)
|
kvzalloc(
- sizeof(unsigned char) * (COUNT)
+ COUNT
, ...)
|
kvzalloc(
- sizeof(u8) * COUNT
+ COUNT
, ...)
|
kvzalloc(
- sizeof(__u8) * COUNT
+ COUNT
, ...)
|
kvzalloc(
- sizeof(char) * COUNT
+ COUNT
, ...)
|
kvzalloc(
- sizeof(unsigned char) * COUNT
+ COUNT
, ...)
)
// 2-factor product with sizeof(type/expression) and identifier or constant.
@@
type TYPE;
expression THING;
identifier COUNT_ID;
constant COUNT_CONST;
@@
(
- kvzalloc
+ kvcalloc
(
- sizeof(TYPE) * (COUNT_ID)
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(TYPE) * COUNT_ID
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(TYPE) * (COUNT_CONST)
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(TYPE) * COUNT_CONST
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(THING) * (COUNT_ID)
+ COUNT_ID, sizeof(THING)
, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(THING) * COUNT_ID
+ COUNT_ID, sizeof(THING)
, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(THING) * (COUNT_CONST)
+ COUNT_CONST, sizeof(THING)
, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(THING) * COUNT_CONST
+ COUNT_CONST, sizeof(THING)
, ...)
)
// 2-factor product, only identifiers.
@@
identifier SIZE, COUNT;
@@
- kvzalloc
+ kvcalloc
(
- SIZE * COUNT
+ COUNT, SIZE
, ...)
// 3-factor product with 1 sizeof(type) or sizeof(expression), with
// redundant parens removed.
@@
expression THING;
identifier STRIDE, COUNT;
type TYPE;
@@
(
kvzalloc(
- sizeof(TYPE) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kvzalloc(
- sizeof(TYPE) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kvzalloc(
- sizeof(TYPE) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kvzalloc(
- sizeof(TYPE) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kvzalloc(
- sizeof(THING) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kvzalloc(
- sizeof(THING) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kvzalloc(
- sizeof(THING) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kvzalloc(
- sizeof(THING) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
)
// 3-factor product with 2 sizeof(variable), with redundant parens removed.
@@
expression THING1, THING2;
identifier COUNT;
type TYPE1, TYPE2;
@@
(
kvzalloc(
- sizeof(TYPE1) * sizeof(TYPE2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kvzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kvzalloc(
- sizeof(THING1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kvzalloc(
- sizeof(THING1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kvzalloc(
- sizeof(TYPE1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
|
kvzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
)
// 3-factor product, only identifiers, with redundant parens removed.
@@
identifier STRIDE, SIZE, COUNT;
@@
(
kvzalloc(
- (COUNT) * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kvzalloc(
- COUNT * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kvzalloc(
- COUNT * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kvzalloc(
- (COUNT) * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kvzalloc(
- COUNT * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kvzalloc(
- (COUNT) * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kvzalloc(
- (COUNT) * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kvzalloc(
- COUNT * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
)
// Any remaining multi-factor products, first at least 3-factor products,
// when they're not all constants...
@@
expression E1, E2, E3;
constant C1, C2, C3;
@@
(
kvzalloc(C1 * C2 * C3, ...)
|
kvzalloc(
- (E1) * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
|
kvzalloc(
- (E1) * (E2) * E3
+ array3_size(E1, E2, E3)
, ...)
|
kvzalloc(
- (E1) * (E2) * (E3)
+ array3_size(E1, E2, E3)
, ...)
|
kvzalloc(
- E1 * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
)
// And then all remaining 2 factors products when they're not all constants,
// keeping sizeof() as the second factor argument.
@@
expression THING, E1, E2;
type TYPE;
constant C1, C2, C3;
@@
(
kvzalloc(sizeof(THING) * C2, ...)
|
kvzalloc(sizeof(TYPE) * C2, ...)
|
kvzalloc(C1 * C2 * C3, ...)
|
kvzalloc(C1 * C2, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(TYPE) * (E2)
+ E2, sizeof(TYPE)
, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(TYPE) * E2
+ E2, sizeof(TYPE)
, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(THING) * (E2)
+ E2, sizeof(THING)
, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(THING) * E2
+ E2, sizeof(THING)
, ...)
|
- kvzalloc
+ kvcalloc
(
- (E1) * E2
+ E1, E2
, ...)
|
- kvzalloc
+ kvcalloc
(
- (E1) * (E2)
+ E1, E2
, ...)
|
- kvzalloc
+ kvcalloc
(
- E1 * E2
+ E1, E2
, ...)
)
Signed-off-by: Kees Cook <keescook@chromium.org>
2018-06-13 04:04:48 +07:00
|
|
|
in->pas = kvcalloc(ncont, sizeof(*in->pas), GFP_KERNEL);
|
2016-06-17 19:33:32 +07:00
|
|
|
if (!in->pas) {
|
2013-07-07 21:25:49 +07:00
|
|
|
err = -ENOMEM;
|
|
|
|
goto err_umem;
|
|
|
|
}
|
|
|
|
|
2016-06-17 19:33:32 +07:00
|
|
|
mlx5_ib_populate_pas(dev, srq->umem, page_shift, in->pas, 0);
|
2013-07-07 21:25:49 +07:00
|
|
|
|
|
|
|
err = mlx5_ib_db_map_user(to_mucontext(pd->uobject->context),
|
|
|
|
ucmd.db_addr, &srq->db);
|
|
|
|
if (err) {
|
|
|
|
mlx5_ib_dbg(dev, "map doorbell failed\n");
|
|
|
|
goto err_in;
|
|
|
|
}
|
|
|
|
|
2016-06-17 19:33:32 +07:00
|
|
|
in->log_page_size = page_shift - MLX5_ADAPTER_PAGE_SHIFT;
|
|
|
|
in->page_offset = offset;
|
2018-09-21 01:39:23 +07:00
|
|
|
in->uid = to_mpd(pd)->uid;
|
2016-06-17 19:33:32 +07:00
|
|
|
if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 &&
|
2017-08-17 19:52:11 +07:00
|
|
|
in->type != IB_SRQT_BASIC)
|
2016-06-17 19:33:32 +07:00
|
|
|
in->user_index = uidx;
|
2016-01-15 00:12:57 +07:00
|
|
|
|
2013-07-07 21:25:49 +07:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
err_in:
|
2016-06-17 19:33:32 +07:00
|
|
|
kvfree(in->pas);
|
2013-07-07 21:25:49 +07:00
|
|
|
|
|
|
|
err_umem:
|
|
|
|
ib_umem_release(srq->umem);
|
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq,
|
2016-06-17 19:33:32 +07:00
|
|
|
struct mlx5_srq_attr *in, int buf_size)
|
2013-07-07 21:25:49 +07:00
|
|
|
{
|
|
|
|
int err;
|
|
|
|
int i;
|
|
|
|
struct mlx5_wqe_srq_next_seg *next;
|
|
|
|
|
2014-07-29 03:30:22 +07:00
|
|
|
err = mlx5_db_alloc(dev->mdev, &srq->db);
|
2013-07-07 21:25:49 +07:00
|
|
|
if (err) {
|
|
|
|
mlx5_ib_warn(dev, "alloc dbell rec failed\n");
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2015-05-29 02:28:38 +07:00
|
|
|
if (mlx5_buf_alloc(dev->mdev, buf_size, &srq->buf)) {
|
2013-07-07 21:25:49 +07:00
|
|
|
mlx5_ib_dbg(dev, "buf alloc failed\n");
|
|
|
|
err = -ENOMEM;
|
|
|
|
goto err_db;
|
|
|
|
}
|
|
|
|
|
|
|
|
srq->head = 0;
|
|
|
|
srq->tail = srq->msrq.max - 1;
|
|
|
|
srq->wqe_ctr = 0;
|
|
|
|
|
|
|
|
for (i = 0; i < srq->msrq.max; i++) {
|
|
|
|
next = get_wqe(srq, i);
|
|
|
|
next->next_wqe_index =
|
|
|
|
cpu_to_be16((i + 1) & (srq->msrq.max - 1));
|
|
|
|
}
|
|
|
|
|
2017-01-18 19:10:30 +07:00
|
|
|
mlx5_ib_dbg(dev, "srq->buf.page_shift = %d\n", srq->buf.page_shift);
|
treewide: kvzalloc() -> kvcalloc()
The kvzalloc() function has a 2-factor argument form, kvcalloc(). This
patch replaces cases of:
kvzalloc(a * b, gfp)
with:
kvcalloc(a * b, gfp)
as well as handling cases of:
kvzalloc(a * b * c, gfp)
with:
kvzalloc(array3_size(a, b, c), gfp)
as it's slightly less ugly than:
kvcalloc(array_size(a, b), c, gfp)
This does, however, attempt to ignore constant size factors like:
kvzalloc(4 * 1024, gfp)
though any constants defined via macros get caught up in the conversion.
Any factors with a sizeof() of "unsigned char", "char", and "u8" were
dropped, since they're redundant.
The Coccinelle script used for this was:
// Fix redundant parens around sizeof().
@@
type TYPE;
expression THING, E;
@@
(
kvzalloc(
- (sizeof(TYPE)) * E
+ sizeof(TYPE) * E
, ...)
|
kvzalloc(
- (sizeof(THING)) * E
+ sizeof(THING) * E
, ...)
)
// Drop single-byte sizes and redundant parens.
@@
expression COUNT;
typedef u8;
typedef __u8;
@@
(
kvzalloc(
- sizeof(u8) * (COUNT)
+ COUNT
, ...)
|
kvzalloc(
- sizeof(__u8) * (COUNT)
+ COUNT
, ...)
|
kvzalloc(
- sizeof(char) * (COUNT)
+ COUNT
, ...)
|
kvzalloc(
- sizeof(unsigned char) * (COUNT)
+ COUNT
, ...)
|
kvzalloc(
- sizeof(u8) * COUNT
+ COUNT
, ...)
|
kvzalloc(
- sizeof(__u8) * COUNT
+ COUNT
, ...)
|
kvzalloc(
- sizeof(char) * COUNT
+ COUNT
, ...)
|
kvzalloc(
- sizeof(unsigned char) * COUNT
+ COUNT
, ...)
)
// 2-factor product with sizeof(type/expression) and identifier or constant.
@@
type TYPE;
expression THING;
identifier COUNT_ID;
constant COUNT_CONST;
@@
(
- kvzalloc
+ kvcalloc
(
- sizeof(TYPE) * (COUNT_ID)
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(TYPE) * COUNT_ID
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(TYPE) * (COUNT_CONST)
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(TYPE) * COUNT_CONST
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(THING) * (COUNT_ID)
+ COUNT_ID, sizeof(THING)
, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(THING) * COUNT_ID
+ COUNT_ID, sizeof(THING)
, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(THING) * (COUNT_CONST)
+ COUNT_CONST, sizeof(THING)
, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(THING) * COUNT_CONST
+ COUNT_CONST, sizeof(THING)
, ...)
)
// 2-factor product, only identifiers.
@@
identifier SIZE, COUNT;
@@
- kvzalloc
+ kvcalloc
(
- SIZE * COUNT
+ COUNT, SIZE
, ...)
// 3-factor product with 1 sizeof(type) or sizeof(expression), with
// redundant parens removed.
@@
expression THING;
identifier STRIDE, COUNT;
type TYPE;
@@
(
kvzalloc(
- sizeof(TYPE) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kvzalloc(
- sizeof(TYPE) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kvzalloc(
- sizeof(TYPE) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kvzalloc(
- sizeof(TYPE) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kvzalloc(
- sizeof(THING) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kvzalloc(
- sizeof(THING) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kvzalloc(
- sizeof(THING) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kvzalloc(
- sizeof(THING) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
)
// 3-factor product with 2 sizeof(variable), with redundant parens removed.
@@
expression THING1, THING2;
identifier COUNT;
type TYPE1, TYPE2;
@@
(
kvzalloc(
- sizeof(TYPE1) * sizeof(TYPE2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kvzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kvzalloc(
- sizeof(THING1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kvzalloc(
- sizeof(THING1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kvzalloc(
- sizeof(TYPE1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
|
kvzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
)
// 3-factor product, only identifiers, with redundant parens removed.
@@
identifier STRIDE, SIZE, COUNT;
@@
(
kvzalloc(
- (COUNT) * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kvzalloc(
- COUNT * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kvzalloc(
- COUNT * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kvzalloc(
- (COUNT) * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kvzalloc(
- COUNT * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kvzalloc(
- (COUNT) * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kvzalloc(
- (COUNT) * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kvzalloc(
- COUNT * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
)
// Any remaining multi-factor products, first at least 3-factor products,
// when they're not all constants...
@@
expression E1, E2, E3;
constant C1, C2, C3;
@@
(
kvzalloc(C1 * C2 * C3, ...)
|
kvzalloc(
- (E1) * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
|
kvzalloc(
- (E1) * (E2) * E3
+ array3_size(E1, E2, E3)
, ...)
|
kvzalloc(
- (E1) * (E2) * (E3)
+ array3_size(E1, E2, E3)
, ...)
|
kvzalloc(
- E1 * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
)
// And then all remaining 2 factors products when they're not all constants,
// keeping sizeof() as the second factor argument.
@@
expression THING, E1, E2;
type TYPE;
constant C1, C2, C3;
@@
(
kvzalloc(sizeof(THING) * C2, ...)
|
kvzalloc(sizeof(TYPE) * C2, ...)
|
kvzalloc(C1 * C2 * C3, ...)
|
kvzalloc(C1 * C2, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(TYPE) * (E2)
+ E2, sizeof(TYPE)
, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(TYPE) * E2
+ E2, sizeof(TYPE)
, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(THING) * (E2)
+ E2, sizeof(THING)
, ...)
|
- kvzalloc
+ kvcalloc
(
- sizeof(THING) * E2
+ E2, sizeof(THING)
, ...)
|
- kvzalloc
+ kvcalloc
(
- (E1) * E2
+ E1, E2
, ...)
|
- kvzalloc
+ kvcalloc
(
- (E1) * (E2)
+ E1, E2
, ...)
|
- kvzalloc
+ kvcalloc
(
- E1 * E2
+ E1, E2
, ...)
)
Signed-off-by: Kees Cook <keescook@chromium.org>
2018-06-13 04:04:48 +07:00
|
|
|
in->pas = kvcalloc(srq->buf.npages, sizeof(*in->pas), GFP_KERNEL);
|
2016-06-17 19:33:32 +07:00
|
|
|
if (!in->pas) {
|
2013-07-07 21:25:49 +07:00
|
|
|
err = -ENOMEM;
|
|
|
|
goto err_buf;
|
|
|
|
}
|
2016-06-17 19:33:32 +07:00
|
|
|
mlx5_fill_page_array(&srq->buf, in->pas);
|
2013-07-07 21:25:49 +07:00
|
|
|
|
2017-08-16 20:31:22 +07:00
|
|
|
srq->wrid = kvmalloc_array(srq->msrq.max, sizeof(u64), GFP_KERNEL);
|
2013-07-07 21:25:49 +07:00
|
|
|
if (!srq->wrid) {
|
|
|
|
err = -ENOMEM;
|
|
|
|
goto err_in;
|
|
|
|
}
|
2018-11-29 01:53:38 +07:00
|
|
|
srq->wq_sig = 0;
|
2013-07-07 21:25:49 +07:00
|
|
|
|
2017-01-18 19:10:30 +07:00
|
|
|
in->log_page_size = srq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT;
|
2016-06-17 19:33:32 +07:00
|
|
|
if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 &&
|
2017-08-17 19:52:11 +07:00
|
|
|
in->type != IB_SRQT_BASIC)
|
2016-06-17 19:33:32 +07:00
|
|
|
in->user_index = MLX5_IB_DEFAULT_UIDX;
|
2016-01-15 00:12:57 +07:00
|
|
|
|
2013-07-07 21:25:49 +07:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
err_in:
|
2016-06-17 19:33:32 +07:00
|
|
|
kvfree(in->pas);
|
2013-07-07 21:25:49 +07:00
|
|
|
|
|
|
|
err_buf:
|
2014-07-29 03:30:22 +07:00
|
|
|
mlx5_buf_free(dev->mdev, &srq->buf);
|
2013-07-07 21:25:49 +07:00
|
|
|
|
|
|
|
err_db:
|
2014-07-29 03:30:22 +07:00
|
|
|
mlx5_db_free(dev->mdev, &srq->db);
|
2013-07-07 21:25:49 +07:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq)
|
|
|
|
{
|
|
|
|
mlx5_ib_db_unmap_user(to_mucontext(pd->uobject->context), &srq->db);
|
|
|
|
ib_umem_release(srq->umem);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static void destroy_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq)
|
|
|
|
{
|
2017-08-16 20:31:22 +07:00
|
|
|
kvfree(srq->wrid);
|
2014-07-29 03:30:22 +07:00
|
|
|
mlx5_buf_free(dev->mdev, &srq->buf);
|
|
|
|
mlx5_db_free(dev->mdev, &srq->db);
|
2013-07-07 21:25:49 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
|
|
|
|
struct ib_srq_init_attr *init_attr,
|
|
|
|
struct ib_udata *udata)
|
|
|
|
{
|
|
|
|
struct mlx5_ib_dev *dev = to_mdev(pd->device);
|
|
|
|
struct mlx5_ib_srq *srq;
|
2018-03-08 20:51:41 +07:00
|
|
|
size_t desc_size;
|
|
|
|
size_t buf_size;
|
2013-07-07 21:25:49 +07:00
|
|
|
int err;
|
2016-06-17 19:33:32 +07:00
|
|
|
struct mlx5_srq_attr in = {0};
|
2015-05-29 02:28:41 +07:00
|
|
|
__u32 max_srq_wqes = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
|
2013-07-07 21:25:49 +07:00
|
|
|
|
|
|
|
/* Sanity check SRQ size before proceeding */
|
2015-05-29 02:28:41 +07:00
|
|
|
if (init_attr->attr.max_wr >= max_srq_wqes) {
|
2013-07-07 21:25:49 +07:00
|
|
|
mlx5_ib_dbg(dev, "max_wr %d, cap %d\n",
|
|
|
|
init_attr->attr.max_wr,
|
2015-05-29 02:28:41 +07:00
|
|
|
max_srq_wqes);
|
2013-07-07 21:25:49 +07:00
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
}
|
|
|
|
|
|
|
|
srq = kmalloc(sizeof(*srq), GFP_KERNEL);
|
|
|
|
if (!srq)
|
|
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
|
|
|
|
mutex_init(&srq->mutex);
|
|
|
|
spin_lock_init(&srq->lock);
|
|
|
|
srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1);
|
|
|
|
srq->msrq.max_gs = init_attr->attr.max_sge;
|
|
|
|
|
|
|
|
desc_size = sizeof(struct mlx5_wqe_srq_next_seg) +
|
|
|
|
srq->msrq.max_gs * sizeof(struct mlx5_wqe_data_seg);
|
2018-07-10 15:56:50 +07:00
|
|
|
if (desc_size == 0 || srq->msrq.max_gs > desc_size) {
|
|
|
|
err = -EINVAL;
|
|
|
|
goto err_srq;
|
|
|
|
}
|
2013-07-07 21:25:49 +07:00
|
|
|
desc_size = roundup_pow_of_two(desc_size);
|
2018-03-08 20:51:41 +07:00
|
|
|
desc_size = max_t(size_t, 32, desc_size);
|
2018-07-10 15:56:50 +07:00
|
|
|
if (desc_size < sizeof(struct mlx5_wqe_srq_next_seg)) {
|
|
|
|
err = -EINVAL;
|
|
|
|
goto err_srq;
|
|
|
|
}
|
2013-07-07 21:25:49 +07:00
|
|
|
srq->msrq.max_avail_gather = (desc_size - sizeof(struct mlx5_wqe_srq_next_seg)) /
|
|
|
|
sizeof(struct mlx5_wqe_data_seg);
|
|
|
|
srq->msrq.wqe_shift = ilog2(desc_size);
|
|
|
|
buf_size = srq->msrq.max * desc_size;
|
2018-07-10 15:56:50 +07:00
|
|
|
if (buf_size < desc_size) {
|
|
|
|
err = -EINVAL;
|
|
|
|
goto err_srq;
|
|
|
|
}
|
2016-11-27 20:18:20 +07:00
|
|
|
in.type = init_attr->srq_type;
|
2013-07-07 21:25:49 +07:00
|
|
|
|
|
|
|
if (pd->uobject)
|
2016-06-17 19:33:32 +07:00
|
|
|
err = create_srq_user(pd, srq, &in, udata, buf_size);
|
2013-07-07 21:25:49 +07:00
|
|
|
else
|
2016-06-17 19:33:32 +07:00
|
|
|
err = create_srq_kernel(dev, srq, &in, buf_size);
|
2013-07-07 21:25:49 +07:00
|
|
|
|
|
|
|
if (err) {
|
|
|
|
mlx5_ib_warn(dev, "create srq %s failed, err %d\n",
|
|
|
|
pd->uobject ? "user" : "kernel", err);
|
|
|
|
goto err_srq;
|
|
|
|
}
|
|
|
|
|
2016-06-17 19:33:32 +07:00
|
|
|
in.log_size = ilog2(srq->msrq.max);
|
|
|
|
in.wqe_shift = srq->msrq.wqe_shift - 4;
|
|
|
|
if (srq->wq_sig)
|
|
|
|
in.flags |= MLX5_SRQ_FLAG_WQ_SIG;
|
2017-08-17 19:52:04 +07:00
|
|
|
|
|
|
|
if (init_attr->srq_type == IB_SRQT_XRC)
|
2016-06-17 19:33:32 +07:00
|
|
|
in.xrcd = to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn;
|
2017-08-17 19:52:04 +07:00
|
|
|
else
|
2016-06-17 19:33:32 +07:00
|
|
|
in.xrcd = to_mxrcd(dev->devr.x0)->xrcdn;
|
2017-08-17 19:52:04 +07:00
|
|
|
|
2017-08-17 19:52:11 +07:00
|
|
|
if (init_attr->srq_type == IB_SRQT_TM) {
|
|
|
|
in.tm_log_list_size =
|
|
|
|
ilog2(init_attr->ext.tag_matching.max_num_tags) + 1;
|
|
|
|
if (in.tm_log_list_size >
|
|
|
|
MLX5_CAP_GEN(dev->mdev, log_tag_matching_list_sz)) {
|
|
|
|
mlx5_ib_dbg(dev, "TM SRQ max_num_tags exceeding limit\n");
|
|
|
|
err = -EINVAL;
|
|
|
|
goto err_usr_kern_srq;
|
|
|
|
}
|
|
|
|
in.flags |= MLX5_SRQ_FLAG_RNDV;
|
|
|
|
}
|
|
|
|
|
2017-08-17 19:52:04 +07:00
|
|
|
if (ib_srq_has_cq(init_attr->srq_type))
|
|
|
|
in.cqn = to_mcq(init_attr->ext.cq)->mcq.cqn;
|
|
|
|
else
|
2016-06-17 19:33:32 +07:00
|
|
|
in.cqn = to_mcq(dev->devr.c0)->mcq.cqn;
|
2013-07-07 21:25:49 +07:00
|
|
|
|
2016-06-17 19:33:32 +07:00
|
|
|
in.pd = to_mpd(pd)->pdn;
|
|
|
|
in.db_record = srq->db.dma;
|
2018-11-29 01:53:40 +07:00
|
|
|
err = mlx5_cmd_create_srq(dev, &srq->msrq, &in);
|
2016-06-17 19:33:32 +07:00
|
|
|
kvfree(in.pas);
|
2013-07-07 21:25:49 +07:00
|
|
|
if (err) {
|
|
|
|
mlx5_ib_dbg(dev, "create SRQ failed, err %d\n", err);
|
2013-09-11 20:35:24 +07:00
|
|
|
goto err_usr_kern_srq;
|
2013-07-07 21:25:49 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
mlx5_ib_dbg(dev, "create SRQ with srqn 0x%x\n", srq->msrq.srqn);
|
|
|
|
|
|
|
|
srq->msrq.event = mlx5_ib_srq_event;
|
|
|
|
srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn;
|
|
|
|
|
|
|
|
if (pd->uobject)
|
|
|
|
if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof(__u32))) {
|
|
|
|
mlx5_ib_dbg(dev, "copy to user failed\n");
|
|
|
|
err = -EFAULT;
|
|
|
|
goto err_core;
|
|
|
|
}
|
|
|
|
|
|
|
|
init_attr->attr.max_wr = srq->msrq.max - 1;
|
|
|
|
|
|
|
|
return &srq->ibsrq;
|
|
|
|
|
|
|
|
err_core:
|
2018-11-29 01:53:40 +07:00
|
|
|
mlx5_cmd_destroy_srq(dev, &srq->msrq);
|
2013-09-11 20:35:24 +07:00
|
|
|
|
|
|
|
err_usr_kern_srq:
|
2013-07-07 21:25:49 +07:00
|
|
|
if (pd->uobject)
|
|
|
|
destroy_srq_user(pd, srq);
|
|
|
|
else
|
|
|
|
destroy_srq_kernel(dev, srq);
|
|
|
|
|
|
|
|
err_srq:
|
|
|
|
kfree(srq);
|
|
|
|
|
|
|
|
return ERR_PTR(err);
|
|
|
|
}
|
|
|
|
|
|
|
|
int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
|
|
|
|
enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
|
|
|
|
{
|
|
|
|
struct mlx5_ib_dev *dev = to_mdev(ibsrq->device);
|
|
|
|
struct mlx5_ib_srq *srq = to_msrq(ibsrq);
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
/* We don't support resizing SRQs yet */
|
|
|
|
if (attr_mask & IB_SRQ_MAX_WR)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (attr_mask & IB_SRQ_LIMIT) {
|
|
|
|
if (attr->srq_limit >= srq->msrq.max)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
mutex_lock(&srq->mutex);
|
2018-11-29 01:53:40 +07:00
|
|
|
ret = mlx5_cmd_arm_srq(dev, &srq->msrq, attr->srq_limit, 1);
|
2013-07-07 21:25:49 +07:00
|
|
|
mutex_unlock(&srq->mutex);
|
|
|
|
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
|
|
|
|
{
|
|
|
|
struct mlx5_ib_dev *dev = to_mdev(ibsrq->device);
|
|
|
|
struct mlx5_ib_srq *srq = to_msrq(ibsrq);
|
|
|
|
int ret;
|
2016-06-17 19:33:32 +07:00
|
|
|
struct mlx5_srq_attr *out;
|
2013-07-07 21:25:49 +07:00
|
|
|
|
|
|
|
out = kzalloc(sizeof(*out), GFP_KERNEL);
|
|
|
|
if (!out)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
2018-11-29 01:53:40 +07:00
|
|
|
ret = mlx5_cmd_query_srq(dev, &srq->msrq, out);
|
2013-07-07 21:25:49 +07:00
|
|
|
if (ret)
|
|
|
|
goto out_box;
|
|
|
|
|
2016-06-17 19:33:32 +07:00
|
|
|
srq_attr->srq_limit = out->lwm;
|
2013-07-07 21:25:49 +07:00
|
|
|
srq_attr->max_wr = srq->msrq.max - 1;
|
|
|
|
srq_attr->max_sge = srq->msrq.max_gs;
|
|
|
|
|
|
|
|
out_box:
|
|
|
|
kfree(out);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
int mlx5_ib_destroy_srq(struct ib_srq *srq)
|
|
|
|
{
|
|
|
|
struct mlx5_ib_dev *dev = to_mdev(srq->device);
|
|
|
|
struct mlx5_ib_srq *msrq = to_msrq(srq);
|
|
|
|
|
2018-11-29 01:53:40 +07:00
|
|
|
mlx5_cmd_destroy_srq(dev, &msrq->msrq);
|
2013-07-07 21:25:49 +07:00
|
|
|
|
|
|
|
if (srq->uobject) {
|
|
|
|
mlx5_ib_db_unmap_user(to_mucontext(srq->uobject->context), &msrq->db);
|
|
|
|
ib_umem_release(msrq->umem);
|
|
|
|
} else {
|
2013-10-23 13:53:16 +07:00
|
|
|
destroy_srq_kernel(dev, msrq);
|
2013-07-07 21:25:49 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
kfree(srq);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index)
|
|
|
|
{
|
|
|
|
struct mlx5_wqe_srq_next_seg *next;
|
|
|
|
|
|
|
|
/* always called with interrupts disabled. */
|
|
|
|
spin_lock(&srq->lock);
|
|
|
|
|
|
|
|
next = get_wqe(srq, srq->tail);
|
|
|
|
next->next_wqe_index = cpu_to_be16(wqe_index);
|
|
|
|
srq->tail = wqe_index;
|
|
|
|
|
|
|
|
spin_unlock(&srq->lock);
|
|
|
|
}
|
|
|
|
|
2018-07-18 23:25:32 +07:00
|
|
|
int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
|
|
|
|
const struct ib_recv_wr **bad_wr)
|
2013-07-07 21:25:49 +07:00
|
|
|
{
|
|
|
|
struct mlx5_ib_srq *srq = to_msrq(ibsrq);
|
|
|
|
struct mlx5_wqe_srq_next_seg *next;
|
|
|
|
struct mlx5_wqe_data_seg *scat;
|
2016-06-17 19:01:38 +07:00
|
|
|
struct mlx5_ib_dev *dev = to_mdev(ibsrq->device);
|
|
|
|
struct mlx5_core_dev *mdev = dev->mdev;
|
2013-07-07 21:25:49 +07:00
|
|
|
unsigned long flags;
|
|
|
|
int err = 0;
|
|
|
|
int nreq;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
spin_lock_irqsave(&srq->lock, flags);
|
|
|
|
|
2016-06-17 19:01:38 +07:00
|
|
|
if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
|
|
|
|
err = -EIO;
|
|
|
|
*bad_wr = wr;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2013-07-07 21:25:49 +07:00
|
|
|
for (nreq = 0; wr; nreq++, wr = wr->next) {
|
|
|
|
if (unlikely(wr->num_sge > srq->msrq.max_gs)) {
|
|
|
|
err = -EINVAL;
|
|
|
|
*bad_wr = wr;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (unlikely(srq->head == srq->tail)) {
|
|
|
|
err = -ENOMEM;
|
|
|
|
*bad_wr = wr;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
srq->wrid[srq->head] = wr->wr_id;
|
|
|
|
|
|
|
|
next = get_wqe(srq, srq->head);
|
|
|
|
srq->head = be16_to_cpu(next->next_wqe_index);
|
|
|
|
scat = (struct mlx5_wqe_data_seg *)(next + 1);
|
|
|
|
|
|
|
|
for (i = 0; i < wr->num_sge; i++) {
|
|
|
|
scat[i].byte_count = cpu_to_be32(wr->sg_list[i].length);
|
|
|
|
scat[i].lkey = cpu_to_be32(wr->sg_list[i].lkey);
|
|
|
|
scat[i].addr = cpu_to_be64(wr->sg_list[i].addr);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (i < srq->msrq.max_avail_gather) {
|
|
|
|
scat[i].byte_count = 0;
|
|
|
|
scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY);
|
|
|
|
scat[i].addr = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (likely(nreq)) {
|
|
|
|
srq->wqe_ctr += nreq;
|
|
|
|
|
|
|
|
/* Make sure that descriptors are written before
|
|
|
|
* doorbell record.
|
|
|
|
*/
|
|
|
|
wmb();
|
|
|
|
|
|
|
|
*srq->db.db = cpu_to_be32(srq->wqe_ctr);
|
|
|
|
}
|
2016-06-17 19:01:38 +07:00
|
|
|
out:
|
2013-07-07 21:25:49 +07:00
|
|
|
spin_unlock_irqrestore(&srq->lock, flags);
|
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|