AuxXxilium/linux_dsm_epyc7002
1
0
Fork 0
mirror of https://github.com/AuxXxilium/linux_dsm_epyc7002.git synced 2024-12-20 11:38:26 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity

net/mlx5: Move SRQ functions to RDMA part

Browse Source 
There is no need to keep SRQ which is RDMA object in mlx5_core.
In this patch, we partially move the execution code, while next patches
will move table initialization/release logic too.

Reviewed-by: Mark Bloch <markb@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
This commit is contained in:
Leon Romanovsky 2018-11-28 20:53:37 +02:00
parent c23f88cb57
commit f02d0d6e53
10 changed files with 717 additions and 724 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
drivers/infiniband/hw/mlx5/Makefile
View File

@ -1,6 +1,8 @@
obj-$(CONFIG_MLX5_INFINIBAND) += mlx5_ib.o
mlx5_ib-y := main.o cq.o doorbell.o qp.o mem.o srq.o mr.o ah.o mad.o gsi.o ib_virt.o cmd.o cong.o
mlx5_ib-y := main.o cq.o doorbell.o qp.o mem.o srq_cmd.o \
srq.o mr.o ah.o mad.o gsi.o ib_virt.o cmd.o \
cong.o
mlx5_ib-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += odp.o
mlx5_ib-$(CONFIG_MLX5_ESWITCH) += ib_rep.o
mlx5_ib-$(CONFIG_INFINIBAND_USER_ACCESS) += devx.o

1
drivers/infiniband/hw/mlx5/cq.c
View File

@ -35,6 +35,7 @@
#include <rdma/ib_user_verbs.h>
#include <rdma/ib_cache.h>
#include "mlx5_ib.h"
#include "srq.h"
static void mlx5_ib_cq_comp(struct mlx5_core_cq *cq)
{

2
drivers/infiniband/hw/mlx5/srq.c
View File

@ -9,8 +9,8 @@
#include <linux/slab.h>
#include <rdma/ib_umem.h>
#include <rdma/ib_user_verbs.h>
#include "mlx5_ib.h"
#include "srq.h"
/* not supported currently */
static int srq_signature;

46
drivers/infiniband/hw/mlx5/srq.h Normal file
View File

@ -0,0 +1,46 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2013-2018, Mellanox Technologies. All rights reserved.
*/
#ifndef MLX5_IB_SRQ_H
#define MLX5_IB_SRQ_H
enum {
MLX5_SRQ_FLAG_ERR = (1 << 0),
MLX5_SRQ_FLAG_WQ_SIG = (1 << 1),
MLX5_SRQ_FLAG_RNDV = (1 << 2),
};
struct mlx5_srq_attr {
u32 type;
u32 flags;
u32 log_size;
u32 wqe_shift;
u32 log_page_size;
u32 wqe_cnt;
u32 srqn;
u32 xrcd;
u32 page_offset;
u32 cqn;
u32 pd;
u32 lwm;
u32 user_index;
u64 db_record;
__be64 *pas;
u32 tm_log_list_size;
u32 tm_next_tag;
u32 tm_hw_phase_cnt;
u32 tm_sw_phase_cnt;
u16 uid;
};
int mlx5_core_create_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
struct mlx5_srq_attr *in);
int mlx5_core_destroy_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq);
int mlx5_core_query_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
struct mlx5_srq_attr *out);
int mlx5_core_arm_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
u16 lwm, int is_srq);
struct mlx5_core_srq *mlx5_core_get_srq(struct mlx5_core_dev *dev, u32 srqn);
#endif /* MLX5_IB_SRQ_H */

666
drivers/infiniband/hw/mlx5/srq_cmd.c Normal file
View File

@ -0,0 +1,666 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
* Copyright (c) 2013-2018, Mellanox Technologies inc. All rights reserved.
*/
#include <linux/kernel.h>
#include <linux/mlx5/driver.h>
#include <linux/mlx5/cmd.h>
#include "srq.h"
static int get_pas_size(struct mlx5_srq_attr *in)
{
u32 log_page_size = in->log_page_size + 12;
u32 log_srq_size = in->log_size;
u32 log_rq_stride = in->wqe_shift;
u32 page_offset = in->page_offset;
u32 po_quanta = 1 << (log_page_size - 6);
u32 rq_sz = 1 << (log_srq_size + 4 + log_rq_stride);
u32 page_size = 1 << log_page_size;
u32 rq_sz_po = rq_sz + (page_offset * po_quanta);
u32 rq_num_pas = DIV_ROUND_UP(rq_sz_po, page_size);
return rq_num_pas * sizeof(u64);
}
static void set_wq(void *wq, struct mlx5_srq_attr *in)
{
MLX5_SET(wq, wq, wq_signature, !!(in->flags
& MLX5_SRQ_FLAG_WQ_SIG));
MLX5_SET(wq, wq, log_wq_pg_sz, in->log_page_size);
MLX5_SET(wq, wq, log_wq_stride, in->wqe_shift + 4);
MLX5_SET(wq, wq, log_wq_sz, in->log_size);
MLX5_SET(wq, wq, page_offset, in->page_offset);
MLX5_SET(wq, wq, lwm, in->lwm);
MLX5_SET(wq, wq, pd, in->pd);
MLX5_SET64(wq, wq, dbr_addr, in->db_record);
}
static void set_srqc(void *srqc, struct mlx5_srq_attr *in)
{
MLX5_SET(srqc, srqc, wq_signature, !!(in->flags
& MLX5_SRQ_FLAG_WQ_SIG));
MLX5_SET(srqc, srqc, log_page_size, in->log_page_size);
MLX5_SET(srqc, srqc, log_rq_stride, in->wqe_shift);
MLX5_SET(srqc, srqc, log_srq_size, in->log_size);
MLX5_SET(srqc, srqc, page_offset, in->page_offset);
MLX5_SET(srqc, srqc, lwm, in->lwm);
MLX5_SET(srqc, srqc, pd, in->pd);
MLX5_SET64(srqc, srqc, dbr_addr, in->db_record);
MLX5_SET(srqc, srqc, xrcd, in->xrcd);
MLX5_SET(srqc, srqc, cqn, in->cqn);
}
static void get_wq(void *wq, struct mlx5_srq_attr *in)
{
if (MLX5_GET(wq, wq, wq_signature))
in->flags &= MLX5_SRQ_FLAG_WQ_SIG;
in->log_page_size = MLX5_GET(wq, wq, log_wq_pg_sz);
in->wqe_shift = MLX5_GET(wq, wq, log_wq_stride) - 4;
in->log_size = MLX5_GET(wq, wq, log_wq_sz);
in->page_offset = MLX5_GET(wq, wq, page_offset);
in->lwm = MLX5_GET(wq, wq, lwm);
in->pd = MLX5_GET(wq, wq, pd);
in->db_record = MLX5_GET64(wq, wq, dbr_addr);
}
static void get_srqc(void *srqc, struct mlx5_srq_attr *in)
{
if (MLX5_GET(srqc, srqc, wq_signature))
in->flags &= MLX5_SRQ_FLAG_WQ_SIG;
in->log_page_size = MLX5_GET(srqc, srqc, log_page_size);
in->wqe_shift = MLX5_GET(srqc, srqc, log_rq_stride);
in->log_size = MLX5_GET(srqc, srqc, log_srq_size);
in->page_offset = MLX5_GET(srqc, srqc, page_offset);
in->lwm = MLX5_GET(srqc, srqc, lwm);
in->pd = MLX5_GET(srqc, srqc, pd);
in->db_record = MLX5_GET64(srqc, srqc, dbr_addr);
}
struct mlx5_core_srq *mlx5_core_get_srq(struct mlx5_core_dev *dev, u32 srqn)
{
struct mlx5_srq_table *table = &dev->priv.srq_table;
struct mlx5_core_srq *srq;
spin_lock(&table->lock);
srq = radix_tree_lookup(&table->tree, srqn);
if (srq)
atomic_inc(&srq->refcount);
spin_unlock(&table->lock);
return srq;
}
static int create_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
struct mlx5_srq_attr *in)
{
u32 create_out[MLX5_ST_SZ_DW(create_srq_out)] = {0};
void *create_in;
void *srqc;
void *pas;
int pas_size;
int inlen;
int err;
pas_size = get_pas_size(in);
inlen = MLX5_ST_SZ_BYTES(create_srq_in) + pas_size;
create_in = kvzalloc(inlen, GFP_KERNEL);
if (!create_in)
return -ENOMEM;
MLX5_SET(create_srq_in, create_in, uid, in->uid);
srqc = MLX5_ADDR_OF(create_srq_in, create_in, srq_context_entry);
pas = MLX5_ADDR_OF(create_srq_in, create_in, pas);
set_srqc(srqc, in);
memcpy(pas, in->pas, pas_size);
MLX5_SET(create_srq_in, create_in, opcode,
MLX5_CMD_OP_CREATE_SRQ);
err = mlx5_cmd_exec(dev, create_in, inlen, create_out,
sizeof(create_out));
kvfree(create_in);
if (!err) {
srq->srqn = MLX5_GET(create_srq_out, create_out, srqn);
srq->uid = in->uid;
}
return err;
}
static int destroy_srq_cmd(struct mlx5_core_dev *dev,
struct mlx5_core_srq *srq)
{
u32 srq_in[MLX5_ST_SZ_DW(destroy_srq_in)] = {0};
u32 srq_out[MLX5_ST_SZ_DW(destroy_srq_out)] = {0};
MLX5_SET(destroy_srq_in, srq_in, opcode,
MLX5_CMD_OP_DESTROY_SRQ);
MLX5_SET(destroy_srq_in, srq_in, srqn, srq->srqn);
MLX5_SET(destroy_srq_in, srq_in, uid, srq->uid);
return mlx5_cmd_exec(dev, srq_in, sizeof(srq_in),
srq_out, sizeof(srq_out));
}
static int arm_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
u16 lwm, int is_srq)
{
u32 srq_in[MLX5_ST_SZ_DW(arm_rq_in)] = {0};
u32 srq_out[MLX5_ST_SZ_DW(arm_rq_out)] = {0};
MLX5_SET(arm_rq_in, srq_in, opcode, MLX5_CMD_OP_ARM_RQ);
MLX5_SET(arm_rq_in, srq_in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_SRQ);
MLX5_SET(arm_rq_in, srq_in, srq_number, srq->srqn);
MLX5_SET(arm_rq_in, srq_in, lwm, lwm);
MLX5_SET(arm_rq_in, srq_in, uid, srq->uid);
return mlx5_cmd_exec(dev, srq_in, sizeof(srq_in),
srq_out, sizeof(srq_out));
}
static int query_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
struct mlx5_srq_attr *out)
{
u32 srq_in[MLX5_ST_SZ_DW(query_srq_in)] = {0};
u32 *srq_out;
void *srqc;
int err;
srq_out = kvzalloc(MLX5_ST_SZ_BYTES(query_srq_out), GFP_KERNEL);
if (!srq_out)
return -ENOMEM;
MLX5_SET(query_srq_in, srq_in, opcode,
MLX5_CMD_OP_QUERY_SRQ);
MLX5_SET(query_srq_in, srq_in, srqn, srq->srqn);
err = mlx5_cmd_exec(dev, srq_in, sizeof(srq_in),
srq_out, MLX5_ST_SZ_BYTES(query_srq_out));
if (err)
goto out;
srqc = MLX5_ADDR_OF(query_srq_out, srq_out, srq_context_entry);
get_srqc(srqc, out);
if (MLX5_GET(srqc, srqc, state) != MLX5_SRQC_STATE_GOOD)
out->flags |= MLX5_SRQ_FLAG_ERR;
out:
kvfree(srq_out);
return err;
}
static int create_xrc_srq_cmd(struct mlx5_core_dev *dev,
struct mlx5_core_srq *srq,
struct mlx5_srq_attr *in)
{
u32 create_out[MLX5_ST_SZ_DW(create_xrc_srq_out)];
void *create_in;
void *xrc_srqc;
void *pas;
int pas_size;
int inlen;
int err;
pas_size = get_pas_size(in);
inlen = MLX5_ST_SZ_BYTES(create_xrc_srq_in) + pas_size;
create_in = kvzalloc(inlen, GFP_KERNEL);
if (!create_in)
return -ENOMEM;
MLX5_SET(create_xrc_srq_in, create_in, uid, in->uid);
xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, create_in,
xrc_srq_context_entry);
pas = MLX5_ADDR_OF(create_xrc_srq_in, create_in, pas);
set_srqc(xrc_srqc, in);
MLX5_SET(xrc_srqc, xrc_srqc, user_index, in->user_index);
memcpy(pas, in->pas, pas_size);
MLX5_SET(create_xrc_srq_in, create_in, opcode,
MLX5_CMD_OP_CREATE_XRC_SRQ);
memset(create_out, 0, sizeof(create_out));
err = mlx5_cmd_exec(dev, create_in, inlen, create_out,
sizeof(create_out));
if (err)
goto out;
srq->srqn = MLX5_GET(create_xrc_srq_out, create_out, xrc_srqn);
srq->uid = in->uid;
out:
kvfree(create_in);
return err;
}
static int destroy_xrc_srq_cmd(struct mlx5_core_dev *dev,
struct mlx5_core_srq *srq)
{
u32 xrcsrq_in[MLX5_ST_SZ_DW(destroy_xrc_srq_in)] = {0};
u32 xrcsrq_out[MLX5_ST_SZ_DW(destroy_xrc_srq_out)] = {0};
MLX5_SET(destroy_xrc_srq_in, xrcsrq_in, opcode,
MLX5_CMD_OP_DESTROY_XRC_SRQ);
MLX5_SET(destroy_xrc_srq_in, xrcsrq_in, xrc_srqn, srq->srqn);
MLX5_SET(destroy_xrc_srq_in, xrcsrq_in, uid, srq->uid);
return mlx5_cmd_exec(dev, xrcsrq_in, sizeof(xrcsrq_in),
xrcsrq_out, sizeof(xrcsrq_out));
}
static int arm_xrc_srq_cmd(struct mlx5_core_dev *dev,
struct mlx5_core_srq *srq, u16 lwm)
{
u32 xrcsrq_in[MLX5_ST_SZ_DW(arm_xrc_srq_in)] = {0};
u32 xrcsrq_out[MLX5_ST_SZ_DW(arm_xrc_srq_out)] = {0};
MLX5_SET(arm_xrc_srq_in, xrcsrq_in, opcode, MLX5_CMD_OP_ARM_XRC_SRQ);
MLX5_SET(arm_xrc_srq_in, xrcsrq_in, op_mod, MLX5_ARM_XRC_SRQ_IN_OP_MOD_XRC_SRQ);
MLX5_SET(arm_xrc_srq_in, xrcsrq_in, xrc_srqn, srq->srqn);
MLX5_SET(arm_xrc_srq_in, xrcsrq_in, lwm, lwm);
MLX5_SET(arm_xrc_srq_in, xrcsrq_in, uid, srq->uid);
return mlx5_cmd_exec(dev, xrcsrq_in, sizeof(xrcsrq_in),
xrcsrq_out, sizeof(xrcsrq_out));
}
static int query_xrc_srq_cmd(struct mlx5_core_dev *dev,
struct mlx5_core_srq *srq,
struct mlx5_srq_attr *out)
{
u32 xrcsrq_in[MLX5_ST_SZ_DW(query_xrc_srq_in)];
u32 *xrcsrq_out;
void *xrc_srqc;
int err;
xrcsrq_out = kvzalloc(MLX5_ST_SZ_BYTES(query_xrc_srq_out), GFP_KERNEL);
if (!xrcsrq_out)
return -ENOMEM;
memset(xrcsrq_in, 0, sizeof(xrcsrq_in));
MLX5_SET(query_xrc_srq_in, xrcsrq_in, opcode,
MLX5_CMD_OP_QUERY_XRC_SRQ);
MLX5_SET(query_xrc_srq_in, xrcsrq_in, xrc_srqn, srq->srqn);
err = mlx5_cmd_exec(dev, xrcsrq_in, sizeof(xrcsrq_in), xrcsrq_out,
MLX5_ST_SZ_BYTES(query_xrc_srq_out));
if (err)
goto out;
xrc_srqc = MLX5_ADDR_OF(query_xrc_srq_out, xrcsrq_out,
xrc_srq_context_entry);
get_srqc(xrc_srqc, out);
if (MLX5_GET(xrc_srqc, xrc_srqc, state) != MLX5_XRC_SRQC_STATE_GOOD)
out->flags |= MLX5_SRQ_FLAG_ERR;
out:
kvfree(xrcsrq_out);
return err;
}
static int mlx5_core_create_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen,
u32 *rmpn)
{
u32 out[MLX5_ST_SZ_DW(create_rmp_out)] = { 0 };
int err;
MLX5_SET(create_rmp_in, in, opcode, MLX5_CMD_OP_CREATE_RMP);
err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
if (!err)
*rmpn = MLX5_GET(create_rmp_out, out, rmpn);
return err;
}
static int mlx5_core_modify_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen)
{
u32 out[MLX5_ST_SZ_DW(modify_rmp_out)] = {0};
MLX5_SET(modify_rmp_in, in, opcode, MLX5_CMD_OP_MODIFY_RMP);
return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
}
static int mlx5_core_query_rmp(struct mlx5_core_dev *dev, u32 rmpn, u32 *out)
{
u32 in[MLX5_ST_SZ_DW(query_rmp_in)] = {0};
int outlen = MLX5_ST_SZ_BYTES(query_rmp_out);
MLX5_SET(query_rmp_in, in, opcode, MLX5_CMD_OP_QUERY_RMP);
MLX5_SET(query_rmp_in, in, rmpn, rmpn);
return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
}
static int create_rmp_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
struct mlx5_srq_attr *in)
{
void *create_in;
void *rmpc;
void *wq;
int pas_size;
int inlen;
int err;
pas_size = get_pas_size(in);
inlen = MLX5_ST_SZ_BYTES(create_rmp_in) + pas_size;
create_in = kvzalloc(inlen, GFP_KERNEL);
if (!create_in)
return -ENOMEM;
rmpc = MLX5_ADDR_OF(create_rmp_in, create_in, ctx);
wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY);
MLX5_SET(create_rmp_in, create_in, uid, in->uid);
set_wq(wq, in);
memcpy(MLX5_ADDR_OF(rmpc, rmpc, wq.pas), in->pas, pas_size);
err = mlx5_core_create_rmp(dev, create_in, inlen, &srq->srqn);
if (!err)
srq->uid = in->uid;
kvfree(create_in);
return err;
}
static int destroy_rmp_cmd(struct mlx5_core_dev *dev,
struct mlx5_core_srq *srq)
{
u32 in[MLX5_ST_SZ_DW(destroy_rmp_in)] = {};
u32 out[MLX5_ST_SZ_DW(destroy_rmp_out)] = {};
MLX5_SET(destroy_rmp_in, in, opcode, MLX5_CMD_OP_DESTROY_RMP);
MLX5_SET(destroy_rmp_in, in, rmpn, srq->srqn);
MLX5_SET(destroy_rmp_in, in, uid, srq->uid);
return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
static int arm_rmp_cmd(struct mlx5_core_dev *dev,
struct mlx5_core_srq *srq,
u16 lwm)
{
void *in;
void *rmpc;
void *wq;
void *bitmask;
int err;
in = kvzalloc(MLX5_ST_SZ_BYTES(modify_rmp_in), GFP_KERNEL);
if (!in)
return -ENOMEM;
rmpc = MLX5_ADDR_OF(modify_rmp_in, in, ctx);
bitmask = MLX5_ADDR_OF(modify_rmp_in, in, bitmask);
wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
MLX5_SET(modify_rmp_in, in, rmp_state, MLX5_RMPC_STATE_RDY);
MLX5_SET(modify_rmp_in, in, rmpn, srq->srqn);
MLX5_SET(modify_rmp_in, in, uid, srq->uid);
MLX5_SET(wq, wq, lwm, lwm);
MLX5_SET(rmp_bitmask, bitmask, lwm, 1);
MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY);
err = mlx5_core_modify_rmp(dev, in, MLX5_ST_SZ_BYTES(modify_rmp_in));
kvfree(in);
return err;
}
static int query_rmp_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
struct mlx5_srq_attr *out)
{
u32 *rmp_out;
void *rmpc;
int err;
rmp_out = kvzalloc(MLX5_ST_SZ_BYTES(query_rmp_out), GFP_KERNEL);
if (!rmp_out)
return -ENOMEM;
err = mlx5_core_query_rmp(dev, srq->srqn, rmp_out);
if (err)
goto out;
rmpc = MLX5_ADDR_OF(query_rmp_out, rmp_out, rmp_context);
get_wq(MLX5_ADDR_OF(rmpc, rmpc, wq), out);
if (MLX5_GET(rmpc, rmpc, state) != MLX5_RMPC_STATE_RDY)
out->flags |= MLX5_SRQ_FLAG_ERR;
out:
kvfree(rmp_out);
return err;
}
static int create_xrq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
struct mlx5_srq_attr *in)
{
u32 create_out[MLX5_ST_SZ_DW(create_xrq_out)] = {0};
void *create_in;
void *xrqc;
void *wq;
int pas_size;
int inlen;
int err;
pas_size = get_pas_size(in);
inlen = MLX5_ST_SZ_BYTES(create_xrq_in) + pas_size;
create_in = kvzalloc(inlen, GFP_KERNEL);
if (!create_in)
return -ENOMEM;
xrqc = MLX5_ADDR_OF(create_xrq_in, create_in, xrq_context);
wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
set_wq(wq, in);
memcpy(MLX5_ADDR_OF(xrqc, xrqc, wq.pas), in->pas, pas_size);
if (in->type == IB_SRQT_TM) {
MLX5_SET(xrqc, xrqc, topology, MLX5_XRQC_TOPOLOGY_TAG_MATCHING);
if (in->flags & MLX5_SRQ_FLAG_RNDV)
MLX5_SET(xrqc, xrqc, offload, MLX5_XRQC_OFFLOAD_RNDV);
MLX5_SET(xrqc, xrqc,
tag_matching_topology_context.log_matching_list_sz,
in->tm_log_list_size);
}
MLX5_SET(xrqc, xrqc, user_index, in->user_index);
MLX5_SET(xrqc, xrqc, cqn, in->cqn);
MLX5_SET(create_xrq_in, create_in, opcode, MLX5_CMD_OP_CREATE_XRQ);
MLX5_SET(create_xrq_in, create_in, uid, in->uid);
err = mlx5_cmd_exec(dev, create_in, inlen, create_out,
sizeof(create_out));
kvfree(create_in);
if (!err) {
srq->srqn = MLX5_GET(create_xrq_out, create_out, xrqn);
srq->uid = in->uid;
}
return err;
}
static int destroy_xrq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq)
{
u32 in[MLX5_ST_SZ_DW(destroy_xrq_in)] = {0};
u32 out[MLX5_ST_SZ_DW(destroy_xrq_out)] = {0};
MLX5_SET(destroy_xrq_in, in, opcode, MLX5_CMD_OP_DESTROY_XRQ);
MLX5_SET(destroy_xrq_in, in, xrqn, srq->srqn);
MLX5_SET(destroy_xrq_in, in, uid, srq->uid);
return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
static int arm_xrq_cmd(struct mlx5_core_dev *dev,
struct mlx5_core_srq *srq,
u16 lwm)
{
u32 out[MLX5_ST_SZ_DW(arm_rq_out)] = {0};
u32 in[MLX5_ST_SZ_DW(arm_rq_in)] = {0};
MLX5_SET(arm_rq_in, in, opcode, MLX5_CMD_OP_ARM_RQ);
MLX5_SET(arm_rq_in, in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_XRQ);
MLX5_SET(arm_rq_in, in, srq_number, srq->srqn);
MLX5_SET(arm_rq_in, in, lwm, lwm);
MLX5_SET(arm_rq_in, in, uid, srq->uid);
return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
static int query_xrq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
struct mlx5_srq_attr *out)
{
u32 in[MLX5_ST_SZ_DW(query_xrq_in)] = {0};
u32 *xrq_out;
int outlen = MLX5_ST_SZ_BYTES(query_xrq_out);
void *xrqc;
int err;
xrq_out = kvzalloc(outlen, GFP_KERNEL);
if (!xrq_out)
return -ENOMEM;
MLX5_SET(query_xrq_in, in, opcode, MLX5_CMD_OP_QUERY_XRQ);
MLX5_SET(query_xrq_in, in, xrqn, srq->srqn);
err = mlx5_cmd_exec(dev, in, sizeof(in), xrq_out, outlen);
if (err)
goto out;
xrqc = MLX5_ADDR_OF(query_xrq_out, xrq_out, xrq_context);
get_wq(MLX5_ADDR_OF(xrqc, xrqc, wq), out);
if (MLX5_GET(xrqc, xrqc, state) != MLX5_XRQC_STATE_GOOD)
out->flags |= MLX5_SRQ_FLAG_ERR;
out->tm_next_tag =
MLX5_GET(xrqc, xrqc,
tag_matching_topology_context.append_next_index);
out->tm_hw_phase_cnt =
MLX5_GET(xrqc, xrqc,
tag_matching_topology_context.hw_phase_cnt);
out->tm_sw_phase_cnt =
MLX5_GET(xrqc, xrqc,
tag_matching_topology_context.sw_phase_cnt);
out:
kvfree(xrq_out);
return err;
}
static int create_srq_split(struct mlx5_core_dev *dev,
struct mlx5_core_srq *srq,
struct mlx5_srq_attr *in)
{
if (!dev->issi)
return create_srq_cmd(dev, srq, in);
switch (srq->common.res) {
case MLX5_RES_XSRQ:
return create_xrc_srq_cmd(dev, srq, in);
case MLX5_RES_XRQ:
return create_xrq_cmd(dev, srq, in);
default:
return create_rmp_cmd(dev, srq, in);
}
}
static int destroy_srq_split(struct mlx5_core_dev *dev,
struct mlx5_core_srq *srq)
{
if (!dev->issi)
return destroy_srq_cmd(dev, srq);
switch (srq->common.res) {
case MLX5_RES_XSRQ:
return destroy_xrc_srq_cmd(dev, srq);
case MLX5_RES_XRQ:
return destroy_xrq_cmd(dev, srq);
default:
return destroy_rmp_cmd(dev, srq);
}
}
int mlx5_core_create_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
struct mlx5_srq_attr *in)
{
int err;
struct mlx5_srq_table *table = &dev->priv.srq_table;
switch (in->type) {
case IB_SRQT_XRC:
srq->common.res = MLX5_RES_XSRQ;
break;
case IB_SRQT_TM:
srq->common.res = MLX5_RES_XRQ;
break;
default:
srq->common.res = MLX5_RES_SRQ;
}
err = create_srq_split(dev, srq, in);
if (err)
return err;
atomic_set(&srq->refcount, 1);
init_completion(&srq->free);
spin_lock_irq(&table->lock);
err = radix_tree_insert(&table->tree, srq->srqn, srq);
spin_unlock_irq(&table->lock);
if (err)
goto err_destroy_srq_split;
return 0;
err_destroy_srq_split:
destroy_srq_split(dev, srq);
return err;
}
int mlx5_core_destroy_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq)
{
struct mlx5_srq_table *table = &dev->priv.srq_table;
struct mlx5_core_srq *tmp;
int err;
spin_lock_irq(&table->lock);
tmp = radix_tree_delete(&table->tree, srq->srqn);
spin_unlock_irq(&table->lock);
if (!tmp || tmp != srq)
return -EINVAL;
err = destroy_srq_split(dev, srq);
if (err)
return err;
if (atomic_dec_and_test(&srq->refcount))
complete(&srq->free);
wait_for_completion(&srq->free);
return 0;
}
int mlx5_core_query_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
struct mlx5_srq_attr *out)
{
if (!dev->issi)
return query_srq_cmd(dev, srq, out);
switch (srq->common.res) {
case MLX5_RES_XSRQ:
return query_xrc_srq_cmd(dev, srq, out);
case MLX5_RES_XRQ:
return query_xrq_cmd(dev, srq, out);
default:
return query_rmp_cmd(dev, srq, out);
}
}
int mlx5_core_arm_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
u16 lwm, int is_srq)
{
if (!dev->issi)
return arm_srq_cmd(dev, srq, lwm, is_srq);
switch (srq->common.res) {
case MLX5_RES_XSRQ:
return arm_xrc_srq_cmd(dev, srq, lwm);
case MLX5_RES_XRQ:
return arm_xrq_cmd(dev, srq, lwm);
default:
return arm_rmp_cmd(dev, srq, lwm);
}
}

634
drivers/net/ethernet/mellanox/mlx5/core/srq.c
View File

@ -4,12 +4,8 @@
*/
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/mlx5/driver.h>
#include <linux/mlx5/cmd.h>
#include <linux/mlx5/srq.h>
#include <rdma/ib_verbs.h>
#include <linux/mlx5/transobj.h>
static int srq_event_notifier(struct notifier_block *nb,
unsigned long type, void *data)
@ -47,636 +43,6 @@ static int srq_event_notifier(struct notifier_block *nb,
return NOTIFY_OK;
}
static int get_pas_size(struct mlx5_srq_attr *in)
{
u32 log_page_size = in->log_page_size + 12;
u32 log_srq_size = in->log_size;
u32 log_rq_stride = in->wqe_shift;
u32 page_offset = in->page_offset;
u32 po_quanta = 1 << (log_page_size - 6);
u32 rq_sz = 1 << (log_srq_size + 4 + log_rq_stride);
u32 page_size = 1 << log_page_size;
u32 rq_sz_po = rq_sz + (page_offset * po_quanta);
u32 rq_num_pas = DIV_ROUND_UP(rq_sz_po, page_size);
return rq_num_pas * sizeof(u64);
}
static void set_wq(void *wq, struct mlx5_srq_attr *in)
{
MLX5_SET(wq, wq, wq_signature, !!(in->flags
& MLX5_SRQ_FLAG_WQ_SIG));
MLX5_SET(wq, wq, log_wq_pg_sz, in->log_page_size);
MLX5_SET(wq, wq, log_wq_stride, in->wqe_shift + 4);
MLX5_SET(wq, wq, log_wq_sz, in->log_size);
MLX5_SET(wq, wq, page_offset, in->page_offset);
MLX5_SET(wq, wq, lwm, in->lwm);
MLX5_SET(wq, wq, pd, in->pd);
MLX5_SET64(wq, wq, dbr_addr, in->db_record);
}
static void set_srqc(void *srqc, struct mlx5_srq_attr *in)
{
MLX5_SET(srqc, srqc, wq_signature, !!(in->flags
& MLX5_SRQ_FLAG_WQ_SIG));
MLX5_SET(srqc, srqc, log_page_size, in->log_page_size);
MLX5_SET(srqc, srqc, log_rq_stride, in->wqe_shift);
MLX5_SET(srqc, srqc, log_srq_size, in->log_size);
MLX5_SET(srqc, srqc, page_offset, in->page_offset);
MLX5_SET(srqc, srqc, lwm, in->lwm);
MLX5_SET(srqc, srqc, pd, in->pd);
MLX5_SET64(srqc, srqc, dbr_addr, in->db_record);
MLX5_SET(srqc, srqc, xrcd, in->xrcd);
MLX5_SET(srqc, srqc, cqn, in->cqn);
}
static void get_wq(void *wq, struct mlx5_srq_attr *in)
{
if (MLX5_GET(wq, wq, wq_signature))
in->flags &= MLX5_SRQ_FLAG_WQ_SIG;
in->log_page_size = MLX5_GET(wq, wq, log_wq_pg_sz);
in->wqe_shift = MLX5_GET(wq, wq, log_wq_stride) - 4;
in->log_size = MLX5_GET(wq, wq, log_wq_sz);
in->page_offset = MLX5_GET(wq, wq, page_offset);
in->lwm = MLX5_GET(wq, wq, lwm);
in->pd = MLX5_GET(wq, wq, pd);
in->db_record = MLX5_GET64(wq, wq, dbr_addr);
}
static void get_srqc(void *srqc, struct mlx5_srq_attr *in)
{
if (MLX5_GET(srqc, srqc, wq_signature))
in->flags &= MLX5_SRQ_FLAG_WQ_SIG;
in->log_page_size = MLX5_GET(srqc, srqc, log_page_size);
in->wqe_shift = MLX5_GET(srqc, srqc, log_rq_stride);
in->log_size = MLX5_GET(srqc, srqc, log_srq_size);
in->page_offset = MLX5_GET(srqc, srqc, page_offset);
in->lwm = MLX5_GET(srqc, srqc, lwm);
in->pd = MLX5_GET(srqc, srqc, pd);
in->db_record = MLX5_GET64(srqc, srqc, dbr_addr);
}
struct mlx5_core_srq *mlx5_core_get_srq(struct mlx5_core_dev *dev, u32 srqn)
{
struct mlx5_srq_table *table = &dev->priv.srq_table;
struct mlx5_core_srq *srq;
spin_lock(&table->lock);
srq = radix_tree_lookup(&table->tree, srqn);
if (srq)
atomic_inc(&srq->refcount);
spin_unlock(&table->lock);
return srq;
}
EXPORT_SYMBOL(mlx5_core_get_srq);
static int create_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
struct mlx5_srq_attr *in)
{
u32 create_out[MLX5_ST_SZ_DW(create_srq_out)] = {0};
void *create_in;
void *srqc;
void *pas;
int pas_size;
int inlen;
int err;
pas_size = get_pas_size(in);
inlen = MLX5_ST_SZ_BYTES(create_srq_in) + pas_size;
create_in = kvzalloc(inlen, GFP_KERNEL);
if (!create_in)
return -ENOMEM;
MLX5_SET(create_srq_in, create_in, uid, in->uid);
srqc = MLX5_ADDR_OF(create_srq_in, create_in, srq_context_entry);
pas = MLX5_ADDR_OF(create_srq_in, create_in, pas);
set_srqc(srqc, in);
memcpy(pas, in->pas, pas_size);
MLX5_SET(create_srq_in, create_in, opcode,
MLX5_CMD_OP_CREATE_SRQ);
err = mlx5_cmd_exec(dev, create_in, inlen, create_out,
sizeof(create_out));
kvfree(create_in);
if (!err) {
srq->srqn = MLX5_GET(create_srq_out, create_out, srqn);
srq->uid = in->uid;
}
return err;
}
static int destroy_srq_cmd(struct mlx5_core_dev *dev,
struct mlx5_core_srq *srq)
{
u32 srq_in[MLX5_ST_SZ_DW(destroy_srq_in)] = {0};
u32 srq_out[MLX5_ST_SZ_DW(destroy_srq_out)] = {0};
MLX5_SET(destroy_srq_in, srq_in, opcode,
MLX5_CMD_OP_DESTROY_SRQ);
MLX5_SET(destroy_srq_in, srq_in, srqn, srq->srqn);
MLX5_SET(destroy_srq_in, srq_in, uid, srq->uid);
return mlx5_cmd_exec(dev, srq_in, sizeof(srq_in),
srq_out, sizeof(srq_out));
}
static int arm_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
u16 lwm, int is_srq)
{
u32 srq_in[MLX5_ST_SZ_DW(arm_rq_in)] = {0};
u32 srq_out[MLX5_ST_SZ_DW(arm_rq_out)] = {0};
MLX5_SET(arm_rq_in, srq_in, opcode, MLX5_CMD_OP_ARM_RQ);
MLX5_SET(arm_rq_in, srq_in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_SRQ);
MLX5_SET(arm_rq_in, srq_in, srq_number, srq->srqn);
MLX5_SET(arm_rq_in, srq_in, lwm, lwm);
MLX5_SET(arm_rq_in, srq_in, uid, srq->uid);
return mlx5_cmd_exec(dev, srq_in, sizeof(srq_in),
srq_out, sizeof(srq_out));
}
static int query_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
struct mlx5_srq_attr *out)
{
u32 srq_in[MLX5_ST_SZ_DW(query_srq_in)] = {0};
u32 *srq_out;
void *srqc;
int err;
srq_out = kvzalloc(MLX5_ST_SZ_BYTES(query_srq_out), GFP_KERNEL);
if (!srq_out)
return -ENOMEM;
MLX5_SET(query_srq_in, srq_in, opcode,
MLX5_CMD_OP_QUERY_SRQ);
MLX5_SET(query_srq_in, srq_in, srqn, srq->srqn);
err = mlx5_cmd_exec(dev, srq_in, sizeof(srq_in),
srq_out, MLX5_ST_SZ_BYTES(query_srq_out));
if (err)
goto out;
srqc = MLX5_ADDR_OF(query_srq_out, srq_out, srq_context_entry);
get_srqc(srqc, out);
if (MLX5_GET(srqc, srqc, state) != MLX5_SRQC_STATE_GOOD)
out->flags |= MLX5_SRQ_FLAG_ERR;
out:
kvfree(srq_out);
return err;
}
static int create_xrc_srq_cmd(struct mlx5_core_dev *dev,
struct mlx5_core_srq *srq,
struct mlx5_srq_attr *in)
{
u32 create_out[MLX5_ST_SZ_DW(create_xrc_srq_out)];
void *create_in;
void *xrc_srqc;
void *pas;
int pas_size;
int inlen;
int err;
pas_size = get_pas_size(in);
inlen = MLX5_ST_SZ_BYTES(create_xrc_srq_in) + pas_size;
create_in = kvzalloc(inlen, GFP_KERNEL);
if (!create_in)
return -ENOMEM;
MLX5_SET(create_xrc_srq_in, create_in, uid, in->uid);
xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, create_in,
xrc_srq_context_entry);
pas = MLX5_ADDR_OF(create_xrc_srq_in, create_in, pas);
set_srqc(xrc_srqc, in);
MLX5_SET(xrc_srqc, xrc_srqc, user_index, in->user_index);
memcpy(pas, in->pas, pas_size);
MLX5_SET(create_xrc_srq_in, create_in, opcode,
MLX5_CMD_OP_CREATE_XRC_SRQ);
memset(create_out, 0, sizeof(create_out));
err = mlx5_cmd_exec(dev, create_in, inlen, create_out,
sizeof(create_out));
if (err)
goto out;
srq->srqn = MLX5_GET(create_xrc_srq_out, create_out, xrc_srqn);
srq->uid = in->uid;
out:
kvfree(create_in);
return err;
}
static int destroy_xrc_srq_cmd(struct mlx5_core_dev *dev,
struct mlx5_core_srq *srq)
{
u32 xrcsrq_in[MLX5_ST_SZ_DW(destroy_xrc_srq_in)] = {0};
u32 xrcsrq_out[MLX5_ST_SZ_DW(destroy_xrc_srq_out)] = {0};
MLX5_SET(destroy_xrc_srq_in, xrcsrq_in, opcode,
MLX5_CMD_OP_DESTROY_XRC_SRQ);
MLX5_SET(destroy_xrc_srq_in, xrcsrq_in, xrc_srqn, srq->srqn);
MLX5_SET(destroy_xrc_srq_in, xrcsrq_in, uid, srq->uid);
return mlx5_cmd_exec(dev, xrcsrq_in, sizeof(xrcsrq_in),
xrcsrq_out, sizeof(xrcsrq_out));
}
static int arm_xrc_srq_cmd(struct mlx5_core_dev *dev,
struct mlx5_core_srq *srq, u16 lwm)
{
u32 xrcsrq_in[MLX5_ST_SZ_DW(arm_xrc_srq_in)] = {0};
u32 xrcsrq_out[MLX5_ST_SZ_DW(arm_xrc_srq_out)] = {0};
MLX5_SET(arm_xrc_srq_in, xrcsrq_in, opcode, MLX5_CMD_OP_ARM_XRC_SRQ);
MLX5_SET(arm_xrc_srq_in, xrcsrq_in, op_mod, MLX5_ARM_XRC_SRQ_IN_OP_MOD_XRC_SRQ);
MLX5_SET(arm_xrc_srq_in, xrcsrq_in, xrc_srqn, srq->srqn);
MLX5_SET(arm_xrc_srq_in, xrcsrq_in, lwm, lwm);
MLX5_SET(arm_xrc_srq_in, xrcsrq_in, uid, srq->uid);
return mlx5_cmd_exec(dev, xrcsrq_in, sizeof(xrcsrq_in),
xrcsrq_out, sizeof(xrcsrq_out));
}
static int query_xrc_srq_cmd(struct mlx5_core_dev *dev,
struct mlx5_core_srq *srq,
struct mlx5_srq_attr *out)
{
u32 xrcsrq_in[MLX5_ST_SZ_DW(query_xrc_srq_in)];
u32 *xrcsrq_out;
void *xrc_srqc;
int err;
xrcsrq_out = kvzalloc(MLX5_ST_SZ_BYTES(query_xrc_srq_out), GFP_KERNEL);
if (!xrcsrq_out)
return -ENOMEM;
memset(xrcsrq_in, 0, sizeof(xrcsrq_in));
MLX5_SET(query_xrc_srq_in, xrcsrq_in, opcode,
MLX5_CMD_OP_QUERY_XRC_SRQ);
MLX5_SET(query_xrc_srq_in, xrcsrq_in, xrc_srqn, srq->srqn);
err = mlx5_cmd_exec(dev, xrcsrq_in, sizeof(xrcsrq_in), xrcsrq_out,
MLX5_ST_SZ_BYTES(query_xrc_srq_out));
if (err)
goto out;
xrc_srqc = MLX5_ADDR_OF(query_xrc_srq_out, xrcsrq_out,
xrc_srq_context_entry);
get_srqc(xrc_srqc, out);
if (MLX5_GET(xrc_srqc, xrc_srqc, state) != MLX5_XRC_SRQC_STATE_GOOD)
out->flags |= MLX5_SRQ_FLAG_ERR;
out:
kvfree(xrcsrq_out);
return err;
}
static int create_rmp_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
struct mlx5_srq_attr *in)
{
void *create_in;
void *rmpc;
void *wq;
int pas_size;
int inlen;
int err;
pas_size = get_pas_size(in);
inlen = MLX5_ST_SZ_BYTES(create_rmp_in) + pas_size;
create_in = kvzalloc(inlen, GFP_KERNEL);
if (!create_in)
return -ENOMEM;
rmpc = MLX5_ADDR_OF(create_rmp_in, create_in, ctx);
wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY);
MLX5_SET(create_rmp_in, create_in, uid, in->uid);
set_wq(wq, in);
memcpy(MLX5_ADDR_OF(rmpc, rmpc, wq.pas), in->pas, pas_size);
err = mlx5_core_create_rmp(dev, create_in, inlen, &srq->srqn);
if (!err)
srq->uid = in->uid;
kvfree(create_in);
return err;
}
static int destroy_rmp_cmd(struct mlx5_core_dev *dev,
struct mlx5_core_srq *srq)
{
u32 in[MLX5_ST_SZ_DW(destroy_rmp_in)] = {};
u32 out[MLX5_ST_SZ_DW(destroy_rmp_out)] = {};
MLX5_SET(destroy_rmp_in, in, opcode, MLX5_CMD_OP_DESTROY_RMP);
MLX5_SET(destroy_rmp_in, in, rmpn, srq->srqn);
MLX5_SET(destroy_rmp_in, in, uid, srq->uid);
return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
static int arm_rmp_cmd(struct mlx5_core_dev *dev,
struct mlx5_core_srq *srq,
u16 lwm)
{
void *in;
void *rmpc;
void *wq;
void *bitmask;
int err;
in = kvzalloc(MLX5_ST_SZ_BYTES(modify_rmp_in), GFP_KERNEL);
if (!in)
return -ENOMEM;
rmpc = MLX5_ADDR_OF(modify_rmp_in, in, ctx);
bitmask = MLX5_ADDR_OF(modify_rmp_in, in, bitmask);
wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
MLX5_SET(modify_rmp_in, in, rmp_state, MLX5_RMPC_STATE_RDY);
MLX5_SET(modify_rmp_in, in, rmpn, srq->srqn);
MLX5_SET(modify_rmp_in, in, uid, srq->uid);
MLX5_SET(wq, wq, lwm, lwm);
MLX5_SET(rmp_bitmask, bitmask, lwm, 1);
MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY);
err = mlx5_core_modify_rmp(dev, in, MLX5_ST_SZ_BYTES(modify_rmp_in));
kvfree(in);
return err;
}
static int query_rmp_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
struct mlx5_srq_attr *out)
{
u32 *rmp_out;
void *rmpc;
int err;
rmp_out = kvzalloc(MLX5_ST_SZ_BYTES(query_rmp_out), GFP_KERNEL);
if (!rmp_out)
return -ENOMEM;
err = mlx5_core_query_rmp(dev, srq->srqn, rmp_out);
if (err)
goto out;
rmpc = MLX5_ADDR_OF(query_rmp_out, rmp_out, rmp_context);
get_wq(MLX5_ADDR_OF(rmpc, rmpc, wq), out);
if (MLX5_GET(rmpc, rmpc, state) != MLX5_RMPC_STATE_RDY)
out->flags |= MLX5_SRQ_FLAG_ERR;
out:
kvfree(rmp_out);
return err;
}
static int create_xrq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
struct mlx5_srq_attr *in)
{
u32 create_out[MLX5_ST_SZ_DW(create_xrq_out)] = {0};
void *create_in;
void *xrqc;
void *wq;
int pas_size;
int inlen;
int err;
pas_size = get_pas_size(in);
inlen = MLX5_ST_SZ_BYTES(create_xrq_in) + pas_size;
create_in = kvzalloc(inlen, GFP_KERNEL);
if (!create_in)
return -ENOMEM;
xrqc = MLX5_ADDR_OF(create_xrq_in, create_in, xrq_context);
wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
set_wq(wq, in);
memcpy(MLX5_ADDR_OF(xrqc, xrqc, wq.pas), in->pas, pas_size);
if (in->type == IB_SRQT_TM) {
MLX5_SET(xrqc, xrqc, topology, MLX5_XRQC_TOPOLOGY_TAG_MATCHING);
if (in->flags & MLX5_SRQ_FLAG_RNDV)
MLX5_SET(xrqc, xrqc, offload, MLX5_XRQC_OFFLOAD_RNDV);
MLX5_SET(xrqc, xrqc,
tag_matching_topology_context.log_matching_list_sz,
in->tm_log_list_size);
}
MLX5_SET(xrqc, xrqc, user_index, in->user_index);
MLX5_SET(xrqc, xrqc, cqn, in->cqn);
MLX5_SET(create_xrq_in, create_in, opcode, MLX5_CMD_OP_CREATE_XRQ);
MLX5_SET(create_xrq_in, create_in, uid, in->uid);
err = mlx5_cmd_exec(dev, create_in, inlen, create_out,
sizeof(create_out));
kvfree(create_in);
if (!err) {
srq->srqn = MLX5_GET(create_xrq_out, create_out, xrqn);
srq->uid = in->uid;
}
return err;
}
static int destroy_xrq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq)
{
u32 in[MLX5_ST_SZ_DW(destroy_xrq_in)] = {0};
u32 out[MLX5_ST_SZ_DW(destroy_xrq_out)] = {0};
MLX5_SET(destroy_xrq_in, in, opcode, MLX5_CMD_OP_DESTROY_XRQ);
MLX5_SET(destroy_xrq_in, in, xrqn, srq->srqn);
MLX5_SET(destroy_xrq_in, in, uid, srq->uid);
return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
static int arm_xrq_cmd(struct mlx5_core_dev *dev,
struct mlx5_core_srq *srq,
u16 lwm)
{
u32 out[MLX5_ST_SZ_DW(arm_rq_out)] = {0};
u32 in[MLX5_ST_SZ_DW(arm_rq_in)] = {0};
MLX5_SET(arm_rq_in, in, opcode, MLX5_CMD_OP_ARM_RQ);
MLX5_SET(arm_rq_in, in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_XRQ);
MLX5_SET(arm_rq_in, in, srq_number, srq->srqn);
MLX5_SET(arm_rq_in, in, lwm, lwm);
MLX5_SET(arm_rq_in, in, uid, srq->uid);
return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
static int query_xrq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
struct mlx5_srq_attr *out)
{
u32 in[MLX5_ST_SZ_DW(query_xrq_in)] = {0};
u32 *xrq_out;
int outlen = MLX5_ST_SZ_BYTES(query_xrq_out);
void *xrqc;
int err;
xrq_out = kvzalloc(outlen, GFP_KERNEL);
if (!xrq_out)
return -ENOMEM;
MLX5_SET(query_xrq_in, in, opcode, MLX5_CMD_OP_QUERY_XRQ);
MLX5_SET(query_xrq_in, in, xrqn, srq->srqn);
err = mlx5_cmd_exec(dev, in, sizeof(in), xrq_out, outlen);
if (err)
goto out;
xrqc = MLX5_ADDR_OF(query_xrq_out, xrq_out, xrq_context);
get_wq(MLX5_ADDR_OF(xrqc, xrqc, wq), out);
if (MLX5_GET(xrqc, xrqc, state) != MLX5_XRQC_STATE_GOOD)
out->flags |= MLX5_SRQ_FLAG_ERR;
out->tm_next_tag =
MLX5_GET(xrqc, xrqc,
tag_matching_topology_context.append_next_index);
out->tm_hw_phase_cnt =
MLX5_GET(xrqc, xrqc,
tag_matching_topology_context.hw_phase_cnt);
out->tm_sw_phase_cnt =
MLX5_GET(xrqc, xrqc,
tag_matching_topology_context.sw_phase_cnt);
out:
kvfree(xrq_out);
return err;
}
static int create_srq_split(struct mlx5_core_dev *dev,
struct mlx5_core_srq *srq,
struct mlx5_srq_attr *in)
{
if (!dev->issi)
return create_srq_cmd(dev, srq, in);
switch (srq->common.res) {
case MLX5_RES_XSRQ:
return create_xrc_srq_cmd(dev, srq, in);
case MLX5_RES_XRQ:
return create_xrq_cmd(dev, srq, in);
default:
return create_rmp_cmd(dev, srq, in);
}
}
static int destroy_srq_split(struct mlx5_core_dev *dev,
struct mlx5_core_srq *srq)
{
if (!dev->issi)
return destroy_srq_cmd(dev, srq);
switch (srq->common.res) {
case MLX5_RES_XSRQ:
return destroy_xrc_srq_cmd(dev, srq);
case MLX5_RES_XRQ:
return destroy_xrq_cmd(dev, srq);
default:
return destroy_rmp_cmd(dev, srq);
}
}
int mlx5_core_create_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
struct mlx5_srq_attr *in)
{
int err;
struct mlx5_srq_table *table = &dev->priv.srq_table;
switch (in->type) {
case IB_SRQT_XRC:
srq->common.res = MLX5_RES_XSRQ;
break;
case IB_SRQT_TM:
srq->common.res = MLX5_RES_XRQ;
break;
default:
srq->common.res = MLX5_RES_SRQ;
}
err = create_srq_split(dev, srq, in);
if (err)
return err;
atomic_set(&srq->refcount, 1);
init_completion(&srq->free);
spin_lock_irq(&table->lock);
err = radix_tree_insert(&table->tree, srq->srqn, srq);
spin_unlock_irq(&table->lock);
if (err)
goto err_destroy_srq_split;
return 0;
err_destroy_srq_split:
destroy_srq_split(dev, srq);
return err;
}
EXPORT_SYMBOL(mlx5_core_create_srq);
int mlx5_core_destroy_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq)
{
struct mlx5_srq_table *table = &dev->priv.srq_table;
struct mlx5_core_srq *tmp;
int err;
spin_lock_irq(&table->lock);
tmp = radix_tree_delete(&table->tree, srq->srqn);
spin_unlock_irq(&table->lock);
if (!tmp || tmp != srq)
return -EINVAL;
err = destroy_srq_split(dev, srq);
if (err)
return err;
if (atomic_dec_and_test(&srq->refcount))
complete(&srq->free);
wait_for_completion(&srq->free);
return 0;
}
EXPORT_SYMBOL(mlx5_core_destroy_srq);
int mlx5_core_query_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
struct mlx5_srq_attr *out)
{
if (!dev->issi)
return query_srq_cmd(dev, srq, out);
switch (srq->common.res) {
case MLX5_RES_XSRQ:
return query_xrc_srq_cmd(dev, srq, out);
case MLX5_RES_XRQ:
return query_xrq_cmd(dev, srq, out);
default:
return query_rmp_cmd(dev, srq, out);
}
}
EXPORT_SYMBOL(mlx5_core_query_srq);
int mlx5_core_arm_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
u16 lwm, int is_srq)
{
if (!dev->issi)
return arm_srq_cmd(dev, srq, lwm, is_srq);
switch (srq->common.res) {
case MLX5_RES_XSRQ:
return arm_xrc_srq_cmd(dev, srq, lwm);
case MLX5_RES_XRQ:
return arm_xrq_cmd(dev, srq, lwm);
default:
return arm_rmp_cmd(dev, srq, lwm);
}
}
EXPORT_SYMBOL(mlx5_core_arm_srq);
void mlx5_init_srq_table(struct mlx5_core_dev *dev)
{
struct mlx5_srq_table *table = &dev->priv.srq_table;

43
drivers/net/ethernet/mellanox/mlx5/core/transobj.c
View File

@ -258,49 +258,6 @@ void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn)
}
EXPORT_SYMBOL(mlx5_core_destroy_tis);
int mlx5_core_create_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen,
u32 *rmpn)
{
u32 out[MLX5_ST_SZ_DW(create_rmp_out)] = {0};
int err;
MLX5_SET(create_rmp_in, in, opcode, MLX5_CMD_OP_CREATE_RMP);
err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
if (!err)
*rmpn = MLX5_GET(create_rmp_out, out, rmpn);
return err;
}
int mlx5_core_modify_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen)
{
u32 out[MLX5_ST_SZ_DW(modify_rmp_out)] = {0};
MLX5_SET(modify_rmp_in, in, opcode, MLX5_CMD_OP_MODIFY_RMP);
return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
}
int mlx5_core_destroy_rmp(struct mlx5_core_dev *dev, u32 rmpn)
{
u32 in[MLX5_ST_SZ_DW(destroy_rmp_in)] = {0};
u32 out[MLX5_ST_SZ_DW(destroy_rmp_out)] = {0};
MLX5_SET(destroy_rmp_in, in, opcode, MLX5_CMD_OP_DESTROY_RMP);
MLX5_SET(destroy_rmp_in, in, rmpn, rmpn);
return mlx5_cmd_exec(dev, in, sizeof(in), out,
sizeof(out));
}
int mlx5_core_query_rmp(struct mlx5_core_dev *dev, u32 rmpn, u32 *out)
{
u32 in[MLX5_ST_SZ_DW(query_rmp_in)] = {0};
int outlen = MLX5_ST_SZ_BYTES(query_rmp_out);
MLX5_SET(query_rmp_in, in, opcode, MLX5_CMD_OP_QUERY_RMP);
MLX5_SET(query_rmp_in, in, rmpn, rmpn);
return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
}
int mlx5_core_create_rqt(struct mlx5_core_dev *dev, u32 *in, int inlen,
u32 *rqtn)
{

8
include/linux/mlx5/driver.h
View File

@ -904,13 +904,6 @@ struct mlx5_cmd_mailbox *mlx5_alloc_cmd_mailbox_chain(struct mlx5_core_dev *dev,
gfp_t flags, int npages);
void mlx5_free_cmd_mailbox_chain(struct mlx5_core_dev *dev,
struct mlx5_cmd_mailbox *head);
int mlx5_core_create_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
struct mlx5_srq_attr *in);
int mlx5_core_destroy_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq);
int mlx5_core_query_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
struct mlx5_srq_attr *out);
int mlx5_core_arm_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
u16 lwm, int is_srq);
void mlx5_init_mkey_table(struct mlx5_core_dev *dev);
void mlx5_cleanup_mkey_table(struct mlx5_core_dev *dev);
int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev,
@ -942,7 +935,6 @@ void mlx5_unregister_debugfs(void);
void mlx5_fill_page_array(struct mlx5_frag_buf *buf, __be64 *pas);
void mlx5_fill_page_frag_array(struct mlx5_frag_buf *frag_buf, __be64 *pas);
struct mlx5_core_srq *mlx5_core_get_srq(struct mlx5_core_dev *dev, u32 srqn);
int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn,
unsigned int *irqn);
int mlx5_core_attach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid, u32 qpn);

31
include/linux/mlx5/srq.h
View File

@ -6,37 +6,6 @@
#ifndef MLX5_SRQ_H
#define MLX5_SRQ_H
#include <linux/mlx5/driver.h>
enum {
MLX5_SRQ_FLAG_ERR = (1 << 0),
MLX5_SRQ_FLAG_WQ_SIG = (1 << 1),
MLX5_SRQ_FLAG_RNDV = (1 << 2),
};
struct mlx5_srq_attr {
u32 type;
u32 flags;
u32 log_size;
u32 wqe_shift;
u32 log_page_size;
u32 wqe_cnt;
u32 srqn;
u32 xrcd;
u32 page_offset;
u32 cqn;
u32 pd;
u32 lwm;
u32 user_index;
u64 db_record;
__be64 *pas;
u32 tm_log_list_size;
u32 tm_next_tag;
u32 tm_hw_phase_cnt;
u32 tm_sw_phase_cnt;
u16 uid;
};
struct mlx5_core_dev;
void mlx5_init_srq_table(struct mlx5_core_dev *dev);

6
include/linux/mlx5/transobj.h
View File

@ -58,12 +58,6 @@ int mlx5_core_create_tis(struct mlx5_core_dev *dev, u32 *in, int inlen,
int mlx5_core_modify_tis(struct mlx5_core_dev *dev, u32 tisn, u32 *in,
int inlen);
void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn);
int mlx5_core_create_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen,
u32 *rmpn);
int mlx5_core_modify_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen);
int mlx5_core_destroy_rmp(struct mlx5_core_dev *dev, u32 rmpn);
int mlx5_core_query_rmp(struct mlx5_core_dev *dev, u32 rmpn, u32 *out);
int mlx5_core_create_rqt(struct mlx5_core_dev *dev, u32 *in, int inlen,
u32 *rqtn);
int mlx5_core_modify_rqt(struct mlx5_core_dev *dev, u32 rqtn, u32 *in,