mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-24 08:04:06 +07:00
1d063c1729
The iscsi class currently does not support writable sysfs attrs for LLD sysfs settings. This patch converts the iscsi class and driver's session attrs to use the attribute container sysfs group and the sysfs group's is_visible callout to be able to support readable or writable sysfs attrs. Signed-off-by: Mike Christie <michaelc@cs.wisc.edu> Signed-off-by: James Bottomley <JBottomley@Parallels.com>
1417 lines
41 KiB
C
1417 lines
41 KiB
C
/*
|
|
* cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management
|
|
*
|
|
* Copyright (C) 2003-2008 Chelsio Communications. All rights reserved.
|
|
*
|
|
* This program is distributed in the hope that it will be useful, but WITHOUT
|
|
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
* FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
|
|
* release for licensing terms and conditions.
|
|
*
|
|
* Written by: Dimitris Michailidis (dm@chelsio.com)
|
|
* Karen Xie (kxie@chelsio.com)
|
|
*/
|
|
|
|
#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
|
|
|
|
#include <linux/version.h>
|
|
#include <linux/module.h>
|
|
#include <linux/moduleparam.h>
|
|
#include <scsi/scsi_host.h>
|
|
|
|
#include "common.h"
|
|
#include "t3_cpl.h"
|
|
#include "t3cdev.h"
|
|
#include "cxgb3_defs.h"
|
|
#include "cxgb3_ctl_defs.h"
|
|
#include "cxgb3_offload.h"
|
|
#include "firmware_exports.h"
|
|
#include "cxgb3i.h"
|
|
|
|
static unsigned int dbg_level;
|
|
#include "../libcxgbi.h"
|
|
|
|
#define DRV_MODULE_NAME "cxgb3i"
|
|
#define DRV_MODULE_DESC "Chelsio T3 iSCSI Driver"
|
|
#define DRV_MODULE_VERSION "2.0.0"
|
|
#define DRV_MODULE_RELDATE "Jun. 2010"
|
|
|
|
static char version[] =
|
|
DRV_MODULE_DESC " " DRV_MODULE_NAME
|
|
" v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
|
|
|
|
MODULE_AUTHOR("Chelsio Communications, Inc.");
|
|
MODULE_DESCRIPTION(DRV_MODULE_DESC);
|
|
MODULE_VERSION(DRV_MODULE_VERSION);
|
|
MODULE_LICENSE("GPL");
|
|
|
|
module_param(dbg_level, uint, 0644);
|
|
MODULE_PARM_DESC(dbg_level, "debug flag (default=0)");
|
|
|
|
static int cxgb3i_rcv_win = 256 * 1024;
|
|
module_param(cxgb3i_rcv_win, int, 0644);
|
|
MODULE_PARM_DESC(cxgb3i_rcv_win, "TCP receive window in bytes (default=256KB)");
|
|
|
|
static int cxgb3i_snd_win = 128 * 1024;
|
|
module_param(cxgb3i_snd_win, int, 0644);
|
|
MODULE_PARM_DESC(cxgb3i_snd_win, "TCP send window in bytes (default=128KB)");
|
|
|
|
static int cxgb3i_rx_credit_thres = 10 * 1024;
|
|
module_param(cxgb3i_rx_credit_thres, int, 0644);
|
|
MODULE_PARM_DESC(rx_credit_thres,
|
|
"RX credits return threshold in bytes (default=10KB)");
|
|
|
|
static unsigned int cxgb3i_max_connect = 8 * 1024;
|
|
module_param(cxgb3i_max_connect, uint, 0644);
|
|
MODULE_PARM_DESC(cxgb3i_max_connect, "Max. # of connections (default=8092)");
|
|
|
|
static unsigned int cxgb3i_sport_base = 20000;
|
|
module_param(cxgb3i_sport_base, uint, 0644);
|
|
MODULE_PARM_DESC(cxgb3i_sport_base, "starting port number (default=20000)");
|
|
|
|
static void cxgb3i_dev_open(struct t3cdev *);
|
|
static void cxgb3i_dev_close(struct t3cdev *);
|
|
static void cxgb3i_dev_event_handler(struct t3cdev *, u32, u32);
|
|
|
|
static struct cxgb3_client t3_client = {
|
|
.name = DRV_MODULE_NAME,
|
|
.handlers = cxgb3i_cpl_handlers,
|
|
.add = cxgb3i_dev_open,
|
|
.remove = cxgb3i_dev_close,
|
|
.event_handler = cxgb3i_dev_event_handler,
|
|
};
|
|
|
|
static struct scsi_host_template cxgb3i_host_template = {
|
|
.module = THIS_MODULE,
|
|
.name = DRV_MODULE_NAME,
|
|
.proc_name = DRV_MODULE_NAME,
|
|
.can_queue = CXGB3I_SCSI_HOST_QDEPTH,
|
|
.queuecommand = iscsi_queuecommand,
|
|
.change_queue_depth = iscsi_change_queue_depth,
|
|
.sg_tablesize = SG_ALL,
|
|
.max_sectors = 0xFFFF,
|
|
.cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
|
|
.eh_abort_handler = iscsi_eh_abort,
|
|
.eh_device_reset_handler = iscsi_eh_device_reset,
|
|
.eh_target_reset_handler = iscsi_eh_recover_target,
|
|
.target_alloc = iscsi_target_alloc,
|
|
.use_clustering = DISABLE_CLUSTERING,
|
|
.this_id = -1,
|
|
};
|
|
|
|
static struct iscsi_transport cxgb3i_iscsi_transport = {
|
|
.owner = THIS_MODULE,
|
|
.name = DRV_MODULE_NAME,
|
|
/* owner and name should be set already */
|
|
.caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
|
|
| CAP_DATADGST | CAP_DIGEST_OFFLOAD |
|
|
CAP_PADDING_OFFLOAD | CAP_TEXT_NEGO,
|
|
.host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
|
|
ISCSI_HOST_INITIATOR_NAME |
|
|
ISCSI_HOST_NETDEV_NAME,
|
|
.attr_is_visible = cxgbi_attr_is_visible,
|
|
.get_host_param = cxgbi_get_host_param,
|
|
.set_host_param = cxgbi_set_host_param,
|
|
/* session management */
|
|
.create_session = cxgbi_create_session,
|
|
.destroy_session = cxgbi_destroy_session,
|
|
.get_session_param = iscsi_session_get_param,
|
|
/* connection management */
|
|
.create_conn = cxgbi_create_conn,
|
|
.bind_conn = cxgbi_bind_conn,
|
|
.destroy_conn = iscsi_tcp_conn_teardown,
|
|
.start_conn = iscsi_conn_start,
|
|
.stop_conn = iscsi_conn_stop,
|
|
.get_conn_param = iscsi_conn_get_param,
|
|
.set_param = cxgbi_set_conn_param,
|
|
.get_stats = cxgbi_get_conn_stats,
|
|
/* pdu xmit req from user space */
|
|
.send_pdu = iscsi_conn_send_pdu,
|
|
/* task */
|
|
.init_task = iscsi_tcp_task_init,
|
|
.xmit_task = iscsi_tcp_task_xmit,
|
|
.cleanup_task = cxgbi_cleanup_task,
|
|
/* pdu */
|
|
.alloc_pdu = cxgbi_conn_alloc_pdu,
|
|
.init_pdu = cxgbi_conn_init_pdu,
|
|
.xmit_pdu = cxgbi_conn_xmit_pdu,
|
|
.parse_pdu_itt = cxgbi_parse_pdu_itt,
|
|
/* TCP connect/disconnect */
|
|
.get_ep_param = cxgbi_get_ep_param,
|
|
.ep_connect = cxgbi_ep_connect,
|
|
.ep_poll = cxgbi_ep_poll,
|
|
.ep_disconnect = cxgbi_ep_disconnect,
|
|
/* Error recovery timeout call */
|
|
.session_recovery_timedout = iscsi_session_recovery_timedout,
|
|
};
|
|
|
|
static struct scsi_transport_template *cxgb3i_stt;
|
|
|
|
/*
|
|
* CPL (Chelsio Protocol Language) defines a message passing interface between
|
|
* the host driver and Chelsio asic.
|
|
* The section below implments CPLs that related to iscsi tcp connection
|
|
* open/close/abort and data send/receive.
|
|
*/
|
|
|
|
static int push_tx_frames(struct cxgbi_sock *csk, int req_completion);
|
|
|
|
static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb,
|
|
const struct l2t_entry *e)
|
|
{
|
|
unsigned int wscale = cxgbi_sock_compute_wscale(cxgb3i_rcv_win);
|
|
struct cpl_act_open_req *req = (struct cpl_act_open_req *)skb->head;
|
|
|
|
skb->priority = CPL_PRIORITY_SETUP;
|
|
|
|
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
|
|
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, csk->atid));
|
|
req->local_port = csk->saddr.sin_port;
|
|
req->peer_port = csk->daddr.sin_port;
|
|
req->local_ip = csk->saddr.sin_addr.s_addr;
|
|
req->peer_ip = csk->daddr.sin_addr.s_addr;
|
|
|
|
req->opt0h = htonl(V_KEEP_ALIVE(1) | F_TCAM_BYPASS |
|
|
V_WND_SCALE(wscale) | V_MSS_IDX(csk->mss_idx) |
|
|
V_L2T_IDX(e->idx) | V_TX_CHANNEL(e->smt_idx));
|
|
req->opt0l = htonl(V_ULP_MODE(ULP2_MODE_ISCSI) |
|
|
V_RCV_BUFSIZ(cxgb3i_rcv_win>>10));
|
|
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
|
|
"csk 0x%p,%u,0x%lx,%u, %pI4:%u-%pI4:%u, %u,%u,%u.\n",
|
|
csk, csk->state, csk->flags, csk->atid,
|
|
&req->local_ip, ntohs(req->local_port),
|
|
&req->peer_ip, ntohs(req->peer_port),
|
|
csk->mss_idx, e->idx, e->smt_idx);
|
|
|
|
l2t_send(csk->cdev->lldev, skb, csk->l2t);
|
|
}
|
|
|
|
static inline void act_open_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
|
|
{
|
|
cxgbi_sock_act_open_req_arp_failure(NULL, skb);
|
|
}
|
|
|
|
/*
|
|
* CPL connection close request: host ->
|
|
*
|
|
* Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to
|
|
* the write queue (i.e., after any unsent txt data).
|
|
*/
|
|
static void send_close_req(struct cxgbi_sock *csk)
|
|
{
|
|
struct sk_buff *skb = csk->cpl_close;
|
|
struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head;
|
|
unsigned int tid = csk->tid;
|
|
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
|
|
"csk 0x%p,%u,0x%lx,%u.\n",
|
|
csk, csk->state, csk->flags, csk->tid);
|
|
|
|
csk->cpl_close = NULL;
|
|
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
|
|
req->wr.wr_lo = htonl(V_WR_TID(tid));
|
|
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
|
|
req->rsvd = htonl(csk->write_seq);
|
|
|
|
cxgbi_sock_skb_entail(csk, skb);
|
|
if (csk->state >= CTP_ESTABLISHED)
|
|
push_tx_frames(csk, 1);
|
|
}
|
|
|
|
/*
|
|
* CPL connection abort request: host ->
|
|
*
|
|
* Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs
|
|
* for the same connection and also that we do not try to send a message
|
|
* after the connection has closed.
|
|
*/
|
|
static void abort_arp_failure(struct t3cdev *tdev, struct sk_buff *skb)
|
|
{
|
|
struct cpl_abort_req *req = cplhdr(skb);
|
|
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
|
|
"t3dev 0x%p, tid %u, skb 0x%p.\n",
|
|
tdev, GET_TID(req), skb);
|
|
req->cmd = CPL_ABORT_NO_RST;
|
|
cxgb3_ofld_send(tdev, skb);
|
|
}
|
|
|
|
static void send_abort_req(struct cxgbi_sock *csk)
|
|
{
|
|
struct sk_buff *skb = csk->cpl_abort_req;
|
|
struct cpl_abort_req *req;
|
|
|
|
if (unlikely(csk->state == CTP_ABORTING || !skb))
|
|
return;
|
|
cxgbi_sock_set_state(csk, CTP_ABORTING);
|
|
cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING);
|
|
/* Purge the send queue so we don't send anything after an abort. */
|
|
cxgbi_sock_purge_write_queue(csk);
|
|
|
|
csk->cpl_abort_req = NULL;
|
|
req = (struct cpl_abort_req *)skb->head;
|
|
skb->priority = CPL_PRIORITY_DATA;
|
|
set_arp_failure_handler(skb, abort_arp_failure);
|
|
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
|
|
req->wr.wr_lo = htonl(V_WR_TID(csk->tid));
|
|
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid));
|
|
req->rsvd0 = htonl(csk->snd_nxt);
|
|
req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT);
|
|
req->cmd = CPL_ABORT_SEND_RST;
|
|
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
|
|
"csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n",
|
|
csk, csk->state, csk->flags, csk->tid, csk->snd_nxt,
|
|
req->rsvd1);
|
|
|
|
l2t_send(csk->cdev->lldev, skb, csk->l2t);
|
|
}
|
|
|
|
/*
|
|
* CPL connection abort reply: host ->
|
|
*
|
|
* Send an ABORT_RPL message in response of the ABORT_REQ received.
|
|
*/
|
|
static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status)
|
|
{
|
|
struct sk_buff *skb = csk->cpl_abort_rpl;
|
|
struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head;
|
|
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
|
|
"csk 0x%p,%u,0x%lx,%u, status %d.\n",
|
|
csk, csk->state, csk->flags, csk->tid, rst_status);
|
|
|
|
csk->cpl_abort_rpl = NULL;
|
|
skb->priority = CPL_PRIORITY_DATA;
|
|
rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
|
|
rpl->wr.wr_lo = htonl(V_WR_TID(csk->tid));
|
|
OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid));
|
|
rpl->cmd = rst_status;
|
|
cxgb3_ofld_send(csk->cdev->lldev, skb);
|
|
}
|
|
|
|
/*
|
|
* CPL connection rx data ack: host ->
|
|
* Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
|
|
* credits sent.
|
|
*/
|
|
static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits)
|
|
{
|
|
struct sk_buff *skb;
|
|
struct cpl_rx_data_ack *req;
|
|
u32 dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
|
|
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
|
|
"csk 0x%p,%u,0x%lx,%u, credit %u, dack %u.\n",
|
|
csk, csk->state, csk->flags, csk->tid, credits, dack);
|
|
|
|
skb = alloc_wr(sizeof(*req), 0, GFP_ATOMIC);
|
|
if (!skb) {
|
|
pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits);
|
|
return 0;
|
|
}
|
|
req = (struct cpl_rx_data_ack *)skb->head;
|
|
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
|
|
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, csk->tid));
|
|
req->credit_dack = htonl(F_RX_DACK_CHANGE | V_RX_DACK_MODE(1) |
|
|
V_RX_CREDITS(credits));
|
|
skb->priority = CPL_PRIORITY_ACK;
|
|
cxgb3_ofld_send(csk->cdev->lldev, skb);
|
|
return credits;
|
|
}
|
|
|
|
/*
|
|
* CPL connection tx data: host ->
|
|
*
|
|
* Send iscsi PDU via TX_DATA CPL message. Returns the number of
|
|
* credits sent.
|
|
* Each TX_DATA consumes work request credit (wrs), so we need to keep track of
|
|
* how many we've used so far and how many are pending (i.e., yet ack'ed by T3).
|
|
*/
|
|
|
|
static unsigned int wrlen __read_mostly;
|
|
static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly;
|
|
|
|
static void init_wr_tab(unsigned int wr_len)
|
|
{
|
|
int i;
|
|
|
|
if (skb_wrs[1]) /* already initialized */
|
|
return;
|
|
for (i = 1; i < SKB_WR_LIST_SIZE; i++) {
|
|
int sgl_len = (3 * i) / 2 + (i & 1);
|
|
|
|
sgl_len += 3;
|
|
skb_wrs[i] = (sgl_len <= wr_len
|
|
? 1 : 1 + (sgl_len - 2) / (wr_len - 1));
|
|
}
|
|
wrlen = wr_len * 8;
|
|
}
|
|
|
|
static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb,
|
|
int len, int req_completion)
|
|
{
|
|
struct tx_data_wr *req;
|
|
struct l2t_entry *l2t = csk->l2t;
|
|
|
|
skb_reset_transport_header(skb);
|
|
req = (struct tx_data_wr *)__skb_push(skb, sizeof(*req));
|
|
req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA) |
|
|
(req_completion ? F_WR_COMPL : 0));
|
|
req->wr_lo = htonl(V_WR_TID(csk->tid));
|
|
/* len includes the length of any HW ULP additions */
|
|
req->len = htonl(len);
|
|
/* V_TX_ULP_SUBMODE sets both the mode and submode */
|
|
req->flags = htonl(V_TX_ULP_SUBMODE(cxgbi_skcb_ulp_mode(skb)) |
|
|
V_TX_SHOVE((skb_peek(&csk->write_queue) ? 0 : 1)));
|
|
req->sndseq = htonl(csk->snd_nxt);
|
|
req->param = htonl(V_TX_PORT(l2t->smt_idx));
|
|
|
|
if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) {
|
|
req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
|
|
V_TX_CPU_IDX(csk->rss_qid));
|
|
/* sendbuffer is in units of 32KB. */
|
|
req->param |= htonl(V_TX_SNDBUF(cxgb3i_snd_win >> 15));
|
|
cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* push_tx_frames -- start transmit
|
|
* @c3cn: the offloaded connection
|
|
* @req_completion: request wr_ack or not
|
|
*
|
|
* Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
|
|
* connection's send queue and sends them on to T3. Must be called with the
|
|
* connection's lock held. Returns the amount of send buffer space that was
|
|
* freed as a result of sending queued data to T3.
|
|
*/
|
|
|
|
static void arp_failure_skb_discard(struct t3cdev *dev, struct sk_buff *skb)
|
|
{
|
|
kfree_skb(skb);
|
|
}
|
|
|
|
static int push_tx_frames(struct cxgbi_sock *csk, int req_completion)
|
|
{
|
|
int total_size = 0;
|
|
struct sk_buff *skb;
|
|
|
|
if (unlikely(csk->state < CTP_ESTABLISHED ||
|
|
csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) {
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
|
|
"csk 0x%p,%u,0x%lx,%u, in closing state.\n",
|
|
csk, csk->state, csk->flags, csk->tid);
|
|
return 0;
|
|
}
|
|
|
|
while (csk->wr_cred && (skb = skb_peek(&csk->write_queue)) != NULL) {
|
|
int len = skb->len; /* length before skb_push */
|
|
int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len);
|
|
int wrs_needed = skb_wrs[frags];
|
|
|
|
if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen)
|
|
wrs_needed = 1;
|
|
|
|
WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1);
|
|
|
|
if (csk->wr_cred < wrs_needed) {
|
|
log_debug(1 << CXGBI_DBG_PDU_TX,
|
|
"csk 0x%p, skb len %u/%u, frag %u, wr %d<%u.\n",
|
|
csk, skb->len, skb->data_len, frags,
|
|
wrs_needed, csk->wr_cred);
|
|
break;
|
|
}
|
|
|
|
__skb_unlink(skb, &csk->write_queue);
|
|
skb->priority = CPL_PRIORITY_DATA;
|
|
skb->csum = wrs_needed; /* remember this until the WR_ACK */
|
|
csk->wr_cred -= wrs_needed;
|
|
csk->wr_una_cred += wrs_needed;
|
|
cxgbi_sock_enqueue_wr(csk, skb);
|
|
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
|
|
"csk 0x%p, enqueue, skb len %u/%u, frag %u, wr %d, "
|
|
"left %u, unack %u.\n",
|
|
csk, skb->len, skb->data_len, frags, skb->csum,
|
|
csk->wr_cred, csk->wr_una_cred);
|
|
|
|
if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) {
|
|
if ((req_completion &&
|
|
csk->wr_una_cred == wrs_needed) ||
|
|
csk->wr_una_cred >= csk->wr_max_cred / 2) {
|
|
req_completion = 1;
|
|
csk->wr_una_cred = 0;
|
|
}
|
|
len += cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb));
|
|
make_tx_data_wr(csk, skb, len, req_completion);
|
|
csk->snd_nxt += len;
|
|
cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR);
|
|
}
|
|
total_size += skb->truesize;
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
|
|
"csk 0x%p, tid 0x%x, send skb 0x%p.\n",
|
|
csk, csk->tid, skb);
|
|
set_arp_failure_handler(skb, arp_failure_skb_discard);
|
|
l2t_send(csk->cdev->lldev, skb, csk->l2t);
|
|
}
|
|
return total_size;
|
|
}
|
|
|
|
/*
|
|
* Process a CPL_ACT_ESTABLISH message: -> host
|
|
* Updates connection state from an active establish CPL message. Runs with
|
|
* the connection lock held.
|
|
*/
|
|
|
|
static inline void free_atid(struct cxgbi_sock *csk)
|
|
{
|
|
if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) {
|
|
cxgb3_free_atid(csk->cdev->lldev, csk->atid);
|
|
cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID);
|
|
cxgbi_sock_put(csk);
|
|
}
|
|
}
|
|
|
|
static int do_act_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
|
|
{
|
|
struct cxgbi_sock *csk = ctx;
|
|
struct cpl_act_establish *req = cplhdr(skb);
|
|
unsigned int tid = GET_TID(req);
|
|
unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
|
|
u32 rcv_isn = ntohl(req->rcv_isn); /* real RCV_ISN + 1 */
|
|
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
|
|
"atid 0x%x,tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n",
|
|
atid, atid, csk, csk->state, csk->flags, rcv_isn);
|
|
|
|
cxgbi_sock_get(csk);
|
|
cxgbi_sock_set_flag(csk, CTPF_HAS_TID);
|
|
csk->tid = tid;
|
|
cxgb3_insert_tid(csk->cdev->lldev, &t3_client, csk, tid);
|
|
|
|
free_atid(csk);
|
|
|
|
csk->rss_qid = G_QNUM(ntohs(skb->csum));
|
|
|
|
spin_lock_bh(&csk->lock);
|
|
if (csk->retry_timer.function) {
|
|
del_timer(&csk->retry_timer);
|
|
csk->retry_timer.function = NULL;
|
|
}
|
|
|
|
if (unlikely(csk->state != CTP_ACTIVE_OPEN))
|
|
pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n",
|
|
csk, csk->state, csk->flags, csk->tid);
|
|
|
|
csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn;
|
|
if (cxgb3i_rcv_win > (M_RCV_BUFSIZ << 10))
|
|
csk->rcv_wup -= cxgb3i_rcv_win - (M_RCV_BUFSIZ << 10);
|
|
|
|
cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt));
|
|
|
|
if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED)))
|
|
/* upper layer has requested closing */
|
|
send_abort_req(csk);
|
|
else {
|
|
if (skb_queue_len(&csk->write_queue))
|
|
push_tx_frames(csk, 1);
|
|
cxgbi_conn_tx_open(csk);
|
|
}
|
|
|
|
spin_unlock_bh(&csk->lock);
|
|
__kfree_skb(skb);
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Process a CPL_ACT_OPEN_RPL message: -> host
|
|
* Handle active open failures.
|
|
*/
|
|
static int act_open_rpl_status_to_errno(int status)
|
|
{
|
|
switch (status) {
|
|
case CPL_ERR_CONN_RESET:
|
|
return -ECONNREFUSED;
|
|
case CPL_ERR_ARP_MISS:
|
|
return -EHOSTUNREACH;
|
|
case CPL_ERR_CONN_TIMEDOUT:
|
|
return -ETIMEDOUT;
|
|
case CPL_ERR_TCAM_FULL:
|
|
return -ENOMEM;
|
|
case CPL_ERR_CONN_EXIST:
|
|
return -EADDRINUSE;
|
|
default:
|
|
return -EIO;
|
|
}
|
|
}
|
|
|
|
static void act_open_retry_timer(unsigned long data)
|
|
{
|
|
struct sk_buff *skb;
|
|
struct cxgbi_sock *csk = (struct cxgbi_sock *)data;
|
|
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
|
|
"csk 0x%p,%u,0x%lx,%u.\n",
|
|
csk, csk->state, csk->flags, csk->tid);
|
|
|
|
cxgbi_sock_get(csk);
|
|
spin_lock_bh(&csk->lock);
|
|
skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_ATOMIC);
|
|
if (!skb)
|
|
cxgbi_sock_fail_act_open(csk, -ENOMEM);
|
|
else {
|
|
skb->sk = (struct sock *)csk;
|
|
set_arp_failure_handler(skb, act_open_arp_failure);
|
|
send_act_open_req(csk, skb, csk->l2t);
|
|
}
|
|
spin_unlock_bh(&csk->lock);
|
|
cxgbi_sock_put(csk);
|
|
}
|
|
|
|
static int do_act_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
|
|
{
|
|
struct cxgbi_sock *csk = ctx;
|
|
struct cpl_act_open_rpl *rpl = cplhdr(skb);
|
|
|
|
pr_info("csk 0x%p,%u,0x%lx,%u, status %u, %pI4:%u-%pI4:%u.\n",
|
|
csk, csk->state, csk->flags, csk->atid, rpl->status,
|
|
&csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port),
|
|
&csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port));
|
|
|
|
if (rpl->status != CPL_ERR_TCAM_FULL &&
|
|
rpl->status != CPL_ERR_CONN_EXIST &&
|
|
rpl->status != CPL_ERR_ARP_MISS)
|
|
cxgb3_queue_tid_release(tdev, GET_TID(rpl));
|
|
|
|
cxgbi_sock_get(csk);
|
|
spin_lock_bh(&csk->lock);
|
|
if (rpl->status == CPL_ERR_CONN_EXIST &&
|
|
csk->retry_timer.function != act_open_retry_timer) {
|
|
csk->retry_timer.function = act_open_retry_timer;
|
|
mod_timer(&csk->retry_timer, jiffies + HZ / 2);
|
|
} else
|
|
cxgbi_sock_fail_act_open(csk,
|
|
act_open_rpl_status_to_errno(rpl->status));
|
|
|
|
spin_unlock_bh(&csk->lock);
|
|
cxgbi_sock_put(csk);
|
|
__kfree_skb(skb);
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Process PEER_CLOSE CPL messages: -> host
|
|
* Handle peer FIN.
|
|
*/
|
|
static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
|
|
{
|
|
struct cxgbi_sock *csk = ctx;
|
|
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
|
|
"csk 0x%p,%u,0x%lx,%u.\n",
|
|
csk, csk->state, csk->flags, csk->tid);
|
|
|
|
cxgbi_sock_rcv_peer_close(csk);
|
|
__kfree_skb(skb);
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Process CLOSE_CONN_RPL CPL message: -> host
|
|
* Process a peer ACK to our FIN.
|
|
*/
|
|
static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb,
|
|
void *ctx)
|
|
{
|
|
struct cxgbi_sock *csk = ctx;
|
|
struct cpl_close_con_rpl *rpl = cplhdr(skb);
|
|
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
|
|
"csk 0x%p,%u,0x%lx,%u, snxt %u.\n",
|
|
csk, csk->state, csk->flags, csk->tid, ntohl(rpl->snd_nxt));
|
|
|
|
cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt));
|
|
__kfree_skb(skb);
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Process ABORT_REQ_RSS CPL message: -> host
|
|
* Process abort requests. If we are waiting for an ABORT_RPL we ignore this
|
|
* request except that we need to reply to it.
|
|
*/
|
|
|
|
static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason,
|
|
int *need_rst)
|
|
{
|
|
switch (abort_reason) {
|
|
case CPL_ERR_BAD_SYN: /* fall through */
|
|
case CPL_ERR_CONN_RESET:
|
|
return csk->state > CTP_ESTABLISHED ? -EPIPE : -ECONNRESET;
|
|
case CPL_ERR_XMIT_TIMEDOUT:
|
|
case CPL_ERR_PERSIST_TIMEDOUT:
|
|
case CPL_ERR_FINWAIT2_TIMEDOUT:
|
|
case CPL_ERR_KEEPALIVE_TIMEDOUT:
|
|
return -ETIMEDOUT;
|
|
default:
|
|
return -EIO;
|
|
}
|
|
}
|
|
|
|
static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
|
|
{
|
|
const struct cpl_abort_req_rss *req = cplhdr(skb);
|
|
struct cxgbi_sock *csk = ctx;
|
|
int rst_status = CPL_ABORT_NO_RST;
|
|
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
|
|
"csk 0x%p,%u,0x%lx,%u.\n",
|
|
csk, csk->state, csk->flags, csk->tid);
|
|
|
|
if (req->status == CPL_ERR_RTX_NEG_ADVICE ||
|
|
req->status == CPL_ERR_PERSIST_NEG_ADVICE) {
|
|
goto done;
|
|
}
|
|
|
|
cxgbi_sock_get(csk);
|
|
spin_lock_bh(&csk->lock);
|
|
|
|
if (!cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD)) {
|
|
cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD);
|
|
cxgbi_sock_set_state(csk, CTP_ABORTING);
|
|
goto out;
|
|
}
|
|
|
|
cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD);
|
|
send_abort_rpl(csk, rst_status);
|
|
|
|
if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) {
|
|
csk->err = abort_status_to_errno(csk, req->status, &rst_status);
|
|
cxgbi_sock_closed(csk);
|
|
}
|
|
|
|
out:
|
|
spin_unlock_bh(&csk->lock);
|
|
cxgbi_sock_put(csk);
|
|
done:
|
|
__kfree_skb(skb);
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Process ABORT_RPL_RSS CPL message: -> host
|
|
* Process abort replies. We only process these messages if we anticipate
|
|
* them as the coordination between SW and HW in this area is somewhat lacking
|
|
* and sometimes we get ABORT_RPLs after we are done with the connection that
|
|
* originated the ABORT_REQ.
|
|
*/
|
|
static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
|
|
{
|
|
struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
|
|
struct cxgbi_sock *csk = ctx;
|
|
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
|
|
"status 0x%x, csk 0x%p, s %u, 0x%lx.\n",
|
|
rpl->status, csk, csk ? csk->state : 0,
|
|
csk ? csk->flags : 0UL);
|
|
/*
|
|
* Ignore replies to post-close aborts indicating that the abort was
|
|
* requested too late. These connections are terminated when we get
|
|
* PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss
|
|
* arrives the TID is either no longer used or it has been recycled.
|
|
*/
|
|
if (rpl->status == CPL_ERR_ABORT_FAILED)
|
|
goto rel_skb;
|
|
/*
|
|
* Sometimes we've already closed the connection, e.g., a post-close
|
|
* abort races with ABORT_REQ_RSS, the latter frees the connection
|
|
* expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,
|
|
* but FW turns the ABORT_REQ into a regular one and so we get
|
|
* ABORT_RPL_RSS with status 0 and no connection.
|
|
*/
|
|
if (csk)
|
|
cxgbi_sock_rcv_abort_rpl(csk);
|
|
rel_skb:
|
|
__kfree_skb(skb);
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Process RX_ISCSI_HDR CPL message: -> host
|
|
* Handle received PDUs, the payload could be DDP'ed. If not, the payload
|
|
* follow after the bhs.
|
|
*/
|
|
static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx)
|
|
{
|
|
struct cxgbi_sock *csk = ctx;
|
|
struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb);
|
|
struct cpl_iscsi_hdr_norss data_cpl;
|
|
struct cpl_rx_data_ddp_norss ddp_cpl;
|
|
unsigned int hdr_len, data_len, status;
|
|
unsigned int len;
|
|
int err;
|
|
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
|
|
"csk 0x%p,%u,0x%lx,%u, skb 0x%p,%u.\n",
|
|
csk, csk->state, csk->flags, csk->tid, skb, skb->len);
|
|
|
|
spin_lock_bh(&csk->lock);
|
|
|
|
if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) {
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
|
|
"csk 0x%p,%u,0x%lx,%u, bad state.\n",
|
|
csk, csk->state, csk->flags, csk->tid);
|
|
if (csk->state != CTP_ABORTING)
|
|
goto abort_conn;
|
|
else
|
|
goto discard;
|
|
}
|
|
|
|
cxgbi_skcb_tcp_seq(skb) = ntohl(hdr_cpl->seq);
|
|
cxgbi_skcb_flags(skb) = 0;
|
|
|
|
skb_reset_transport_header(skb);
|
|
__skb_pull(skb, sizeof(struct cpl_iscsi_hdr));
|
|
|
|
len = hdr_len = ntohs(hdr_cpl->len);
|
|
/* msg coalesce is off or not enough data received */
|
|
if (skb->len <= hdr_len) {
|
|
pr_err("%s: tid %u, CPL_ISCSI_HDR, skb len %u < %u.\n",
|
|
csk->cdev->ports[csk->port_id]->name, csk->tid,
|
|
skb->len, hdr_len);
|
|
goto abort_conn;
|
|
}
|
|
cxgbi_skcb_set_flag(skb, SKCBF_RX_COALESCED);
|
|
|
|
err = skb_copy_bits(skb, skb->len - sizeof(ddp_cpl), &ddp_cpl,
|
|
sizeof(ddp_cpl));
|
|
if (err < 0) {
|
|
pr_err("%s: tid %u, copy cpl_ddp %u-%zu failed %d.\n",
|
|
csk->cdev->ports[csk->port_id]->name, csk->tid,
|
|
skb->len, sizeof(ddp_cpl), err);
|
|
goto abort_conn;
|
|
}
|
|
|
|
cxgbi_skcb_set_flag(skb, SKCBF_RX_STATUS);
|
|
cxgbi_skcb_rx_pdulen(skb) = ntohs(ddp_cpl.len);
|
|
cxgbi_skcb_rx_ddigest(skb) = ntohl(ddp_cpl.ulp_crc);
|
|
status = ntohl(ddp_cpl.ddp_status);
|
|
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
|
|
"csk 0x%p, skb 0x%p,%u, pdulen %u, status 0x%x.\n",
|
|
csk, skb, skb->len, cxgbi_skcb_rx_pdulen(skb), status);
|
|
|
|
if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT))
|
|
cxgbi_skcb_set_flag(skb, SKCBF_RX_HCRC_ERR);
|
|
if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT))
|
|
cxgbi_skcb_set_flag(skb, SKCBF_RX_DCRC_ERR);
|
|
if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT))
|
|
cxgbi_skcb_set_flag(skb, SKCBF_RX_PAD_ERR);
|
|
|
|
if (skb->len > (hdr_len + sizeof(ddp_cpl))) {
|
|
err = skb_copy_bits(skb, hdr_len, &data_cpl, sizeof(data_cpl));
|
|
if (err < 0) {
|
|
pr_err("%s: tid %u, cp %zu/%u failed %d.\n",
|
|
csk->cdev->ports[csk->port_id]->name,
|
|
csk->tid, sizeof(data_cpl), skb->len, err);
|
|
goto abort_conn;
|
|
}
|
|
data_len = ntohs(data_cpl.len);
|
|
log_debug(1 << CXGBI_DBG_DDP | 1 << CXGBI_DBG_PDU_RX,
|
|
"skb 0x%p, pdu not ddp'ed %u/%u, status 0x%x.\n",
|
|
skb, data_len, cxgbi_skcb_rx_pdulen(skb), status);
|
|
len += sizeof(data_cpl) + data_len;
|
|
} else if (status & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT))
|
|
cxgbi_skcb_set_flag(skb, SKCBF_RX_DATA_DDPD);
|
|
|
|
csk->rcv_nxt = ntohl(ddp_cpl.seq) + cxgbi_skcb_rx_pdulen(skb);
|
|
__pskb_trim(skb, len);
|
|
__skb_queue_tail(&csk->receive_queue, skb);
|
|
cxgbi_conn_pdu_ready(csk);
|
|
|
|
spin_unlock_bh(&csk->lock);
|
|
return 0;
|
|
|
|
abort_conn:
|
|
send_abort_req(csk);
|
|
discard:
|
|
spin_unlock_bh(&csk->lock);
|
|
__kfree_skb(skb);
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Process TX_DATA_ACK CPL messages: -> host
|
|
* Process an acknowledgment of WR completion. Advance snd_una and send the
|
|
* next batch of work requests from the write queue.
|
|
*/
|
|
static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
|
|
{
|
|
struct cxgbi_sock *csk = ctx;
|
|
struct cpl_wr_ack *hdr = cplhdr(skb);
|
|
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
|
|
"csk 0x%p,%u,0x%lx,%u, cr %u.\n",
|
|
csk, csk->state, csk->flags, csk->tid, ntohs(hdr->credits));
|
|
|
|
cxgbi_sock_rcv_wr_ack(csk, ntohs(hdr->credits), ntohl(hdr->snd_una), 1);
|
|
__kfree_skb(skb);
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* for each connection, pre-allocate skbs needed for close/abort requests. So
|
|
* that we can service the request right away.
|
|
*/
|
|
static int alloc_cpls(struct cxgbi_sock *csk)
|
|
{
|
|
csk->cpl_close = alloc_wr(sizeof(struct cpl_close_con_req), 0,
|
|
GFP_KERNEL);
|
|
if (!csk->cpl_close)
|
|
return -ENOMEM;
|
|
csk->cpl_abort_req = alloc_wr(sizeof(struct cpl_abort_req), 0,
|
|
GFP_KERNEL);
|
|
if (!csk->cpl_abort_req)
|
|
goto free_cpl_skbs;
|
|
|
|
csk->cpl_abort_rpl = alloc_wr(sizeof(struct cpl_abort_rpl), 0,
|
|
GFP_KERNEL);
|
|
if (!csk->cpl_abort_rpl)
|
|
goto free_cpl_skbs;
|
|
|
|
return 0;
|
|
|
|
free_cpl_skbs:
|
|
cxgbi_sock_free_cpl_skbs(csk);
|
|
return -ENOMEM;
|
|
}
|
|
|
|
/**
|
|
* release_offload_resources - release offload resource
|
|
* @c3cn: the offloaded iscsi tcp connection.
|
|
* Release resources held by an offload connection (TID, L2T entry, etc.)
|
|
*/
|
|
static void l2t_put(struct cxgbi_sock *csk)
|
|
{
|
|
struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev;
|
|
|
|
if (csk->l2t) {
|
|
l2t_release(L2DATA(t3dev), csk->l2t);
|
|
csk->l2t = NULL;
|
|
cxgbi_sock_put(csk);
|
|
}
|
|
}
|
|
|
|
static void release_offload_resources(struct cxgbi_sock *csk)
|
|
{
|
|
struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev;
|
|
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
|
|
"csk 0x%p,%u,0x%lx,%u.\n",
|
|
csk, csk->state, csk->flags, csk->tid);
|
|
|
|
csk->rss_qid = 0;
|
|
cxgbi_sock_free_cpl_skbs(csk);
|
|
|
|
if (csk->wr_cred != csk->wr_max_cred) {
|
|
cxgbi_sock_purge_wr_queue(csk);
|
|
cxgbi_sock_reset_wr_list(csk);
|
|
}
|
|
l2t_put(csk);
|
|
if (cxgbi_sock_flag(csk, CTPF_HAS_ATID))
|
|
free_atid(csk);
|
|
else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) {
|
|
cxgb3_remove_tid(t3dev, (void *)csk, csk->tid);
|
|
cxgbi_sock_clear_flag(csk, CTPF_HAS_TID);
|
|
cxgbi_sock_put(csk);
|
|
}
|
|
csk->dst = NULL;
|
|
csk->cdev = NULL;
|
|
}
|
|
|
|
static void update_address(struct cxgbi_hba *chba)
|
|
{
|
|
if (chba->ipv4addr) {
|
|
if (chba->vdev &&
|
|
chba->ipv4addr != cxgb3i_get_private_ipv4addr(chba->vdev)) {
|
|
cxgb3i_set_private_ipv4addr(chba->vdev, chba->ipv4addr);
|
|
cxgb3i_set_private_ipv4addr(chba->ndev, 0);
|
|
pr_info("%s set %pI4.\n",
|
|
chba->vdev->name, &chba->ipv4addr);
|
|
} else if (chba->ipv4addr !=
|
|
cxgb3i_get_private_ipv4addr(chba->ndev)) {
|
|
cxgb3i_set_private_ipv4addr(chba->ndev, chba->ipv4addr);
|
|
pr_info("%s set %pI4.\n",
|
|
chba->ndev->name, &chba->ipv4addr);
|
|
}
|
|
} else if (cxgb3i_get_private_ipv4addr(chba->ndev)) {
|
|
if (chba->vdev)
|
|
cxgb3i_set_private_ipv4addr(chba->vdev, 0);
|
|
cxgb3i_set_private_ipv4addr(chba->ndev, 0);
|
|
}
|
|
}
|
|
|
|
static int init_act_open(struct cxgbi_sock *csk)
|
|
{
|
|
struct dst_entry *dst = csk->dst;
|
|
struct cxgbi_device *cdev = csk->cdev;
|
|
struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev;
|
|
struct net_device *ndev = cdev->ports[csk->port_id];
|
|
struct cxgbi_hba *chba = cdev->hbas[csk->port_id];
|
|
struct sk_buff *skb = NULL;
|
|
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
|
|
"csk 0x%p,%u,0x%lx.\n", csk, csk->state, csk->flags);
|
|
|
|
update_address(chba);
|
|
if (chba->ipv4addr)
|
|
csk->saddr.sin_addr.s_addr = chba->ipv4addr;
|
|
|
|
csk->rss_qid = 0;
|
|
csk->l2t = t3_l2t_get(t3dev, dst_get_neighbour(dst), ndev);
|
|
if (!csk->l2t) {
|
|
pr_err("NO l2t available.\n");
|
|
return -EINVAL;
|
|
}
|
|
cxgbi_sock_get(csk);
|
|
|
|
csk->atid = cxgb3_alloc_atid(t3dev, &t3_client, csk);
|
|
if (csk->atid < 0) {
|
|
pr_err("NO atid available.\n");
|
|
goto rel_resource;
|
|
}
|
|
cxgbi_sock_set_flag(csk, CTPF_HAS_ATID);
|
|
cxgbi_sock_get(csk);
|
|
|
|
skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_KERNEL);
|
|
if (!skb)
|
|
goto rel_resource;
|
|
skb->sk = (struct sock *)csk;
|
|
set_arp_failure_handler(skb, act_open_arp_failure);
|
|
|
|
csk->wr_max_cred = csk->wr_cred = T3C_DATA(t3dev)->max_wrs - 1;
|
|
csk->wr_una_cred = 0;
|
|
csk->mss_idx = cxgbi_sock_select_mss(csk, dst_mtu(dst));
|
|
cxgbi_sock_reset_wr_list(csk);
|
|
csk->err = 0;
|
|
|
|
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
|
|
"csk 0x%p,%u,0x%lx, %pI4:%u-%pI4:%u.\n",
|
|
csk, csk->state, csk->flags,
|
|
&csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port),
|
|
&csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port));
|
|
|
|
cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN);
|
|
send_act_open_req(csk, skb, csk->l2t);
|
|
return 0;
|
|
|
|
rel_resource:
|
|
if (skb)
|
|
__kfree_skb(skb);
|
|
return -EINVAL;
|
|
}
|
|
|
|
cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS] = {
|
|
[CPL_ACT_ESTABLISH] = do_act_establish,
|
|
[CPL_ACT_OPEN_RPL] = do_act_open_rpl,
|
|
[CPL_PEER_CLOSE] = do_peer_close,
|
|
[CPL_ABORT_REQ_RSS] = do_abort_req,
|
|
[CPL_ABORT_RPL_RSS] = do_abort_rpl,
|
|
[CPL_CLOSE_CON_RPL] = do_close_con_rpl,
|
|
[CPL_TX_DMA_ACK] = do_wr_ack,
|
|
[CPL_ISCSI_HDR] = do_iscsi_hdr,
|
|
};
|
|
|
|
/**
|
|
* cxgb3i_ofld_init - allocate and initialize resources for each adapter found
|
|
* @cdev: cxgbi adapter
|
|
*/
|
|
int cxgb3i_ofld_init(struct cxgbi_device *cdev)
|
|
{
|
|
struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev;
|
|
struct adap_ports port;
|
|
struct ofld_page_info rx_page_info;
|
|
unsigned int wr_len;
|
|
int rc;
|
|
|
|
if (t3dev->ctl(t3dev, GET_WR_LEN, &wr_len) < 0 ||
|
|
t3dev->ctl(t3dev, GET_PORTS, &port) < 0 ||
|
|
t3dev->ctl(t3dev, GET_RX_PAGE_INFO, &rx_page_info) < 0) {
|
|
pr_warn("t3 0x%p, offload up, ioctl failed.\n", t3dev);
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (cxgb3i_max_connect > CXGBI_MAX_CONN)
|
|
cxgb3i_max_connect = CXGBI_MAX_CONN;
|
|
|
|
rc = cxgbi_device_portmap_create(cdev, cxgb3i_sport_base,
|
|
cxgb3i_max_connect);
|
|
if (rc < 0)
|
|
return rc;
|
|
|
|
init_wr_tab(wr_len);
|
|
cdev->csk_release_offload_resources = release_offload_resources;
|
|
cdev->csk_push_tx_frames = push_tx_frames;
|
|
cdev->csk_send_abort_req = send_abort_req;
|
|
cdev->csk_send_close_req = send_close_req;
|
|
cdev->csk_send_rx_credits = send_rx_credits;
|
|
cdev->csk_alloc_cpls = alloc_cpls;
|
|
cdev->csk_init_act_open = init_act_open;
|
|
|
|
pr_info("cdev 0x%p, offload up, added.\n", cdev);
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* functions to program the pagepod in h/w
|
|
*/
|
|
static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr)
|
|
{
|
|
struct ulp_mem_io *req = (struct ulp_mem_io *)skb->head;
|
|
|
|
memset(req, 0, sizeof(*req));
|
|
|
|
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
|
|
req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(addr >> 5) |
|
|
V_ULPTX_CMD(ULP_MEM_WRITE));
|
|
req->len = htonl(V_ULP_MEMIO_DATA_LEN(PPOD_SIZE >> 5) |
|
|
V_ULPTX_NFLITS((PPOD_SIZE >> 3) + 1));
|
|
}
|
|
|
|
static int ddp_set_map(struct cxgbi_sock *csk, struct cxgbi_pagepod_hdr *hdr,
|
|
unsigned int idx, unsigned int npods,
|
|
struct cxgbi_gather_list *gl)
|
|
{
|
|
struct cxgbi_device *cdev = csk->cdev;
|
|
struct cxgbi_ddp_info *ddp = cdev->ddp;
|
|
unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit;
|
|
int i;
|
|
|
|
log_debug(1 << CXGBI_DBG_DDP,
|
|
"csk 0x%p, idx %u, npods %u, gl 0x%p.\n",
|
|
csk, idx, npods, gl);
|
|
|
|
for (i = 0; i < npods; i++, idx++, pm_addr += PPOD_SIZE) {
|
|
struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) +
|
|
PPOD_SIZE, 0, GFP_ATOMIC);
|
|
|
|
if (!skb)
|
|
return -ENOMEM;
|
|
|
|
ulp_mem_io_set_hdr(skb, pm_addr);
|
|
cxgbi_ddp_ppod_set((struct cxgbi_pagepod *)(skb->head +
|
|
sizeof(struct ulp_mem_io)),
|
|
hdr, gl, i * PPOD_PAGES_MAX);
|
|
skb->priority = CPL_PRIORITY_CONTROL;
|
|
cxgb3_ofld_send(cdev->lldev, skb);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static void ddp_clear_map(struct cxgbi_hba *chba, unsigned int tag,
|
|
unsigned int idx, unsigned int npods)
|
|
{
|
|
struct cxgbi_device *cdev = chba->cdev;
|
|
struct cxgbi_ddp_info *ddp = cdev->ddp;
|
|
unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit;
|
|
int i;
|
|
|
|
log_debug(1 << CXGBI_DBG_DDP,
|
|
"cdev 0x%p, idx %u, npods %u, tag 0x%x.\n",
|
|
cdev, idx, npods, tag);
|
|
|
|
for (i = 0; i < npods; i++, idx++, pm_addr += PPOD_SIZE) {
|
|
struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) +
|
|
PPOD_SIZE, 0, GFP_ATOMIC);
|
|
|
|
if (!skb) {
|
|
pr_err("tag 0x%x, 0x%x, %d/%u, skb OOM.\n",
|
|
tag, idx, i, npods);
|
|
continue;
|
|
}
|
|
ulp_mem_io_set_hdr(skb, pm_addr);
|
|
skb->priority = CPL_PRIORITY_CONTROL;
|
|
cxgb3_ofld_send(cdev->lldev, skb);
|
|
}
|
|
}
|
|
|
|
static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk,
|
|
unsigned int tid, int pg_idx, bool reply)
|
|
{
|
|
struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0,
|
|
GFP_KERNEL);
|
|
struct cpl_set_tcb_field *req;
|
|
u64 val = pg_idx < DDP_PGIDX_MAX ? pg_idx : 0;
|
|
|
|
log_debug(1 << CXGBI_DBG_DDP,
|
|
"csk 0x%p, tid %u, pg_idx %d.\n", csk, tid, pg_idx);
|
|
if (!skb)
|
|
return -ENOMEM;
|
|
|
|
/* set up ulp submode and page size */
|
|
req = (struct cpl_set_tcb_field *)skb->head;
|
|
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
|
|
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
|
|
req->reply = V_NO_REPLY(reply ? 0 : 1);
|
|
req->cpu_idx = 0;
|
|
req->word = htons(31);
|
|
req->mask = cpu_to_be64(0xF0000000);
|
|
req->val = cpu_to_be64(val << 28);
|
|
skb->priority = CPL_PRIORITY_CONTROL;
|
|
|
|
cxgb3_ofld_send(csk->cdev->lldev, skb);
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* cxgb3i_setup_conn_digest - setup conn. digest setting
|
|
* @csk: cxgb tcp socket
|
|
* @tid: connection id
|
|
* @hcrc: header digest enabled
|
|
* @dcrc: data digest enabled
|
|
* @reply: request reply from h/w
|
|
* set up the iscsi digest settings for a connection identified by tid
|
|
*/
|
|
static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid,
|
|
int hcrc, int dcrc, int reply)
|
|
{
|
|
struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0,
|
|
GFP_KERNEL);
|
|
struct cpl_set_tcb_field *req;
|
|
u64 val = (hcrc ? 1 : 0) | (dcrc ? 2 : 0);
|
|
|
|
log_debug(1 << CXGBI_DBG_DDP,
|
|
"csk 0x%p, tid %u, crc %d,%d.\n", csk, tid, hcrc, dcrc);
|
|
if (!skb)
|
|
return -ENOMEM;
|
|
|
|
/* set up ulp submode and page size */
|
|
req = (struct cpl_set_tcb_field *)skb->head;
|
|
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
|
|
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
|
|
req->reply = V_NO_REPLY(reply ? 0 : 1);
|
|
req->cpu_idx = 0;
|
|
req->word = htons(31);
|
|
req->mask = cpu_to_be64(0x0F000000);
|
|
req->val = cpu_to_be64(val << 24);
|
|
skb->priority = CPL_PRIORITY_CONTROL;
|
|
|
|
cxgb3_ofld_send(csk->cdev->lldev, skb);
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* t3_ddp_cleanup - release the cxgb3 adapter's ddp resource
|
|
* @cdev: cxgb3i adapter
|
|
* release all the resource held by the ddp pagepod manager for a given
|
|
* adapter if needed
|
|
*/
|
|
|
|
static void t3_ddp_cleanup(struct cxgbi_device *cdev)
|
|
{
|
|
struct t3cdev *tdev = (struct t3cdev *)cdev->lldev;
|
|
|
|
if (cxgbi_ddp_cleanup(cdev)) {
|
|
pr_info("t3dev 0x%p, ulp_iscsi no more user.\n", tdev);
|
|
tdev->ulp_iscsi = NULL;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* ddp_init - initialize the cxgb3 adapter's ddp resource
|
|
* @cdev: cxgb3i adapter
|
|
* initialize the ddp pagepod manager for a given adapter
|
|
*/
|
|
static int cxgb3i_ddp_init(struct cxgbi_device *cdev)
|
|
{
|
|
struct t3cdev *tdev = (struct t3cdev *)cdev->lldev;
|
|
struct cxgbi_ddp_info *ddp = tdev->ulp_iscsi;
|
|
struct ulp_iscsi_info uinfo;
|
|
unsigned int pgsz_factor[4];
|
|
int i, err;
|
|
|
|
if (ddp) {
|
|
kref_get(&ddp->refcnt);
|
|
pr_warn("t3dev 0x%p, ddp 0x%p already set up.\n",
|
|
tdev, tdev->ulp_iscsi);
|
|
cdev->ddp = ddp;
|
|
return -EALREADY;
|
|
}
|
|
|
|
err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo);
|
|
if (err < 0) {
|
|
pr_err("%s, failed to get iscsi param err=%d.\n",
|
|
tdev->name, err);
|
|
return err;
|
|
}
|
|
|
|
err = cxgbi_ddp_init(cdev, uinfo.llimit, uinfo.ulimit,
|
|
uinfo.max_txsz, uinfo.max_rxsz);
|
|
if (err < 0)
|
|
return err;
|
|
|
|
ddp = cdev->ddp;
|
|
|
|
uinfo.tagmask = ddp->idx_mask << PPOD_IDX_SHIFT;
|
|
cxgbi_ddp_page_size_factor(pgsz_factor);
|
|
for (i = 0; i < 4; i++)
|
|
uinfo.pgsz_factor[i] = pgsz_factor[i];
|
|
uinfo.ulimit = uinfo.llimit + (ddp->nppods << PPOD_SIZE_SHIFT);
|
|
|
|
err = tdev->ctl(tdev, ULP_ISCSI_SET_PARAMS, &uinfo);
|
|
if (err < 0) {
|
|
pr_warn("%s unable to set iscsi param err=%d, ddp disabled.\n",
|
|
tdev->name, err);
|
|
cxgbi_ddp_cleanup(cdev);
|
|
return err;
|
|
}
|
|
tdev->ulp_iscsi = ddp;
|
|
|
|
cdev->csk_ddp_setup_digest = ddp_setup_conn_digest;
|
|
cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx;
|
|
cdev->csk_ddp_set = ddp_set_map;
|
|
cdev->csk_ddp_clear = ddp_clear_map;
|
|
|
|
pr_info("tdev 0x%p, nppods %u, bits %u, mask 0x%x,0x%x pkt %u/%u, "
|
|
"%u/%u.\n",
|
|
tdev, ddp->nppods, ddp->idx_bits, ddp->idx_mask,
|
|
ddp->rsvd_tag_mask, ddp->max_txsz, uinfo.max_txsz,
|
|
ddp->max_rxsz, uinfo.max_rxsz);
|
|
return 0;
|
|
}
|
|
|
|
static void cxgb3i_dev_close(struct t3cdev *t3dev)
|
|
{
|
|
struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
|
|
|
|
if (!cdev || cdev->flags & CXGBI_FLAG_ADAPTER_RESET) {
|
|
pr_info("0x%p close, f 0x%x.\n", cdev, cdev ? cdev->flags : 0);
|
|
return;
|
|
}
|
|
|
|
cxgbi_device_unregister(cdev);
|
|
}
|
|
|
|
/**
|
|
* cxgb3i_dev_open - init a t3 adapter structure and any h/w settings
|
|
* @t3dev: t3cdev adapter
|
|
*/
|
|
static void cxgb3i_dev_open(struct t3cdev *t3dev)
|
|
{
|
|
struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
|
|
struct adapter *adapter = tdev2adap(t3dev);
|
|
int i, err;
|
|
|
|
if (cdev) {
|
|
pr_info("0x%p, updating.\n", cdev);
|
|
return;
|
|
}
|
|
|
|
cdev = cxgbi_device_register(0, adapter->params.nports);
|
|
if (!cdev) {
|
|
pr_warn("device 0x%p register failed.\n", t3dev);
|
|
return;
|
|
}
|
|
|
|
cdev->flags = CXGBI_FLAG_DEV_T3 | CXGBI_FLAG_IPV4_SET;
|
|
cdev->lldev = t3dev;
|
|
cdev->pdev = adapter->pdev;
|
|
cdev->ports = adapter->port;
|
|
cdev->nports = adapter->params.nports;
|
|
cdev->mtus = adapter->params.mtus;
|
|
cdev->nmtus = NMTUS;
|
|
cdev->snd_win = cxgb3i_snd_win;
|
|
cdev->rcv_win = cxgb3i_rcv_win;
|
|
cdev->rx_credit_thres = cxgb3i_rx_credit_thres;
|
|
cdev->skb_tx_rsvd = CXGB3I_TX_HEADER_LEN;
|
|
cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr_norss);
|
|
cdev->dev_ddp_cleanup = t3_ddp_cleanup;
|
|
cdev->itp = &cxgb3i_iscsi_transport;
|
|
|
|
err = cxgb3i_ddp_init(cdev);
|
|
if (err) {
|
|
pr_info("0x%p ddp init failed\n", cdev);
|
|
goto err_out;
|
|
}
|
|
|
|
err = cxgb3i_ofld_init(cdev);
|
|
if (err) {
|
|
pr_info("0x%p offload init failed\n", cdev);
|
|
goto err_out;
|
|
}
|
|
|
|
err = cxgbi_hbas_add(cdev, CXGB3I_MAX_LUN, CXGBI_MAX_CONN,
|
|
&cxgb3i_host_template, cxgb3i_stt);
|
|
if (err)
|
|
goto err_out;
|
|
|
|
for (i = 0; i < cdev->nports; i++)
|
|
cdev->hbas[i]->ipv4addr =
|
|
cxgb3i_get_private_ipv4addr(cdev->ports[i]);
|
|
|
|
pr_info("cdev 0x%p, f 0x%x, t3dev 0x%p open, err %d.\n",
|
|
cdev, cdev ? cdev->flags : 0, t3dev, err);
|
|
return;
|
|
|
|
err_out:
|
|
cxgbi_device_unregister(cdev);
|
|
}
|
|
|
|
static void cxgb3i_dev_event_handler(struct t3cdev *t3dev, u32 event, u32 port)
|
|
{
|
|
struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
|
|
|
|
log_debug(1 << CXGBI_DBG_TOE,
|
|
"0x%p, cdev 0x%p, event 0x%x, port 0x%x.\n",
|
|
t3dev, cdev, event, port);
|
|
if (!cdev)
|
|
return;
|
|
|
|
switch (event) {
|
|
case OFFLOAD_STATUS_DOWN:
|
|
cdev->flags |= CXGBI_FLAG_ADAPTER_RESET;
|
|
break;
|
|
case OFFLOAD_STATUS_UP:
|
|
cdev->flags &= ~CXGBI_FLAG_ADAPTER_RESET;
|
|
break;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* cxgb3i_init_module - module init entry point
|
|
*
|
|
* initialize any driver wide global data structures and register itself
|
|
* with the cxgb3 module
|
|
*/
|
|
static int __init cxgb3i_init_module(void)
|
|
{
|
|
int rc;
|
|
|
|
printk(KERN_INFO "%s", version);
|
|
|
|
rc = cxgbi_iscsi_init(&cxgb3i_iscsi_transport, &cxgb3i_stt);
|
|
if (rc < 0)
|
|
return rc;
|
|
|
|
cxgb3_register_client(&t3_client);
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* cxgb3i_exit_module - module cleanup/exit entry point
|
|
*
|
|
* go through the driver hba list and for each hba, release any resource held.
|
|
* and unregisters iscsi transport and the cxgb3 module
|
|
*/
|
|
static void __exit cxgb3i_exit_module(void)
|
|
{
|
|
cxgb3_unregister_client(&t3_client);
|
|
cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T3);
|
|
cxgbi_iscsi_cleanup(&cxgb3i_iscsi_transport, &cxgb3i_stt);
|
|
}
|
|
|
|
module_init(cxgb3i_init_module);
|
|
module_exit(cxgb3i_exit_module);
|