2011-01-21 00:50:14 +07:00
|
|
|
/*
|
|
|
|
* NVM Express device driver
|
2014-03-24 21:11:22 +07:00
|
|
|
* Copyright (c) 2011-2014, Intel Corporation.
|
2011-01-21 00:50:14 +07:00
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify it
|
|
|
|
* under the terms and conditions of the GNU General Public License,
|
|
|
|
* version 2, as published by the Free Software Foundation.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope it will be useful, but WITHOUT
|
|
|
|
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
|
|
* more details.
|
|
|
|
*/
|
|
|
|
|
2015-12-08 05:30:31 +07:00
|
|
|
#include <linux/aer.h>
|
2018-04-28 02:42:52 +07:00
|
|
|
#include <linux/async.h>
|
2011-01-21 00:50:14 +07:00
|
|
|
#include <linux/blkdev.h>
|
2014-11-04 22:20:14 +07:00
|
|
|
#include <linux/blk-mq.h>
|
2016-09-14 21:18:57 +07:00
|
|
|
#include <linux/blk-mq-pci.h>
|
2017-04-21 03:37:55 +07:00
|
|
|
#include <linux/dmi.h>
|
2011-01-21 00:50:14 +07:00
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/interrupt.h>
|
|
|
|
#include <linux/io.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/module.h>
|
2015-11-26 18:21:29 +07:00
|
|
|
#include <linux/mutex.h>
|
2017-09-16 00:05:38 +07:00
|
|
|
#include <linux/once.h>
|
2011-01-21 00:50:14 +07:00
|
|
|
#include <linux/pci.h>
|
NVMe: Metadata format support
Adds support for NVMe metadata formats and exposes block devices for
all namespaces regardless of their format. Namespace formats that are
unusable will have disk capacity set to 0, but a handle to the block
device is created to simplify device management. A namespace is not
usable when the format requires host interleave block and metadata in
single buffer, has no provisioned storage, or has better data but failed
to register with blk integrity.
The namespace has to be scanned in two phases to support separate
metadata formats. The first establishes the sector size and capacity
prior to invoking add_disk. If metadata is required, the capacity will
be temporarilly set to 0 until it can be revalidated and registered with
the integrity extenstions after add_disk completes.
The driver relies on the integrity extensions to provide the metadata
buffer. NVMe requires this be a single physically contiguous region,
so only one integrity segment is allowed per command. If the metadata
is used for T10 PI, the driver provides mappings to save and restore
the reftag physical block translation. The driver provides no-op
functions for generate and verify if metadata is not used for protection
information. This way the setup is always provided by the block layer.
If a request does not supply a required metadata buffer, the command
is failed with bad address. This could only happen if a user manually
disables verify/generate on such a disk. The only exception to where
this is okay is if the controller is capable of stripping/generating
the metadata, which is possible on some types of formats.
The metadata scatter gather list now occupies the spot in the nvme_iod
that used to be used to link retryable IOD's, but we don't do that
anymore, so the field was unused.
Signed-off-by: Keith Busch <keith.busch@intel.com>
2015-02-20 03:39:03 +07:00
|
|
|
#include <linux/t10-pi.h>
|
2011-01-21 00:50:14 +07:00
|
|
|
#include <linux/types.h>
|
2015-08-28 14:27:14 +07:00
|
|
|
#include <linux/io-64-nonatomic-lo-hi.h>
|
2017-02-04 02:50:32 +07:00
|
|
|
#include <linux/sed-opal.h>
|
2012-02-07 09:45:33 +07:00
|
|
|
|
2015-10-03 20:46:41 +07:00
|
|
|
#include "nvme.h"
|
|
|
|
|
2011-01-21 00:50:14 +07:00
|
|
|
#define SQ_SIZE(depth) (depth * sizeof(struct nvme_command))
|
|
|
|
#define CQ_SIZE(depth) (depth * sizeof(struct nvme_completion))
|
2016-12-17 01:54:50 +07:00
|
|
|
|
2017-10-17 08:24:20 +07:00
|
|
|
#define SGES_PER_PAGE (PAGE_SIZE / sizeof(struct nvme_sgl_desc))
|
2014-05-14 00:42:02 +07:00
|
|
|
|
2018-06-21 22:49:37 +07:00
|
|
|
/*
|
|
|
|
* These can be higher, but we need to ensure that any command doesn't
|
|
|
|
* require an sg allocation that needs more than a page of data.
|
|
|
|
*/
|
|
|
|
#define NVME_MAX_KB_SZ 4096
|
|
|
|
#define NVME_MAX_SEGS 127
|
|
|
|
|
2011-02-06 19:28:06 +07:00
|
|
|
static int use_threaded_interrupts;
|
|
|
|
module_param(use_threaded_interrupts, int, 0);
|
|
|
|
|
2015-07-20 23:14:09 +07:00
|
|
|
static bool use_cmb_sqes = true;
|
2018-06-06 21:13:09 +07:00
|
|
|
module_param(use_cmb_sqes, bool, 0444);
|
2015-07-20 23:14:09 +07:00
|
|
|
MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");
|
|
|
|
|
2017-05-12 22:02:58 +07:00
|
|
|
static unsigned int max_host_mem_size_mb = 128;
|
|
|
|
module_param(max_host_mem_size_mb, uint, 0444);
|
|
|
|
MODULE_PARM_DESC(max_host_mem_size_mb,
|
|
|
|
"Maximum Host Memory Buffer (HMB) size per controller (in MiB)");
|
2011-03-03 06:37:18 +07:00
|
|
|
|
2017-10-17 08:24:20 +07:00
|
|
|
static unsigned int sgl_threshold = SZ_32K;
|
|
|
|
module_param(sgl_threshold, uint, 0644);
|
|
|
|
MODULE_PARM_DESC(sgl_threshold,
|
|
|
|
"Use SGLs when average request segment size is larger or equal to "
|
|
|
|
"this size. Use 0 to disable SGLs.");
|
|
|
|
|
2017-07-10 15:46:59 +07:00
|
|
|
static int io_queue_depth_set(const char *val, const struct kernel_param *kp);
|
|
|
|
static const struct kernel_param_ops io_queue_depth_ops = {
|
|
|
|
.set = io_queue_depth_set,
|
|
|
|
.get = param_get_int,
|
|
|
|
};
|
|
|
|
|
|
|
|
static int io_queue_depth = 1024;
|
|
|
|
module_param_cb(io_queue_depth, &io_queue_depth_ops, &io_queue_depth, 0644);
|
|
|
|
MODULE_PARM_DESC(io_queue_depth, "set io queue depth, should >= 2");
|
|
|
|
|
2015-11-26 16:06:56 +07:00
|
|
|
struct nvme_dev;
|
|
|
|
struct nvme_queue;
|
2015-02-04 01:21:42 +07:00
|
|
|
|
2016-01-13 04:41:18 +07:00
|
|
|
static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown);
|
2013-12-11 03:10:37 +07:00
|
|
|
|
2015-11-26 16:06:56 +07:00
|
|
|
/*
|
|
|
|
* Represents an NVM Express device. Each nvme_dev is a PCI function.
|
|
|
|
*/
|
|
|
|
struct nvme_dev {
|
2018-01-14 17:39:01 +07:00
|
|
|
struct nvme_queue *queues;
|
2015-11-26 16:06:56 +07:00
|
|
|
struct blk_mq_tag_set tagset;
|
|
|
|
struct blk_mq_tag_set admin_tagset;
|
|
|
|
u32 __iomem *dbs;
|
|
|
|
struct device *dev;
|
|
|
|
struct dma_pool *prp_page_pool;
|
|
|
|
struct dma_pool *prp_small_pool;
|
|
|
|
unsigned online_queues;
|
|
|
|
unsigned max_qid;
|
2018-04-12 22:16:10 +07:00
|
|
|
unsigned int num_vecs;
|
2015-11-26 16:06:56 +07:00
|
|
|
int q_depth;
|
|
|
|
u32 db_stride;
|
|
|
|
void __iomem *bar;
|
2017-05-24 15:39:55 +07:00
|
|
|
unsigned long bar_mapped_size;
|
2015-11-26 18:35:49 +07:00
|
|
|
struct work_struct remove_work;
|
2015-11-26 18:21:29 +07:00
|
|
|
struct mutex shutdown_lock;
|
2015-11-26 16:06:56 +07:00
|
|
|
bool subsystem;
|
|
|
|
void __iomem *cmb;
|
2017-10-01 14:37:35 +07:00
|
|
|
pci_bus_addr_t cmb_bus_addr;
|
2015-11-26 16:06:56 +07:00
|
|
|
u64 cmb_size;
|
|
|
|
u32 cmbsz;
|
2016-10-06 09:01:12 +07:00
|
|
|
u32 cmbloc;
|
2015-11-26 16:06:56 +07:00
|
|
|
struct nvme_ctrl ctrl;
|
2016-01-13 04:41:17 +07:00
|
|
|
struct completion ioq_wait;
|
2017-05-12 22:02:58 +07:00
|
|
|
|
2018-06-21 22:49:37 +07:00
|
|
|
mempool_t *iod_mempool;
|
|
|
|
|
2017-05-12 22:02:58 +07:00
|
|
|
/* shadow doorbell buffer support: */
|
2017-04-10 22:51:07 +07:00
|
|
|
u32 *dbbuf_dbs;
|
|
|
|
dma_addr_t dbbuf_dbs_dma_addr;
|
|
|
|
u32 *dbbuf_eis;
|
|
|
|
dma_addr_t dbbuf_eis_dma_addr;
|
2017-05-12 22:02:58 +07:00
|
|
|
|
|
|
|
/* host memory buffer support: */
|
|
|
|
u64 host_mem_size;
|
|
|
|
u32 nr_host_mem_descs;
|
2017-08-28 15:47:18 +07:00
|
|
|
dma_addr_t host_mem_descs_dma;
|
2017-05-12 22:02:58 +07:00
|
|
|
struct nvme_host_mem_buf_desc *host_mem_descs;
|
|
|
|
void **host_mem_desc_bufs;
|
2013-12-11 03:10:40 +07:00
|
|
|
};
|
2011-03-03 06:37:18 +07:00
|
|
|
|
2017-07-10 15:46:59 +07:00
|
|
|
static int io_queue_depth_set(const char *val, const struct kernel_param *kp)
|
|
|
|
{
|
|
|
|
int n = 0, ret;
|
|
|
|
|
|
|
|
ret = kstrtoint(val, 10, &n);
|
|
|
|
if (ret != 0 || n < 2)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
return param_set_int(val, kp);
|
|
|
|
}
|
|
|
|
|
2017-04-10 22:51:07 +07:00
|
|
|
static inline unsigned int sq_idx(unsigned int qid, u32 stride)
|
|
|
|
{
|
|
|
|
return qid * 2 * stride;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline unsigned int cq_idx(unsigned int qid, u32 stride)
|
|
|
|
{
|
|
|
|
return (qid * 2 + 1) * stride;
|
|
|
|
}
|
|
|
|
|
2015-11-26 16:06:56 +07:00
|
|
|
static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
|
|
|
|
{
|
|
|
|
return container_of(ctrl, struct nvme_dev, ctrl);
|
|
|
|
}
|
|
|
|
|
2011-01-21 00:50:14 +07:00
|
|
|
/*
|
|
|
|
* An NVM Express queue. Each device has at least two (one for admin
|
|
|
|
* commands and one for I/O commands).
|
|
|
|
*/
|
|
|
|
struct nvme_queue {
|
|
|
|
struct device *q_dmadev;
|
2011-02-10 21:56:01 +07:00
|
|
|
struct nvme_dev *dev;
|
2018-05-17 23:31:51 +07:00
|
|
|
spinlock_t sq_lock;
|
2011-01-21 00:50:14 +07:00
|
|
|
struct nvme_command *sq_cmds;
|
2015-07-20 23:14:09 +07:00
|
|
|
struct nvme_command __iomem *sq_cmds_io;
|
2018-05-17 23:31:51 +07:00
|
|
|
spinlock_t cq_lock ____cacheline_aligned_in_smp;
|
2011-01-21 00:50:14 +07:00
|
|
|
volatile struct nvme_completion *cqes;
|
2015-06-01 22:29:54 +07:00
|
|
|
struct blk_mq_tags **tags;
|
2011-01-21 00:50:14 +07:00
|
|
|
dma_addr_t sq_dma_addr;
|
|
|
|
dma_addr_t cq_dma_addr;
|
|
|
|
u32 __iomem *q_db;
|
|
|
|
u16 q_depth;
|
2015-01-16 05:19:10 +07:00
|
|
|
s16 cq_vector;
|
2011-01-21 00:50:14 +07:00
|
|
|
u16 sq_tail;
|
|
|
|
u16 cq_head;
|
2018-05-21 21:41:52 +07:00
|
|
|
u16 last_cq_head;
|
2013-12-11 03:10:38 +07:00
|
|
|
u16 qid;
|
2013-06-24 22:47:34 +07:00
|
|
|
u8 cq_phase;
|
2017-04-10 22:51:07 +07:00
|
|
|
u32 *dbbuf_sq_db;
|
|
|
|
u32 *dbbuf_cq_db;
|
|
|
|
u32 *dbbuf_sq_ei;
|
|
|
|
u32 *dbbuf_cq_ei;
|
2011-01-21 00:50:14 +07:00
|
|
|
};
|
|
|
|
|
2015-10-16 12:58:32 +07:00
|
|
|
/*
|
|
|
|
* The nvme_iod describes the data in an I/O, including the list of PRP
|
|
|
|
* entries. You can't see it in this data structure because C doesn't let
|
2015-11-28 21:43:10 +07:00
|
|
|
* me express that. Use nvme_init_iod to ensure there's enough space
|
2015-10-16 12:58:32 +07:00
|
|
|
* allocated to store the PRP list.
|
|
|
|
*/
|
|
|
|
struct nvme_iod {
|
2016-11-10 22:32:33 +07:00
|
|
|
struct nvme_request req;
|
2015-11-28 21:43:10 +07:00
|
|
|
struct nvme_queue *nvmeq;
|
2017-10-17 08:24:20 +07:00
|
|
|
bool use_sgl;
|
2015-11-28 21:43:10 +07:00
|
|
|
int aborted;
|
2015-10-16 12:58:32 +07:00
|
|
|
int npages; /* In the PRP list. 0 means small pool in use */
|
|
|
|
int nents; /* Used in scatterlist */
|
|
|
|
int length; /* Of data, in bytes */
|
|
|
|
dma_addr_t first_dma;
|
2015-10-26 15:12:51 +07:00
|
|
|
struct scatterlist meta_sg; /* metadata requires single contiguous buffer */
|
2015-11-28 21:43:10 +07:00
|
|
|
struct scatterlist *sg;
|
|
|
|
struct scatterlist inline_sg[0];
|
2011-01-21 00:50:14 +07:00
|
|
|
};
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Check we didin't inadvertently grow the command struct
|
|
|
|
*/
|
|
|
|
static inline void _nvme_check_size(void)
|
|
|
|
{
|
|
|
|
BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
|
|
|
|
BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
|
|
|
|
BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
|
|
|
|
BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
|
|
|
|
BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
|
2013-03-27 18:13:41 +07:00
|
|
|
BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
|
2013-12-11 03:10:38 +07:00
|
|
|
BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
|
2011-01-21 00:50:14 +07:00
|
|
|
BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
|
2017-06-07 16:45:29 +07:00
|
|
|
BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
|
|
|
|
BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
|
2011-01-21 00:50:14 +07:00
|
|
|
BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
|
2012-09-27 01:49:27 +07:00
|
|
|
BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
|
2017-04-10 22:51:07 +07:00
|
|
|
BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline unsigned int nvme_dbbuf_size(u32 stride)
|
|
|
|
{
|
|
|
|
return ((num_possible_cpus() + 1) * 8 * stride);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int nvme_dbbuf_dma_alloc(struct nvme_dev *dev)
|
|
|
|
{
|
|
|
|
unsigned int mem_size = nvme_dbbuf_size(dev->db_stride);
|
|
|
|
|
|
|
|
if (dev->dbbuf_dbs)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
dev->dbbuf_dbs = dma_alloc_coherent(dev->dev, mem_size,
|
|
|
|
&dev->dbbuf_dbs_dma_addr,
|
|
|
|
GFP_KERNEL);
|
|
|
|
if (!dev->dbbuf_dbs)
|
|
|
|
return -ENOMEM;
|
|
|
|
dev->dbbuf_eis = dma_alloc_coherent(dev->dev, mem_size,
|
|
|
|
&dev->dbbuf_eis_dma_addr,
|
|
|
|
GFP_KERNEL);
|
|
|
|
if (!dev->dbbuf_eis) {
|
|
|
|
dma_free_coherent(dev->dev, mem_size,
|
|
|
|
dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr);
|
|
|
|
dev->dbbuf_dbs = NULL;
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void nvme_dbbuf_dma_free(struct nvme_dev *dev)
|
|
|
|
{
|
|
|
|
unsigned int mem_size = nvme_dbbuf_size(dev->db_stride);
|
|
|
|
|
|
|
|
if (dev->dbbuf_dbs) {
|
|
|
|
dma_free_coherent(dev->dev, mem_size,
|
|
|
|
dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr);
|
|
|
|
dev->dbbuf_dbs = NULL;
|
|
|
|
}
|
|
|
|
if (dev->dbbuf_eis) {
|
|
|
|
dma_free_coherent(dev->dev, mem_size,
|
|
|
|
dev->dbbuf_eis, dev->dbbuf_eis_dma_addr);
|
|
|
|
dev->dbbuf_eis = NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void nvme_dbbuf_init(struct nvme_dev *dev,
|
|
|
|
struct nvme_queue *nvmeq, int qid)
|
|
|
|
{
|
|
|
|
if (!dev->dbbuf_dbs || !qid)
|
|
|
|
return;
|
|
|
|
|
|
|
|
nvmeq->dbbuf_sq_db = &dev->dbbuf_dbs[sq_idx(qid, dev->db_stride)];
|
|
|
|
nvmeq->dbbuf_cq_db = &dev->dbbuf_dbs[cq_idx(qid, dev->db_stride)];
|
|
|
|
nvmeq->dbbuf_sq_ei = &dev->dbbuf_eis[sq_idx(qid, dev->db_stride)];
|
|
|
|
nvmeq->dbbuf_cq_ei = &dev->dbbuf_eis[cq_idx(qid, dev->db_stride)];
|
|
|
|
}
|
|
|
|
|
|
|
|
static void nvme_dbbuf_set(struct nvme_dev *dev)
|
|
|
|
{
|
|
|
|
struct nvme_command c;
|
|
|
|
|
|
|
|
if (!dev->dbbuf_dbs)
|
|
|
|
return;
|
|
|
|
|
|
|
|
memset(&c, 0, sizeof(c));
|
|
|
|
c.dbbuf.opcode = nvme_admin_dbbuf;
|
|
|
|
c.dbbuf.prp1 = cpu_to_le64(dev->dbbuf_dbs_dma_addr);
|
|
|
|
c.dbbuf.prp2 = cpu_to_le64(dev->dbbuf_eis_dma_addr);
|
|
|
|
|
|
|
|
if (nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0)) {
|
2017-05-20 20:14:43 +07:00
|
|
|
dev_warn(dev->ctrl.device, "unable to set dbbuf\n");
|
2017-04-10 22:51:07 +07:00
|
|
|
/* Free memory and continue on */
|
|
|
|
nvme_dbbuf_dma_free(dev);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int nvme_dbbuf_need_event(u16 event_idx, u16 new_idx, u16 old)
|
|
|
|
{
|
|
|
|
return (u16)(new_idx - event_idx - 1) < (u16)(new_idx - old);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Update dbbuf and return true if an MMIO is required */
|
|
|
|
static bool nvme_dbbuf_update_and_check_event(u16 value, u32 *dbbuf_db,
|
|
|
|
volatile u32 *dbbuf_ei)
|
|
|
|
{
|
|
|
|
if (dbbuf_db) {
|
|
|
|
u16 old_value;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Ensure that the queue is written before updating
|
|
|
|
* the doorbell in memory
|
|
|
|
*/
|
|
|
|
wmb();
|
|
|
|
|
|
|
|
old_value = *dbbuf_db;
|
|
|
|
*dbbuf_db = value;
|
|
|
|
|
|
|
|
if (!nvme_dbbuf_need_event(*dbbuf_ei, value, old_value))
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
|
|
|
|
2015-01-23 02:07:58 +07:00
|
|
|
/*
|
|
|
|
* Max size of iod being embedded in the request payload
|
|
|
|
*/
|
|
|
|
#define NVME_INT_PAGES 2
|
2015-11-28 21:03:49 +07:00
|
|
|
#define NVME_INT_BYTES(dev) (NVME_INT_PAGES * (dev)->ctrl.page_size)
|
2015-01-23 02:07:58 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Will slightly overestimate the number of pages needed. This is OK
|
|
|
|
* as it only leads to a small amount of wasted memory for the lifetime of
|
|
|
|
* the I/O.
|
|
|
|
*/
|
|
|
|
static int nvme_npages(unsigned size, struct nvme_dev *dev)
|
|
|
|
{
|
2015-11-28 21:03:49 +07:00
|
|
|
unsigned nprps = DIV_ROUND_UP(size + dev->ctrl.page_size,
|
|
|
|
dev->ctrl.page_size);
|
2015-01-23 02:07:58 +07:00
|
|
|
return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
|
|
|
|
}
|
|
|
|
|
2017-10-17 08:24:20 +07:00
|
|
|
/*
|
|
|
|
* Calculates the number of pages needed for the SGL segments. For example a 4k
|
|
|
|
* page can accommodate 256 SGL descriptors.
|
|
|
|
*/
|
|
|
|
static int nvme_pci_npages_sgl(unsigned int num_seg)
|
2015-01-23 02:07:58 +07:00
|
|
|
{
|
2017-10-17 08:24:20 +07:00
|
|
|
return DIV_ROUND_UP(num_seg * sizeof(struct nvme_sgl_desc), PAGE_SIZE);
|
2015-11-28 21:43:10 +07:00
|
|
|
}
|
2015-01-23 02:07:58 +07:00
|
|
|
|
2017-10-17 08:24:20 +07:00
|
|
|
static unsigned int nvme_pci_iod_alloc_size(struct nvme_dev *dev,
|
|
|
|
unsigned int size, unsigned int nseg, bool use_sgl)
|
2015-11-28 21:43:10 +07:00
|
|
|
{
|
2017-10-17 08:24:20 +07:00
|
|
|
size_t alloc_size;
|
|
|
|
|
|
|
|
if (use_sgl)
|
|
|
|
alloc_size = sizeof(__le64 *) * nvme_pci_npages_sgl(nseg);
|
|
|
|
else
|
|
|
|
alloc_size = sizeof(__le64 *) * nvme_npages(size, dev);
|
|
|
|
|
|
|
|
return alloc_size + sizeof(struct scatterlist) * nseg;
|
2015-11-28 21:43:10 +07:00
|
|
|
}
|
2015-01-23 02:07:58 +07:00
|
|
|
|
2017-10-17 08:24:20 +07:00
|
|
|
static unsigned int nvme_pci_cmd_size(struct nvme_dev *dev, bool use_sgl)
|
2015-11-28 21:43:10 +07:00
|
|
|
{
|
2017-10-17 08:24:20 +07:00
|
|
|
unsigned int alloc_size = nvme_pci_iod_alloc_size(dev,
|
|
|
|
NVME_INT_BYTES(dev), NVME_INT_PAGES,
|
|
|
|
use_sgl);
|
|
|
|
|
|
|
|
return sizeof(struct nvme_iod) + alloc_size;
|
2015-01-23 02:07:58 +07:00
|
|
|
}
|
|
|
|
|
2014-11-04 22:20:14 +07:00
|
|
|
static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
|
|
|
|
unsigned int hctx_idx)
|
2011-02-07 06:30:16 +07:00
|
|
|
{
|
2014-11-04 22:20:14 +07:00
|
|
|
struct nvme_dev *dev = data;
|
2018-01-14 17:39:01 +07:00
|
|
|
struct nvme_queue *nvmeq = &dev->queues[0];
|
2014-11-04 22:20:14 +07:00
|
|
|
|
2015-06-01 22:29:54 +07:00
|
|
|
WARN_ON(hctx_idx != 0);
|
|
|
|
WARN_ON(dev->admin_tagset.tags[0] != hctx->tags);
|
|
|
|
WARN_ON(nvmeq->tags);
|
|
|
|
|
2014-11-04 22:20:14 +07:00
|
|
|
hctx->driver_data = nvmeq;
|
2015-06-01 22:29:54 +07:00
|
|
|
nvmeq->tags = &dev->admin_tagset.tags[0];
|
2014-11-04 22:20:14 +07:00
|
|
|
return 0;
|
2011-02-07 06:30:16 +07:00
|
|
|
}
|
|
|
|
|
2015-06-08 23:08:13 +07:00
|
|
|
static void nvme_admin_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
|
|
|
|
{
|
|
|
|
struct nvme_queue *nvmeq = hctx->driver_data;
|
|
|
|
|
|
|
|
nvmeq->tags = NULL;
|
|
|
|
}
|
|
|
|
|
2014-11-04 22:20:14 +07:00
|
|
|
static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
|
|
|
|
unsigned int hctx_idx)
|
2011-01-21 00:50:14 +07:00
|
|
|
{
|
2014-11-04 22:20:14 +07:00
|
|
|
struct nvme_dev *dev = data;
|
2018-01-14 17:39:01 +07:00
|
|
|
struct nvme_queue *nvmeq = &dev->queues[hctx_idx + 1];
|
2014-11-04 22:20:14 +07:00
|
|
|
|
2015-06-01 22:29:54 +07:00
|
|
|
if (!nvmeq->tags)
|
|
|
|
nvmeq->tags = &dev->tagset.tags[hctx_idx];
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2015-06-01 22:29:54 +07:00
|
|
|
WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags);
|
2014-11-04 22:20:14 +07:00
|
|
|
hctx->driver_data = nvmeq;
|
|
|
|
return 0;
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
|
|
|
|
2017-05-01 23:19:08 +07:00
|
|
|
static int nvme_init_request(struct blk_mq_tag_set *set, struct request *req,
|
|
|
|
unsigned int hctx_idx, unsigned int numa_node)
|
2011-01-21 00:50:14 +07:00
|
|
|
{
|
2017-05-01 23:19:08 +07:00
|
|
|
struct nvme_dev *dev = set->driver_data;
|
2015-11-28 21:43:10 +07:00
|
|
|
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
|
2017-06-13 14:15:18 +07:00
|
|
|
int queue_idx = (set == &dev->tagset) ? hctx_idx + 1 : 0;
|
2018-01-14 17:39:01 +07:00
|
|
|
struct nvme_queue *nvmeq = &dev->queues[queue_idx];
|
2014-11-04 22:20:14 +07:00
|
|
|
|
|
|
|
BUG_ON(!nvmeq);
|
2015-11-28 21:43:10 +07:00
|
|
|
iod->nvmeq = nvmeq;
|
2014-11-04 22:20:14 +07:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-09-14 21:18:57 +07:00
|
|
|
static int nvme_pci_map_queues(struct blk_mq_tag_set *set)
|
|
|
|
{
|
|
|
|
struct nvme_dev *dev = set->driver_data;
|
|
|
|
|
2018-04-12 22:16:10 +07:00
|
|
|
return blk_mq_pci_map_queues(set, to_pci_dev(dev->dev),
|
|
|
|
dev->num_vecs > 1 ? 1 /* admin queue */ : 0);
|
2016-09-14 21:18:57 +07:00
|
|
|
}
|
|
|
|
|
2011-01-21 00:50:14 +07:00
|
|
|
/**
|
2018-05-26 18:45:55 +07:00
|
|
|
* nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
|
2011-01-21 00:50:14 +07:00
|
|
|
* @nvmeq: The queue to use
|
|
|
|
* @cmd: The command to send
|
|
|
|
*/
|
2018-05-26 18:45:55 +07:00
|
|
|
static void nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
|
2011-01-21 00:50:14 +07:00
|
|
|
{
|
2018-05-26 18:45:55 +07:00
|
|
|
spin_lock(&nvmeq->sq_lock);
|
2015-07-20 23:14:09 +07:00
|
|
|
if (nvmeq->sq_cmds_io)
|
2018-05-26 18:45:55 +07:00
|
|
|
memcpy_toio(&nvmeq->sq_cmds_io[nvmeq->sq_tail], cmd,
|
|
|
|
sizeof(*cmd));
|
2015-07-20 23:14:09 +07:00
|
|
|
else
|
2018-05-26 18:45:55 +07:00
|
|
|
memcpy(&nvmeq->sq_cmds[nvmeq->sq_tail], cmd, sizeof(*cmd));
|
2015-07-20 23:14:09 +07:00
|
|
|
|
2018-05-26 18:45:55 +07:00
|
|
|
if (++nvmeq->sq_tail == nvmeq->q_depth)
|
|
|
|
nvmeq->sq_tail = 0;
|
|
|
|
if (nvme_dbbuf_update_and_check_event(nvmeq->sq_tail,
|
|
|
|
nvmeq->dbbuf_sq_db, nvmeq->dbbuf_sq_ei))
|
|
|
|
writel(nvmeq->sq_tail, nvmeq->q_db);
|
|
|
|
spin_unlock(&nvmeq->sq_lock);
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
|
|
|
|
2017-10-17 08:24:20 +07:00
|
|
|
static void **nvme_pci_iod_list(struct request *req)
|
2011-01-21 00:50:14 +07:00
|
|
|
{
|
2015-11-28 21:43:10 +07:00
|
|
|
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
|
2017-10-17 08:24:20 +07:00
|
|
|
return (void **)(iod->sg + blk_rq_nr_phys_segments(req));
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
|
|
|
|
2017-12-20 14:30:50 +07:00
|
|
|
static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req)
|
|
|
|
{
|
|
|
|
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
|
2018-01-18 04:04:37 +07:00
|
|
|
int nseg = blk_rq_nr_phys_segments(req);
|
2017-12-20 14:30:50 +07:00
|
|
|
unsigned int avg_seg_size;
|
|
|
|
|
2018-01-18 04:04:37 +07:00
|
|
|
if (nseg == 0)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
avg_seg_size = DIV_ROUND_UP(blk_rq_payload_bytes(req), nseg);
|
2017-12-20 14:30:50 +07:00
|
|
|
|
|
|
|
if (!(dev->ctrl.sgls & ((1 << 0) | (1 << 1))))
|
|
|
|
return false;
|
|
|
|
if (!iod->nvmeq->qid)
|
|
|
|
return false;
|
|
|
|
if (!sgl_threshold || avg_seg_size < sgl_threshold)
|
|
|
|
return false;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2017-06-03 14:38:05 +07:00
|
|
|
static blk_status_t nvme_init_iod(struct request *rq, struct nvme_dev *dev)
|
2015-01-23 02:07:58 +07:00
|
|
|
{
|
2015-11-28 21:43:10 +07:00
|
|
|
struct nvme_iod *iod = blk_mq_rq_to_pdu(rq);
|
2016-12-09 05:20:32 +07:00
|
|
|
int nseg = blk_rq_nr_phys_segments(rq);
|
2017-01-13 18:29:12 +07:00
|
|
|
unsigned int size = blk_rq_payload_bytes(rq);
|
2015-01-23 02:07:58 +07:00
|
|
|
|
2017-12-20 14:30:50 +07:00
|
|
|
iod->use_sgl = nvme_pci_use_sgls(dev, rq);
|
|
|
|
|
2015-11-28 21:43:10 +07:00
|
|
|
if (nseg > NVME_INT_PAGES || size > NVME_INT_BYTES(dev)) {
|
2018-06-21 22:49:37 +07:00
|
|
|
iod->sg = mempool_alloc(dev->iod_mempool, GFP_ATOMIC);
|
2015-11-28 21:43:10 +07:00
|
|
|
if (!iod->sg)
|
2017-06-03 14:38:05 +07:00
|
|
|
return BLK_STS_RESOURCE;
|
2015-11-28 21:43:10 +07:00
|
|
|
} else {
|
|
|
|
iod->sg = iod->inline_sg;
|
2015-01-23 02:07:58 +07:00
|
|
|
}
|
|
|
|
|
2015-11-28 21:43:10 +07:00
|
|
|
iod->aborted = 0;
|
|
|
|
iod->npages = -1;
|
|
|
|
iod->nents = 0;
|
|
|
|
iod->length = size;
|
2016-07-13 06:20:31 +07:00
|
|
|
|
2017-06-03 14:38:05 +07:00
|
|
|
return BLK_STS_OK;
|
2015-01-23 02:07:58 +07:00
|
|
|
}
|
|
|
|
|
2015-11-28 21:43:10 +07:00
|
|
|
static void nvme_free_iod(struct nvme_dev *dev, struct request *req)
|
2011-01-21 00:50:14 +07:00
|
|
|
{
|
2015-11-28 21:43:10 +07:00
|
|
|
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
|
2017-10-17 08:24:20 +07:00
|
|
|
const int last_prp = dev->ctrl.page_size / sizeof(__le64) - 1;
|
|
|
|
dma_addr_t dma_addr = iod->first_dma, next_dma_addr;
|
|
|
|
|
2011-12-21 01:34:52 +07:00
|
|
|
int i;
|
|
|
|
|
|
|
|
if (iod->npages == 0)
|
2017-10-17 08:24:20 +07:00
|
|
|
dma_pool_free(dev->prp_small_pool, nvme_pci_iod_list(req)[0],
|
|
|
|
dma_addr);
|
|
|
|
|
2011-12-21 01:34:52 +07:00
|
|
|
for (i = 0; i < iod->npages; i++) {
|
2017-10-17 08:24:20 +07:00
|
|
|
void *addr = nvme_pci_iod_list(req)[i];
|
|
|
|
|
|
|
|
if (iod->use_sgl) {
|
|
|
|
struct nvme_sgl_desc *sg_list = addr;
|
|
|
|
|
|
|
|
next_dma_addr =
|
|
|
|
le64_to_cpu((sg_list[SGES_PER_PAGE - 1]).addr);
|
|
|
|
} else {
|
|
|
|
__le64 *prp_list = addr;
|
|
|
|
|
|
|
|
next_dma_addr = le64_to_cpu(prp_list[last_prp]);
|
|
|
|
}
|
|
|
|
|
|
|
|
dma_pool_free(dev->prp_page_pool, addr, dma_addr);
|
|
|
|
dma_addr = next_dma_addr;
|
2011-12-21 01:34:52 +07:00
|
|
|
}
|
2015-01-23 02:07:58 +07:00
|
|
|
|
2015-11-28 21:43:10 +07:00
|
|
|
if (iod->sg != iod->inline_sg)
|
2018-06-21 22:49:37 +07:00
|
|
|
mempool_free(iod->sg, dev->iod_mempool);
|
2014-08-29 22:06:12 +07:00
|
|
|
}
|
|
|
|
|
2015-02-23 23:16:21 +07:00
|
|
|
#ifdef CONFIG_BLK_DEV_INTEGRITY
|
NVMe: Metadata format support
Adds support for NVMe metadata formats and exposes block devices for
all namespaces regardless of their format. Namespace formats that are
unusable will have disk capacity set to 0, but a handle to the block
device is created to simplify device management. A namespace is not
usable when the format requires host interleave block and metadata in
single buffer, has no provisioned storage, or has better data but failed
to register with blk integrity.
The namespace has to be scanned in two phases to support separate
metadata formats. The first establishes the sector size and capacity
prior to invoking add_disk. If metadata is required, the capacity will
be temporarilly set to 0 until it can be revalidated and registered with
the integrity extenstions after add_disk completes.
The driver relies on the integrity extensions to provide the metadata
buffer. NVMe requires this be a single physically contiguous region,
so only one integrity segment is allowed per command. If the metadata
is used for T10 PI, the driver provides mappings to save and restore
the reftag physical block translation. The driver provides no-op
functions for generate and verify if metadata is not used for protection
information. This way the setup is always provided by the block layer.
If a request does not supply a required metadata buffer, the command
is failed with bad address. This could only happen if a user manually
disables verify/generate on such a disk. The only exception to where
this is okay is if the controller is capable of stripping/generating
the metadata, which is possible on some types of formats.
The metadata scatter gather list now occupies the spot in the nvme_iod
that used to be used to link retryable IOD's, but we don't do that
anymore, so the field was unused.
Signed-off-by: Keith Busch <keith.busch@intel.com>
2015-02-20 03:39:03 +07:00
|
|
|
static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
|
|
|
|
{
|
|
|
|
if (be32_to_cpu(pi->ref_tag) == v)
|
|
|
|
pi->ref_tag = cpu_to_be32(p);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
|
|
|
|
{
|
|
|
|
if (be32_to_cpu(pi->ref_tag) == p)
|
|
|
|
pi->ref_tag = cpu_to_be32(v);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* nvme_dif_remap - remaps ref tags to bip seed and physical lba
|
|
|
|
*
|
|
|
|
* The virtual start sector is the one that was originally submitted by the
|
|
|
|
* block layer. Due to partitioning, MD/DM cloning, etc. the actual physical
|
|
|
|
* start sector may be different. Remap protection information to match the
|
|
|
|
* physical LBA on writes, and back to the original seed on reads.
|
|
|
|
*
|
|
|
|
* Type 0 and 3 do not have a ref tag, so no remapping required.
|
|
|
|
*/
|
|
|
|
static void nvme_dif_remap(struct request *req,
|
|
|
|
void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
|
|
|
|
{
|
|
|
|
struct nvme_ns *ns = req->rq_disk->private_data;
|
|
|
|
struct bio_integrity_payload *bip;
|
|
|
|
struct t10_pi_tuple *pi;
|
|
|
|
void *p, *pmap;
|
|
|
|
u32 i, nlb, ts, phys, virt;
|
|
|
|
|
|
|
|
if (!ns->pi_type || ns->pi_type == NVME_NS_DPS_PI_TYPE3)
|
|
|
|
return;
|
|
|
|
|
|
|
|
bip = bio_integrity(req->bio);
|
|
|
|
if (!bip)
|
|
|
|
return;
|
|
|
|
|
|
|
|
pmap = kmap_atomic(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset;
|
|
|
|
|
|
|
|
p = pmap;
|
|
|
|
virt = bip_get_seed(bip);
|
|
|
|
phys = nvme_block_nr(ns, blk_rq_pos(req));
|
|
|
|
nlb = (blk_rq_bytes(req) >> ns->lba_shift);
|
2015-10-22 00:20:18 +07:00
|
|
|
ts = ns->disk->queue->integrity.tuple_size;
|
NVMe: Metadata format support
Adds support for NVMe metadata formats and exposes block devices for
all namespaces regardless of their format. Namespace formats that are
unusable will have disk capacity set to 0, but a handle to the block
device is created to simplify device management. A namespace is not
usable when the format requires host interleave block and metadata in
single buffer, has no provisioned storage, or has better data but failed
to register with blk integrity.
The namespace has to be scanned in two phases to support separate
metadata formats. The first establishes the sector size and capacity
prior to invoking add_disk. If metadata is required, the capacity will
be temporarilly set to 0 until it can be revalidated and registered with
the integrity extenstions after add_disk completes.
The driver relies on the integrity extensions to provide the metadata
buffer. NVMe requires this be a single physically contiguous region,
so only one integrity segment is allowed per command. If the metadata
is used for T10 PI, the driver provides mappings to save and restore
the reftag physical block translation. The driver provides no-op
functions for generate and verify if metadata is not used for protection
information. This way the setup is always provided by the block layer.
If a request does not supply a required metadata buffer, the command
is failed with bad address. This could only happen if a user manually
disables verify/generate on such a disk. The only exception to where
this is okay is if the controller is capable of stripping/generating
the metadata, which is possible on some types of formats.
The metadata scatter gather list now occupies the spot in the nvme_iod
that used to be used to link retryable IOD's, but we don't do that
anymore, so the field was unused.
Signed-off-by: Keith Busch <keith.busch@intel.com>
2015-02-20 03:39:03 +07:00
|
|
|
|
|
|
|
for (i = 0; i < nlb; i++, virt++, phys++) {
|
|
|
|
pi = (struct t10_pi_tuple *)p;
|
|
|
|
dif_swap(phys, virt, pi);
|
|
|
|
p += ts;
|
|
|
|
}
|
|
|
|
kunmap_atomic(pmap);
|
|
|
|
}
|
2015-02-23 23:16:21 +07:00
|
|
|
#else /* CONFIG_BLK_DEV_INTEGRITY */
|
|
|
|
static void nvme_dif_remap(struct request *req,
|
|
|
|
void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
|
|
|
|
{
|
|
|
|
}
|
|
|
|
static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2017-09-16 00:05:38 +07:00
|
|
|
static void nvme_print_sgl(struct scatterlist *sgl, int nents)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
struct scatterlist *sg;
|
|
|
|
|
|
|
|
for_each_sg(sgl, sg, nents, i) {
|
|
|
|
dma_addr_t phys = sg_phys(sg);
|
|
|
|
pr_warn("sg[%d] phys_addr:%pad offset:%d length:%d "
|
|
|
|
"dma_address:%pad dma_length:%d\n",
|
|
|
|
i, &phys, sg->offset, sg->length, &sg_dma_address(sg),
|
|
|
|
sg_dma_len(sg));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-10-17 08:24:20 +07:00
|
|
|
static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev,
|
|
|
|
struct request *req, struct nvme_rw_command *cmnd)
|
2011-01-26 22:02:29 +07:00
|
|
|
{
|
2015-11-28 21:43:10 +07:00
|
|
|
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
|
2011-02-10 22:30:34 +07:00
|
|
|
struct dma_pool *pool;
|
2017-01-13 18:29:12 +07:00
|
|
|
int length = blk_rq_payload_bytes(req);
|
2011-12-21 01:34:52 +07:00
|
|
|
struct scatterlist *sg = iod->sg;
|
2011-01-26 22:02:29 +07:00
|
|
|
int dma_len = sg_dma_len(sg);
|
|
|
|
u64 dma_addr = sg_dma_address(sg);
|
2015-11-28 21:03:49 +07:00
|
|
|
u32 page_size = dev->ctrl.page_size;
|
2015-03-26 23:07:51 +07:00
|
|
|
int offset = dma_addr & (page_size - 1);
|
2011-02-10 20:51:24 +07:00
|
|
|
__le64 *prp_list;
|
2017-10-17 08:24:20 +07:00
|
|
|
void **list = nvme_pci_iod_list(req);
|
2011-02-10 20:51:24 +07:00
|
|
|
dma_addr_t prp_dma;
|
2011-12-21 01:34:52 +07:00
|
|
|
int nprps, i;
|
2011-01-26 22:02:29 +07:00
|
|
|
|
2014-06-24 00:34:01 +07:00
|
|
|
length -= (page_size - offset);
|
2017-08-27 20:56:37 +07:00
|
|
|
if (length <= 0) {
|
|
|
|
iod->first_dma = 0;
|
2017-10-17 08:24:20 +07:00
|
|
|
goto done;
|
2017-08-27 20:56:37 +07:00
|
|
|
}
|
2011-01-26 22:02:29 +07:00
|
|
|
|
2014-06-24 00:34:01 +07:00
|
|
|
dma_len -= (page_size - offset);
|
2011-01-26 22:02:29 +07:00
|
|
|
if (dma_len) {
|
2014-06-24 00:34:01 +07:00
|
|
|
dma_addr += (page_size - offset);
|
2011-01-26 22:02:29 +07:00
|
|
|
} else {
|
|
|
|
sg = sg_next(sg);
|
|
|
|
dma_addr = sg_dma_address(sg);
|
|
|
|
dma_len = sg_dma_len(sg);
|
|
|
|
}
|
|
|
|
|
2014-06-24 00:34:01 +07:00
|
|
|
if (length <= page_size) {
|
2014-04-04 05:45:23 +07:00
|
|
|
iod->first_dma = dma_addr;
|
2017-10-17 08:24:20 +07:00
|
|
|
goto done;
|
2011-02-10 20:51:24 +07:00
|
|
|
}
|
|
|
|
|
2014-06-24 00:34:01 +07:00
|
|
|
nprps = DIV_ROUND_UP(length, page_size);
|
2011-02-10 22:30:34 +07:00
|
|
|
if (nprps <= (256 / 8)) {
|
|
|
|
pool = dev->prp_small_pool;
|
2011-12-21 01:34:52 +07:00
|
|
|
iod->npages = 0;
|
2011-02-10 22:30:34 +07:00
|
|
|
} else {
|
|
|
|
pool = dev->prp_page_pool;
|
2011-12-21 01:34:52 +07:00
|
|
|
iod->npages = 1;
|
2011-02-10 22:30:34 +07:00
|
|
|
}
|
|
|
|
|
2015-10-16 12:58:37 +07:00
|
|
|
prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
|
2011-05-13 00:51:41 +07:00
|
|
|
if (!prp_list) {
|
2014-04-04 05:45:23 +07:00
|
|
|
iod->first_dma = dma_addr;
|
2011-12-21 01:34:52 +07:00
|
|
|
iod->npages = -1;
|
2017-07-13 02:59:07 +07:00
|
|
|
return BLK_STS_RESOURCE;
|
2011-05-13 00:51:41 +07:00
|
|
|
}
|
2011-12-21 01:34:52 +07:00
|
|
|
list[0] = prp_list;
|
|
|
|
iod->first_dma = prp_dma;
|
2011-02-10 20:51:24 +07:00
|
|
|
i = 0;
|
|
|
|
for (;;) {
|
2014-06-24 00:34:01 +07:00
|
|
|
if (i == page_size >> 3) {
|
2011-02-10 20:51:24 +07:00
|
|
|
__le64 *old_prp_list = prp_list;
|
2015-10-16 12:58:37 +07:00
|
|
|
prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
|
2011-12-21 01:34:52 +07:00
|
|
|
if (!prp_list)
|
2017-07-13 02:59:07 +07:00
|
|
|
return BLK_STS_RESOURCE;
|
2011-12-21 01:34:52 +07:00
|
|
|
list[iod->npages++] = prp_list;
|
2011-03-17 03:43:40 +07:00
|
|
|
prp_list[0] = old_prp_list[i - 1];
|
|
|
|
old_prp_list[i - 1] = cpu_to_le64(prp_dma);
|
|
|
|
i = 1;
|
2011-02-10 20:51:24 +07:00
|
|
|
}
|
|
|
|
prp_list[i++] = cpu_to_le64(dma_addr);
|
2014-06-24 00:34:01 +07:00
|
|
|
dma_len -= page_size;
|
|
|
|
dma_addr += page_size;
|
|
|
|
length -= page_size;
|
2011-02-10 20:51:24 +07:00
|
|
|
if (length <= 0)
|
|
|
|
break;
|
|
|
|
if (dma_len > 0)
|
|
|
|
continue;
|
2017-07-13 02:59:07 +07:00
|
|
|
if (unlikely(dma_len < 0))
|
|
|
|
goto bad_sgl;
|
2011-02-10 20:51:24 +07:00
|
|
|
sg = sg_next(sg);
|
|
|
|
dma_addr = sg_dma_address(sg);
|
|
|
|
dma_len = sg_dma_len(sg);
|
2011-01-26 22:02:29 +07:00
|
|
|
}
|
|
|
|
|
2017-10-17 08:24:20 +07:00
|
|
|
done:
|
|
|
|
cmnd->dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
|
|
|
|
cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma);
|
|
|
|
|
2017-07-13 02:59:07 +07:00
|
|
|
return BLK_STS_OK;
|
|
|
|
|
|
|
|
bad_sgl:
|
2017-09-16 00:05:38 +07:00
|
|
|
WARN(DO_ONCE(nvme_print_sgl, iod->sg, iod->nents),
|
|
|
|
"Invalid SGL for payload:%d nents:%d\n",
|
|
|
|
blk_rq_payload_bytes(req), iod->nents);
|
2017-07-13 02:59:07 +07:00
|
|
|
return BLK_STS_IOERR;
|
2011-01-26 22:02:29 +07:00
|
|
|
}
|
|
|
|
|
2017-10-17 08:24:20 +07:00
|
|
|
static void nvme_pci_sgl_set_data(struct nvme_sgl_desc *sge,
|
|
|
|
struct scatterlist *sg)
|
|
|
|
{
|
|
|
|
sge->addr = cpu_to_le64(sg_dma_address(sg));
|
|
|
|
sge->length = cpu_to_le32(sg_dma_len(sg));
|
|
|
|
sge->type = NVME_SGL_FMT_DATA_DESC << 4;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void nvme_pci_sgl_set_seg(struct nvme_sgl_desc *sge,
|
|
|
|
dma_addr_t dma_addr, int entries)
|
|
|
|
{
|
|
|
|
sge->addr = cpu_to_le64(dma_addr);
|
|
|
|
if (entries < SGES_PER_PAGE) {
|
|
|
|
sge->length = cpu_to_le32(entries * sizeof(*sge));
|
|
|
|
sge->type = NVME_SGL_FMT_LAST_SEG_DESC << 4;
|
|
|
|
} else {
|
|
|
|
sge->length = cpu_to_le32(PAGE_SIZE);
|
|
|
|
sge->type = NVME_SGL_FMT_SEG_DESC << 4;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev,
|
2018-01-18 04:04:38 +07:00
|
|
|
struct request *req, struct nvme_rw_command *cmd, int entries)
|
2017-10-17 08:24:20 +07:00
|
|
|
{
|
|
|
|
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
|
|
|
|
struct dma_pool *pool;
|
|
|
|
struct nvme_sgl_desc *sg_list;
|
|
|
|
struct scatterlist *sg = iod->sg;
|
|
|
|
dma_addr_t sgl_dma;
|
2018-01-18 04:04:38 +07:00
|
|
|
int i = 0;
|
2017-10-17 08:24:20 +07:00
|
|
|
|
|
|
|
/* setting the transfer type as SGL */
|
|
|
|
cmd->flags = NVME_CMD_SGL_METABUF;
|
|
|
|
|
2018-01-18 04:04:38 +07:00
|
|
|
if (entries == 1) {
|
2017-10-17 08:24:20 +07:00
|
|
|
nvme_pci_sgl_set_data(&cmd->dptr.sgl, sg);
|
|
|
|
return BLK_STS_OK;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (entries <= (256 / sizeof(struct nvme_sgl_desc))) {
|
|
|
|
pool = dev->prp_small_pool;
|
|
|
|
iod->npages = 0;
|
|
|
|
} else {
|
|
|
|
pool = dev->prp_page_pool;
|
|
|
|
iod->npages = 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma);
|
|
|
|
if (!sg_list) {
|
|
|
|
iod->npages = -1;
|
|
|
|
return BLK_STS_RESOURCE;
|
|
|
|
}
|
|
|
|
|
|
|
|
nvme_pci_iod_list(req)[0] = sg_list;
|
|
|
|
iod->first_dma = sgl_dma;
|
|
|
|
|
|
|
|
nvme_pci_sgl_set_seg(&cmd->dptr.sgl, sgl_dma, entries);
|
|
|
|
|
|
|
|
do {
|
|
|
|
if (i == SGES_PER_PAGE) {
|
|
|
|
struct nvme_sgl_desc *old_sg_desc = sg_list;
|
|
|
|
struct nvme_sgl_desc *link = &old_sg_desc[i - 1];
|
|
|
|
|
|
|
|
sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma);
|
|
|
|
if (!sg_list)
|
|
|
|
return BLK_STS_RESOURCE;
|
|
|
|
|
|
|
|
i = 0;
|
|
|
|
nvme_pci_iod_list(req)[iod->npages++] = sg_list;
|
|
|
|
sg_list[i++] = *link;
|
|
|
|
nvme_pci_sgl_set_seg(link, sgl_dma, entries);
|
|
|
|
}
|
|
|
|
|
|
|
|
nvme_pci_sgl_set_data(&sg_list[i++], sg);
|
|
|
|
sg = sg_next(sg);
|
2018-01-18 04:04:38 +07:00
|
|
|
} while (--entries > 0);
|
2017-10-17 08:24:20 +07:00
|
|
|
|
|
|
|
return BLK_STS_OK;
|
|
|
|
}
|
|
|
|
|
2017-06-03 14:38:05 +07:00
|
|
|
static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
|
2017-01-13 18:29:12 +07:00
|
|
|
struct nvme_command *cmnd)
|
2015-05-22 16:12:46 +07:00
|
|
|
{
|
2015-11-28 21:43:10 +07:00
|
|
|
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
|
2015-10-16 12:58:38 +07:00
|
|
|
struct request_queue *q = req->q;
|
|
|
|
enum dma_data_direction dma_dir = rq_data_dir(req) ?
|
|
|
|
DMA_TO_DEVICE : DMA_FROM_DEVICE;
|
2017-06-03 14:38:05 +07:00
|
|
|
blk_status_t ret = BLK_STS_IOERR;
|
2018-01-18 04:04:38 +07:00
|
|
|
int nr_mapped;
|
2015-05-22 16:12:46 +07:00
|
|
|
|
2016-12-09 05:20:32 +07:00
|
|
|
sg_init_table(iod->sg, blk_rq_nr_phys_segments(req));
|
2015-10-16 12:58:38 +07:00
|
|
|
iod->nents = blk_rq_map_sg(q, req, iod->sg);
|
|
|
|
if (!iod->nents)
|
|
|
|
goto out;
|
2015-05-22 16:12:46 +07:00
|
|
|
|
2017-06-03 14:38:05 +07:00
|
|
|
ret = BLK_STS_RESOURCE;
|
2018-01-18 04:04:38 +07:00
|
|
|
nr_mapped = dma_map_sg_attrs(dev->dev, iod->sg, iod->nents, dma_dir,
|
|
|
|
DMA_ATTR_NO_WARN);
|
|
|
|
if (!nr_mapped)
|
2015-10-16 12:58:38 +07:00
|
|
|
goto out;
|
2015-05-22 16:12:46 +07:00
|
|
|
|
2017-12-20 14:30:50 +07:00
|
|
|
if (iod->use_sgl)
|
2018-01-18 04:04:38 +07:00
|
|
|
ret = nvme_pci_setup_sgls(dev, req, &cmnd->rw, nr_mapped);
|
2017-10-17 08:24:20 +07:00
|
|
|
else
|
|
|
|
ret = nvme_pci_setup_prps(dev, req, &cmnd->rw);
|
|
|
|
|
2017-07-13 02:59:07 +07:00
|
|
|
if (ret != BLK_STS_OK)
|
2015-10-16 12:58:38 +07:00
|
|
|
goto out_unmap;
|
2012-11-10 06:33:05 +07:00
|
|
|
|
2017-06-03 14:38:05 +07:00
|
|
|
ret = BLK_STS_IOERR;
|
2015-10-16 12:58:38 +07:00
|
|
|
if (blk_integrity_rq(req)) {
|
|
|
|
if (blk_rq_count_integrity_sg(q, req->bio) != 1)
|
|
|
|
goto out_unmap;
|
2012-11-10 06:33:05 +07:00
|
|
|
|
2015-10-26 15:12:51 +07:00
|
|
|
sg_init_table(&iod->meta_sg, 1);
|
|
|
|
if (blk_rq_map_integrity_sg(q, req->bio, &iod->meta_sg) != 1)
|
2015-10-16 12:58:38 +07:00
|
|
|
goto out_unmap;
|
2012-11-10 06:33:05 +07:00
|
|
|
|
2017-08-30 04:46:02 +07:00
|
|
|
if (req_op(req) == REQ_OP_WRITE)
|
2015-10-16 12:58:38 +07:00
|
|
|
nvme_dif_remap(req, nvme_dif_prep);
|
2012-11-10 06:33:05 +07:00
|
|
|
|
2015-10-26 15:12:51 +07:00
|
|
|
if (!dma_map_sg(dev->dev, &iod->meta_sg, 1, dma_dir))
|
2015-10-16 12:58:38 +07:00
|
|
|
goto out_unmap;
|
2015-05-22 16:12:46 +07:00
|
|
|
}
|
2011-02-23 02:18:30 +07:00
|
|
|
|
2015-10-16 12:58:38 +07:00
|
|
|
if (blk_integrity_rq(req))
|
2015-10-26 15:12:51 +07:00
|
|
|
cmnd->rw.metadata = cpu_to_le64(sg_dma_address(&iod->meta_sg));
|
2017-06-03 14:38:05 +07:00
|
|
|
return BLK_STS_OK;
|
2011-02-23 02:18:30 +07:00
|
|
|
|
2015-10-16 12:58:38 +07:00
|
|
|
out_unmap:
|
|
|
|
dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
|
|
|
|
out:
|
|
|
|
return ret;
|
2011-02-23 02:18:30 +07:00
|
|
|
}
|
|
|
|
|
2015-11-28 21:43:10 +07:00
|
|
|
static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
|
2011-01-21 00:50:14 +07:00
|
|
|
{
|
2015-11-28 21:43:10 +07:00
|
|
|
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
|
2015-11-26 16:51:23 +07:00
|
|
|
enum dma_data_direction dma_dir = rq_data_dir(req) ?
|
|
|
|
DMA_TO_DEVICE : DMA_FROM_DEVICE;
|
|
|
|
|
|
|
|
if (iod->nents) {
|
|
|
|
dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
|
|
|
|
if (blk_integrity_rq(req)) {
|
2017-08-30 04:46:02 +07:00
|
|
|
if (req_op(req) == REQ_OP_READ)
|
2015-11-26 16:51:23 +07:00
|
|
|
nvme_dif_remap(req, nvme_dif_complete);
|
2015-10-26 15:12:51 +07:00
|
|
|
dma_unmap_sg(dev->dev, &iod->meta_sg, 1, dma_dir);
|
NVMe: Metadata format support
Adds support for NVMe metadata formats and exposes block devices for
all namespaces regardless of their format. Namespace formats that are
unusable will have disk capacity set to 0, but a handle to the block
device is created to simplify device management. A namespace is not
usable when the format requires host interleave block and metadata in
single buffer, has no provisioned storage, or has better data but failed
to register with blk integrity.
The namespace has to be scanned in two phases to support separate
metadata formats. The first establishes the sector size and capacity
prior to invoking add_disk. If metadata is required, the capacity will
be temporarilly set to 0 until it can be revalidated and registered with
the integrity extenstions after add_disk completes.
The driver relies on the integrity extensions to provide the metadata
buffer. NVMe requires this be a single physically contiguous region,
so only one integrity segment is allowed per command. If the metadata
is used for T10 PI, the driver provides mappings to save and restore
the reftag physical block translation. The driver provides no-op
functions for generate and verify if metadata is not used for protection
information. This way the setup is always provided by the block layer.
If a request does not supply a required metadata buffer, the command
is failed with bad address. This could only happen if a user manually
disables verify/generate on such a disk. The only exception to where
this is okay is if the controller is capable of stripping/generating
the metadata, which is possible on some types of formats.
The metadata scatter gather list now occupies the spot in the nvme_iod
that used to be used to link retryable IOD's, but we don't do that
anymore, so the field was unused.
Signed-off-by: Keith Busch <keith.busch@intel.com>
2015-02-20 03:39:03 +07:00
|
|
|
}
|
2015-08-26 21:56:14 +07:00
|
|
|
}
|
NVMe: Metadata format support
Adds support for NVMe metadata formats and exposes block devices for
all namespaces regardless of their format. Namespace formats that are
unusable will have disk capacity set to 0, but a handle to the block
device is created to simplify device management. A namespace is not
usable when the format requires host interleave block and metadata in
single buffer, has no provisioned storage, or has better data but failed
to register with blk integrity.
The namespace has to be scanned in two phases to support separate
metadata formats. The first establishes the sector size and capacity
prior to invoking add_disk. If metadata is required, the capacity will
be temporarilly set to 0 until it can be revalidated and registered with
the integrity extenstions after add_disk completes.
The driver relies on the integrity extensions to provide the metadata
buffer. NVMe requires this be a single physically contiguous region,
so only one integrity segment is allowed per command. If the metadata
is used for T10 PI, the driver provides mappings to save and restore
the reftag physical block translation. The driver provides no-op
functions for generate and verify if metadata is not used for protection
information. This way the setup is always provided by the block layer.
If a request does not supply a required metadata buffer, the command
is failed with bad address. This could only happen if a user manually
disables verify/generate on such a disk. The only exception to where
this is okay is if the controller is capable of stripping/generating
the metadata, which is possible on some types of formats.
The metadata scatter gather list now occupies the spot in the nvme_iod
that used to be used to link retryable IOD's, but we don't do that
anymore, so the field was unused.
Signed-off-by: Keith Busch <keith.busch@intel.com>
2015-02-20 03:39:03 +07:00
|
|
|
|
2016-12-09 05:20:32 +07:00
|
|
|
nvme_cleanup_cmd(req);
|
2015-11-28 21:43:10 +07:00
|
|
|
nvme_free_iod(dev, req);
|
2015-11-26 16:51:23 +07:00
|
|
|
}
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2015-05-22 16:12:46 +07:00
|
|
|
/*
|
|
|
|
* NOTE: ns is NULL when called on the admin queue.
|
|
|
|
*/
|
2017-06-03 14:38:05 +07:00
|
|
|
static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
|
2014-11-04 22:20:14 +07:00
|
|
|
const struct blk_mq_queue_data *bd)
|
2014-04-04 05:45:23 +07:00
|
|
|
{
|
2014-11-04 22:20:14 +07:00
|
|
|
struct nvme_ns *ns = hctx->queue->queuedata;
|
|
|
|
struct nvme_queue *nvmeq = hctx->driver_data;
|
2015-05-22 16:12:46 +07:00
|
|
|
struct nvme_dev *dev = nvmeq->dev;
|
2014-11-04 22:20:14 +07:00
|
|
|
struct request *req = bd->rq;
|
2015-10-16 12:58:38 +07:00
|
|
|
struct nvme_command cmnd;
|
2017-06-12 23:36:32 +07:00
|
|
|
blk_status_t ret;
|
NVMe: Metadata format support
Adds support for NVMe metadata formats and exposes block devices for
all namespaces regardless of their format. Namespace formats that are
unusable will have disk capacity set to 0, but a handle to the block
device is created to simplify device management. A namespace is not
usable when the format requires host interleave block and metadata in
single buffer, has no provisioned storage, or has better data but failed
to register with blk integrity.
The namespace has to be scanned in two phases to support separate
metadata formats. The first establishes the sector size and capacity
prior to invoking add_disk. If metadata is required, the capacity will
be temporarilly set to 0 until it can be revalidated and registered with
the integrity extenstions after add_disk completes.
The driver relies on the integrity extensions to provide the metadata
buffer. NVMe requires this be a single physically contiguous region,
so only one integrity segment is allowed per command. If the metadata
is used for T10 PI, the driver provides mappings to save and restore
the reftag physical block translation. The driver provides no-op
functions for generate and verify if metadata is not used for protection
information. This way the setup is always provided by the block layer.
If a request does not supply a required metadata buffer, the command
is failed with bad address. This could only happen if a user manually
disables verify/generate on such a disk. The only exception to where
this is okay is if the controller is capable of stripping/generating
the metadata, which is possible on some types of formats.
The metadata scatter gather list now occupies the spot in the nvme_iod
that used to be used to link retryable IOD's, but we don't do that
anymore, so the field was unused.
Signed-off-by: Keith Busch <keith.busch@intel.com>
2015-02-20 03:39:03 +07:00
|
|
|
|
2018-05-17 23:31:49 +07:00
|
|
|
/*
|
|
|
|
* We should not need to do this, but we're still using this to
|
|
|
|
* ensure we can drain requests on a dying queue.
|
|
|
|
*/
|
|
|
|
if (unlikely(nvmeq->cq_vector < 0))
|
|
|
|
return BLK_STS_IOERR;
|
|
|
|
|
2016-12-09 05:20:32 +07:00
|
|
|
ret = nvme_setup_cmd(ns, req, &cmnd);
|
2017-06-03 14:38:05 +07:00
|
|
|
if (ret)
|
2015-11-28 21:43:10 +07:00
|
|
|
return ret;
|
2014-11-04 22:20:14 +07:00
|
|
|
|
2017-01-13 18:29:12 +07:00
|
|
|
ret = nvme_init_iod(req, dev);
|
2017-06-03 14:38:05 +07:00
|
|
|
if (ret)
|
2016-12-09 05:20:32 +07:00
|
|
|
goto out_free_cmd;
|
2014-11-04 22:20:14 +07:00
|
|
|
|
2017-06-03 14:38:05 +07:00
|
|
|
if (blk_rq_nr_phys_segments(req)) {
|
2017-01-13 18:29:12 +07:00
|
|
|
ret = nvme_map_data(dev, req, &cmnd);
|
2017-06-03 14:38:05 +07:00
|
|
|
if (ret)
|
|
|
|
goto out_cleanup_iod;
|
|
|
|
}
|
2014-11-04 22:20:14 +07:00
|
|
|
|
2015-11-26 18:59:50 +07:00
|
|
|
blk_mq_start_request(req);
|
2018-05-26 18:45:55 +07:00
|
|
|
nvme_submit_cmd(nvmeq, &cmnd);
|
2017-06-03 14:38:05 +07:00
|
|
|
return BLK_STS_OK;
|
2016-12-09 05:20:32 +07:00
|
|
|
out_cleanup_iod:
|
2015-11-28 21:43:10 +07:00
|
|
|
nvme_free_iod(dev, req);
|
2016-12-09 05:20:32 +07:00
|
|
|
out_free_cmd:
|
|
|
|
nvme_cleanup_cmd(req);
|
2015-10-16 12:58:38 +07:00
|
|
|
return ret;
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
NVMe: Metadata format support
Adds support for NVMe metadata formats and exposes block devices for
all namespaces regardless of their format. Namespace formats that are
unusable will have disk capacity set to 0, but a handle to the block
device is created to simplify device management. A namespace is not
usable when the format requires host interleave block and metadata in
single buffer, has no provisioned storage, or has better data but failed
to register with blk integrity.
The namespace has to be scanned in two phases to support separate
metadata formats. The first establishes the sector size and capacity
prior to invoking add_disk. If metadata is required, the capacity will
be temporarilly set to 0 until it can be revalidated and registered with
the integrity extenstions after add_disk completes.
The driver relies on the integrity extensions to provide the metadata
buffer. NVMe requires this be a single physically contiguous region,
so only one integrity segment is allowed per command. If the metadata
is used for T10 PI, the driver provides mappings to save and restore
the reftag physical block translation. The driver provides no-op
functions for generate and verify if metadata is not used for protection
information. This way the setup is always provided by the block layer.
If a request does not supply a required metadata buffer, the command
is failed with bad address. This could only happen if a user manually
disables verify/generate on such a disk. The only exception to where
this is okay is if the controller is capable of stripping/generating
the metadata, which is possible on some types of formats.
The metadata scatter gather list now occupies the spot in the nvme_iod
that used to be used to link retryable IOD's, but we don't do that
anymore, so the field was unused.
Signed-off-by: Keith Busch <keith.busch@intel.com>
2015-02-20 03:39:03 +07:00
|
|
|
|
2017-03-30 18:41:32 +07:00
|
|
|
static void nvme_pci_complete_rq(struct request *req)
|
2015-11-26 19:03:13 +07:00
|
|
|
{
|
2015-11-28 21:43:10 +07:00
|
|
|
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
|
2014-11-04 22:20:14 +07:00
|
|
|
|
2017-03-30 18:41:32 +07:00
|
|
|
nvme_unmap_data(iod->nvmeq->dev, req);
|
|
|
|
nvme_complete_rq(req);
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
|
|
|
|
2016-03-22 22:02:06 +07:00
|
|
|
/* We read the CQE phase first to check if the rest of the entry is valid */
|
2018-05-18 21:37:04 +07:00
|
|
|
static inline bool nvme_cqe_pending(struct nvme_queue *nvmeq)
|
2016-03-22 22:02:06 +07:00
|
|
|
{
|
2018-05-18 21:37:04 +07:00
|
|
|
return (le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) ==
|
|
|
|
nvmeq->cq_phase;
|
2016-03-22 22:02:06 +07:00
|
|
|
}
|
|
|
|
|
2017-06-18 21:28:07 +07:00
|
|
|
static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq)
|
2011-01-21 00:50:14 +07:00
|
|
|
{
|
2017-06-18 21:28:07 +07:00
|
|
|
u16 head = nvmeq->cq_head;
|
2015-11-28 21:42:28 +07:00
|
|
|
|
2018-06-06 21:13:05 +07:00
|
|
|
if (nvme_dbbuf_update_and_check_event(head, nvmeq->dbbuf_cq_db,
|
|
|
|
nvmeq->dbbuf_cq_ei))
|
|
|
|
writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
|
2017-06-18 21:28:07 +07:00
|
|
|
}
|
2015-11-26 18:59:50 +07:00
|
|
|
|
2018-05-17 23:31:50 +07:00
|
|
|
static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx)
|
2017-06-18 21:28:08 +07:00
|
|
|
{
|
2018-05-17 23:31:50 +07:00
|
|
|
volatile struct nvme_completion *cqe = &nvmeq->cqes[idx];
|
2017-06-18 21:28:08 +07:00
|
|
|
struct request *req;
|
2015-11-28 21:42:28 +07:00
|
|
|
|
2017-06-18 21:28:08 +07:00
|
|
|
if (unlikely(cqe->command_id >= nvmeq->q_depth)) {
|
|
|
|
dev_warn(nvmeq->dev->ctrl.device,
|
|
|
|
"invalid id %d completed on queue %d\n",
|
|
|
|
cqe->command_id, le16_to_cpu(cqe->sq_id));
|
|
|
|
return;
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
|
|
|
|
2017-06-18 21:28:08 +07:00
|
|
|
/*
|
|
|
|
* AEN requests are special as they don't time out and can
|
|
|
|
* survive any kind of queue freeze and often don't respond to
|
|
|
|
* aborts. We don't even bother to allocate a struct request
|
|
|
|
* for them but rather special case them here.
|
|
|
|
*/
|
|
|
|
if (unlikely(nvmeq->qid == 0 &&
|
2017-11-08 05:13:10 +07:00
|
|
|
cqe->command_id >= NVME_AQ_BLK_MQ_DEPTH)) {
|
2017-06-18 21:28:08 +07:00
|
|
|
nvme_complete_async_event(&nvmeq->dev->ctrl,
|
|
|
|
cqe->status, &cqe->result);
|
2015-11-04 10:37:26 +07:00
|
|
|
return;
|
2017-06-18 21:28:08 +07:00
|
|
|
}
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2017-06-18 21:28:08 +07:00
|
|
|
req = blk_mq_tag_to_rq(*nvmeq->tags, cqe->command_id);
|
|
|
|
nvme_end_request(req, cqe->status, cqe->result);
|
|
|
|
}
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2018-05-17 23:31:50 +07:00
|
|
|
static void nvme_complete_cqes(struct nvme_queue *nvmeq, u16 start, u16 end)
|
2011-01-21 00:50:14 +07:00
|
|
|
{
|
2018-05-17 23:31:50 +07:00
|
|
|
while (start != end) {
|
|
|
|
nvme_handle_cqe(nvmeq, start);
|
|
|
|
if (++start == nvmeq->q_depth)
|
|
|
|
start = 0;
|
|
|
|
}
|
|
|
|
}
|
2015-11-28 21:42:28 +07:00
|
|
|
|
2018-05-17 23:31:50 +07:00
|
|
|
static inline void nvme_update_cq_head(struct nvme_queue *nvmeq)
|
|
|
|
{
|
|
|
|
if (++nvmeq->cq_head == nvmeq->q_depth) {
|
|
|
|
nvmeq->cq_head = 0;
|
|
|
|
nvmeq->cq_phase = !nvmeq->cq_phase;
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
2015-11-04 10:37:26 +07:00
|
|
|
}
|
|
|
|
|
2018-05-17 23:31:50 +07:00
|
|
|
static inline bool nvme_process_cq(struct nvme_queue *nvmeq, u16 *start,
|
|
|
|
u16 *end, int tag)
|
2015-11-04 10:37:26 +07:00
|
|
|
{
|
2018-05-17 23:31:50 +07:00
|
|
|
bool found = false;
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2018-05-17 23:31:50 +07:00
|
|
|
*start = nvmeq->cq_head;
|
|
|
|
while (!found && nvme_cqe_pending(nvmeq)) {
|
|
|
|
if (nvmeq->cqes[nvmeq->cq_head].command_id == tag)
|
|
|
|
found = true;
|
|
|
|
nvme_update_cq_head(nvmeq);
|
2017-06-18 21:28:09 +07:00
|
|
|
}
|
2018-05-17 23:31:50 +07:00
|
|
|
*end = nvmeq->cq_head;
|
2017-06-18 21:28:07 +07:00
|
|
|
|
2018-05-17 23:31:50 +07:00
|
|
|
if (*start != *end)
|
2017-06-18 21:28:09 +07:00
|
|
|
nvme_ring_cq_doorbell(nvmeq);
|
2018-05-17 23:31:50 +07:00
|
|
|
return found;
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static irqreturn_t nvme_irq(int irq, void *data)
|
2011-02-06 19:28:06 +07:00
|
|
|
{
|
|
|
|
struct nvme_queue *nvmeq = data;
|
2018-05-21 21:41:52 +07:00
|
|
|
irqreturn_t ret = IRQ_NONE;
|
2018-05-17 23:31:50 +07:00
|
|
|
u16 start, end;
|
|
|
|
|
2018-05-17 23:31:51 +07:00
|
|
|
spin_lock(&nvmeq->cq_lock);
|
2018-05-21 21:41:52 +07:00
|
|
|
if (nvmeq->cq_head != nvmeq->last_cq_head)
|
|
|
|
ret = IRQ_HANDLED;
|
2018-05-17 23:31:50 +07:00
|
|
|
nvme_process_cq(nvmeq, &start, &end, -1);
|
2018-05-21 21:41:52 +07:00
|
|
|
nvmeq->last_cq_head = nvmeq->cq_head;
|
2018-05-17 23:31:51 +07:00
|
|
|
spin_unlock(&nvmeq->cq_lock);
|
2018-05-17 23:31:50 +07:00
|
|
|
|
2018-05-21 21:41:52 +07:00
|
|
|
if (start != end) {
|
|
|
|
nvme_complete_cqes(nvmeq, start, end);
|
|
|
|
return IRQ_HANDLED;
|
|
|
|
}
|
|
|
|
|
|
|
|
return ret;
|
2011-02-06 19:28:06 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static irqreturn_t nvme_irq_check(int irq, void *data)
|
|
|
|
{
|
|
|
|
struct nvme_queue *nvmeq = data;
|
2018-05-18 21:37:04 +07:00
|
|
|
if (nvme_cqe_pending(nvmeq))
|
2016-03-22 22:02:06 +07:00
|
|
|
return IRQ_WAKE_THREAD;
|
|
|
|
return IRQ_NONE;
|
2011-02-06 19:28:06 +07:00
|
|
|
}
|
|
|
|
|
2017-02-25 05:59:28 +07:00
|
|
|
static int __nvme_poll(struct nvme_queue *nvmeq, unsigned int tag)
|
2015-11-04 10:37:26 +07:00
|
|
|
{
|
2018-05-17 23:31:50 +07:00
|
|
|
u16 start, end;
|
|
|
|
bool found;
|
2015-11-04 10:37:26 +07:00
|
|
|
|
2018-05-18 21:37:04 +07:00
|
|
|
if (!nvme_cqe_pending(nvmeq))
|
2017-06-18 21:28:10 +07:00
|
|
|
return 0;
|
2015-11-04 10:37:26 +07:00
|
|
|
|
2018-05-17 23:31:51 +07:00
|
|
|
spin_lock_irq(&nvmeq->cq_lock);
|
2018-05-17 23:31:50 +07:00
|
|
|
found = nvme_process_cq(nvmeq, &start, &end, tag);
|
2018-05-17 23:31:51 +07:00
|
|
|
spin_unlock_irq(&nvmeq->cq_lock);
|
2017-06-18 21:28:10 +07:00
|
|
|
|
2018-05-17 23:31:50 +07:00
|
|
|
nvme_complete_cqes(nvmeq, start, end);
|
2017-06-18 21:28:10 +07:00
|
|
|
return found;
|
2015-11-04 10:37:26 +07:00
|
|
|
}
|
|
|
|
|
2017-02-25 05:59:28 +07:00
|
|
|
static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
|
|
|
|
{
|
|
|
|
struct nvme_queue *nvmeq = hctx->driver_data;
|
|
|
|
|
|
|
|
return __nvme_poll(nvmeq, tag);
|
|
|
|
}
|
|
|
|
|
2017-11-08 05:13:12 +07:00
|
|
|
static void nvme_pci_submit_async_event(struct nvme_ctrl *ctrl)
|
2011-01-21 00:50:14 +07:00
|
|
|
{
|
2016-04-26 18:52:00 +07:00
|
|
|
struct nvme_dev *dev = to_nvme_dev(ctrl);
|
2018-01-14 17:39:01 +07:00
|
|
|
struct nvme_queue *nvmeq = &dev->queues[0];
|
2014-11-04 22:20:14 +07:00
|
|
|
struct nvme_command c;
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2014-11-04 22:20:14 +07:00
|
|
|
memset(&c, 0, sizeof(c));
|
|
|
|
c.common.opcode = nvme_admin_async_event;
|
2017-11-08 05:13:12 +07:00
|
|
|
c.common.command_id = NVME_AQ_BLK_MQ_DEPTH;
|
2018-05-26 18:45:55 +07:00
|
|
|
nvme_submit_cmd(nvmeq, &c);
|
2015-05-22 16:12:38 +07:00
|
|
|
}
|
|
|
|
|
2011-01-21 00:50:14 +07:00
|
|
|
static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
|
2015-05-22 16:12:38 +07:00
|
|
|
{
|
2011-01-21 00:50:14 +07:00
|
|
|
struct nvme_command c;
|
|
|
|
|
|
|
|
memset(&c, 0, sizeof(c));
|
|
|
|
c.delete_queue.opcode = opcode;
|
|
|
|
c.delete_queue.qid = cpu_to_le16(id);
|
|
|
|
|
2015-11-26 16:06:56 +07:00
|
|
|
return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
|
2018-05-24 16:51:33 +07:00
|
|
|
struct nvme_queue *nvmeq, s16 vector)
|
2011-01-21 00:50:14 +07:00
|
|
|
{
|
|
|
|
struct nvme_command c;
|
|
|
|
int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;
|
|
|
|
|
2015-05-22 16:12:46 +07:00
|
|
|
/*
|
2017-10-18 20:56:09 +07:00
|
|
|
* Note: we (ab)use the fact that the prp fields survive if no data
|
2015-05-22 16:12:46 +07:00
|
|
|
* is attached to the request.
|
|
|
|
*/
|
2011-01-21 00:50:14 +07:00
|
|
|
memset(&c, 0, sizeof(c));
|
|
|
|
c.create_cq.opcode = nvme_admin_create_cq;
|
|
|
|
c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
|
|
|
|
c.create_cq.cqid = cpu_to_le16(qid);
|
|
|
|
c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
|
|
|
|
c.create_cq.cq_flags = cpu_to_le16(flags);
|
2018-05-24 16:51:33 +07:00
|
|
|
c.create_cq.irq_vector = cpu_to_le16(vector);
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2015-11-26 16:06:56 +07:00
|
|
|
return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
|
|
|
|
struct nvme_queue *nvmeq)
|
|
|
|
{
|
2018-05-08 23:25:15 +07:00
|
|
|
struct nvme_ctrl *ctrl = &dev->ctrl;
|
2011-01-21 00:50:14 +07:00
|
|
|
struct nvme_command c;
|
2017-04-05 05:18:12 +07:00
|
|
|
int flags = NVME_QUEUE_PHYS_CONTIG;
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2018-05-08 23:25:15 +07:00
|
|
|
/*
|
|
|
|
* Some drives have a bug that auto-enables WRRU if MEDIUM isn't
|
|
|
|
* set. Since URGENT priority is zeroes, it makes all queues
|
|
|
|
* URGENT.
|
|
|
|
*/
|
|
|
|
if (ctrl->quirks & NVME_QUIRK_MEDIUM_PRIO_SQ)
|
|
|
|
flags |= NVME_SQ_PRIO_MEDIUM;
|
|
|
|
|
2015-05-22 16:12:46 +07:00
|
|
|
/*
|
2017-10-18 20:56:09 +07:00
|
|
|
* Note: we (ab)use the fact that the prp fields survive if no data
|
2015-05-22 16:12:46 +07:00
|
|
|
* is attached to the request.
|
|
|
|
*/
|
2011-01-21 00:50:14 +07:00
|
|
|
memset(&c, 0, sizeof(c));
|
|
|
|
c.create_sq.opcode = nvme_admin_create_sq;
|
|
|
|
c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
|
|
|
|
c.create_sq.sqid = cpu_to_le16(qid);
|
|
|
|
c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
|
|
|
|
c.create_sq.sq_flags = cpu_to_le16(flags);
|
|
|
|
c.create_sq.cqid = cpu_to_le16(qid);
|
|
|
|
|
2015-11-26 16:06:56 +07:00
|
|
|
return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
|
|
|
|
{
|
|
|
|
return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
|
|
|
|
{
|
|
|
|
return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
|
|
|
|
}
|
|
|
|
|
2017-06-03 14:38:04 +07:00
|
|
|
static void abort_endio(struct request *req, blk_status_t error)
|
2011-09-20 04:08:14 +07:00
|
|
|
{
|
2015-11-28 21:43:10 +07:00
|
|
|
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
|
|
|
|
struct nvme_queue *nvmeq = iod->nvmeq;
|
2015-06-28 01:20:34 +07:00
|
|
|
|
2017-04-20 21:02:57 +07:00
|
|
|
dev_warn(nvmeq->dev->ctrl.device,
|
|
|
|
"Abort status: 0x%x", nvme_req(req)->status);
|
2015-11-16 16:39:48 +07:00
|
|
|
atomic_inc(&nvmeq->dev->ctrl.abort_limit);
|
|
|
|
blk_mq_free_request(req);
|
2011-09-20 04:08:14 +07:00
|
|
|
}
|
|
|
|
|
2017-06-08 01:32:50 +07:00
|
|
|
static bool nvme_should_reset(struct nvme_dev *dev, u32 csts)
|
|
|
|
{
|
|
|
|
|
|
|
|
/* If true, indicates loss of adapter communication, possibly by a
|
|
|
|
* NVMe Subsystem reset.
|
|
|
|
*/
|
|
|
|
bool nssro = dev->subsystem && (csts & NVME_CSTS_NSSRO);
|
|
|
|
|
2018-01-22 21:03:16 +07:00
|
|
|
/* If there is a reset/reinit ongoing, we shouldn't reset again. */
|
|
|
|
switch (dev->ctrl.state) {
|
|
|
|
case NVME_CTRL_RESETTING:
|
2018-01-31 23:31:24 +07:00
|
|
|
case NVME_CTRL_CONNECTING:
|
2017-06-08 01:32:50 +07:00
|
|
|
return false;
|
2018-01-22 21:03:16 +07:00
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
2017-06-08 01:32:50 +07:00
|
|
|
|
|
|
|
/* We shouldn't reset unless the controller is on fatal error state
|
|
|
|
* _or_ if we lost the communication with it.
|
|
|
|
*/
|
|
|
|
if (!(csts & NVME_CSTS_CFS) && !nssro)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void nvme_warn_reset(struct nvme_dev *dev, u32 csts)
|
|
|
|
{
|
|
|
|
/* Read a config register to help see what died. */
|
|
|
|
u16 pci_status;
|
|
|
|
int result;
|
|
|
|
|
|
|
|
result = pci_read_config_word(to_pci_dev(dev->dev), PCI_STATUS,
|
|
|
|
&pci_status);
|
|
|
|
if (result == PCIBIOS_SUCCESSFUL)
|
|
|
|
dev_warn(dev->ctrl.device,
|
|
|
|
"controller is down; will reset: CSTS=0x%x, PCI_STATUS=0x%hx\n",
|
|
|
|
csts, pci_status);
|
|
|
|
else
|
|
|
|
dev_warn(dev->ctrl.device,
|
|
|
|
"controller is down; will reset: CSTS=0x%x, PCI_STATUS read failed (%d)\n",
|
|
|
|
csts, result);
|
|
|
|
}
|
|
|
|
|
2015-10-22 19:03:35 +07:00
|
|
|
static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
|
2013-12-11 03:10:38 +07:00
|
|
|
{
|
2015-11-28 21:43:10 +07:00
|
|
|
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
|
|
|
|
struct nvme_queue *nvmeq = iod->nvmeq;
|
2013-12-11 03:10:38 +07:00
|
|
|
struct nvme_dev *dev = nvmeq->dev;
|
2014-11-04 22:20:14 +07:00
|
|
|
struct request *abort_req;
|
|
|
|
struct nvme_command cmd;
|
2017-06-08 01:32:50 +07:00
|
|
|
u32 csts = readl(dev->bar + NVME_REG_CSTS);
|
|
|
|
|
2018-02-16 03:05:10 +07:00
|
|
|
/* If PCI error recovery process is happening, we cannot reset or
|
|
|
|
* the recovery mechanism will surely fail.
|
|
|
|
*/
|
|
|
|
mb();
|
|
|
|
if (pci_channel_offline(to_pci_dev(dev->dev)))
|
|
|
|
return BLK_EH_RESET_TIMER;
|
|
|
|
|
2017-06-08 01:32:50 +07:00
|
|
|
/*
|
|
|
|
* Reset immediately if the controller is failed
|
|
|
|
*/
|
|
|
|
if (nvme_should_reset(dev, csts)) {
|
|
|
|
nvme_warn_reset(dev, csts);
|
|
|
|
nvme_dev_disable(dev, false);
|
2017-06-15 20:41:08 +07:00
|
|
|
nvme_reset_ctrl(&dev->ctrl);
|
2018-05-29 20:52:30 +07:00
|
|
|
return BLK_EH_DONE;
|
2017-06-08 01:32:50 +07:00
|
|
|
}
|
2013-12-11 03:10:38 +07:00
|
|
|
|
2017-02-25 05:59:28 +07:00
|
|
|
/*
|
|
|
|
* Did we miss an interrupt?
|
|
|
|
*/
|
|
|
|
if (__nvme_poll(nvmeq, req->tag)) {
|
|
|
|
dev_warn(dev->ctrl.device,
|
|
|
|
"I/O %d QID %d timeout, completion polled\n",
|
|
|
|
req->tag, nvmeq->qid);
|
2018-05-29 20:52:30 +07:00
|
|
|
return BLK_EH_DONE;
|
2017-02-25 05:59:28 +07:00
|
|
|
}
|
|
|
|
|
2015-10-22 19:03:35 +07:00
|
|
|
/*
|
2015-11-26 18:42:26 +07:00
|
|
|
* Shutdown immediately if controller times out while starting. The
|
|
|
|
* reset work will see the pci device disabled when it gets the forced
|
|
|
|
* cancellation error. All outstanding requests are completed on
|
2018-05-29 20:52:30 +07:00
|
|
|
* shutdown, so we return BLK_EH_DONE.
|
2015-11-26 18:42:26 +07:00
|
|
|
*/
|
2018-02-08 22:55:34 +07:00
|
|
|
switch (dev->ctrl.state) {
|
|
|
|
case NVME_CTRL_CONNECTING:
|
|
|
|
case NVME_CTRL_RESETTING:
|
2018-05-25 03:34:55 +07:00
|
|
|
dev_warn_ratelimited(dev->ctrl.device,
|
2015-11-26 18:42:26 +07:00
|
|
|
"I/O %d QID %d timeout, disable controller\n",
|
|
|
|
req->tag, nvmeq->qid);
|
2016-01-13 04:41:18 +07:00
|
|
|
nvme_dev_disable(dev, false);
|
2017-04-20 21:02:57 +07:00
|
|
|
nvme_req(req)->flags |= NVME_REQ_CANCELLED;
|
2018-05-29 20:52:30 +07:00
|
|
|
return BLK_EH_DONE;
|
2018-02-08 22:55:34 +07:00
|
|
|
default:
|
|
|
|
break;
|
2013-12-11 03:10:38 +07:00
|
|
|
}
|
|
|
|
|
2015-11-26 18:42:26 +07:00
|
|
|
/*
|
|
|
|
* Shutdown the controller immediately and schedule a reset if the
|
|
|
|
* command was already aborted once before and still hasn't been
|
|
|
|
* returned to the driver, or if this is the admin queue.
|
2015-10-22 19:03:35 +07:00
|
|
|
*/
|
2015-11-28 21:43:10 +07:00
|
|
|
if (!nvmeq->qid || iod->aborted) {
|
2016-02-10 22:51:15 +07:00
|
|
|
dev_warn(dev->ctrl.device,
|
2015-11-26 18:11:07 +07:00
|
|
|
"I/O %d QID %d timeout, reset controller\n",
|
|
|
|
req->tag, nvmeq->qid);
|
2016-01-13 04:41:18 +07:00
|
|
|
nvme_dev_disable(dev, false);
|
2017-06-15 20:41:08 +07:00
|
|
|
nvme_reset_ctrl(&dev->ctrl);
|
2013-12-11 03:10:38 +07:00
|
|
|
|
2017-04-20 21:02:57 +07:00
|
|
|
nvme_req(req)->flags |= NVME_REQ_CANCELLED;
|
2018-05-29 20:52:30 +07:00
|
|
|
return BLK_EH_DONE;
|
2013-12-11 03:10:38 +07:00
|
|
|
}
|
|
|
|
|
2015-11-16 16:39:48 +07:00
|
|
|
if (atomic_dec_return(&dev->ctrl.abort_limit) < 0) {
|
2015-11-20 15:36:44 +07:00
|
|
|
atomic_inc(&dev->ctrl.abort_limit);
|
2015-10-22 19:03:35 +07:00
|
|
|
return BLK_EH_RESET_TIMER;
|
2015-11-20 15:36:44 +07:00
|
|
|
}
|
2017-01-25 06:07:00 +07:00
|
|
|
iod->aborted = 1;
|
2014-11-04 22:20:14 +07:00
|
|
|
|
2013-12-11 03:10:38 +07:00
|
|
|
memset(&cmd, 0, sizeof(cmd));
|
|
|
|
cmd.abort.opcode = nvme_admin_abort_cmd;
|
2014-11-04 22:20:14 +07:00
|
|
|
cmd.abort.cid = req->tag;
|
2013-12-11 03:10:38 +07:00
|
|
|
cmd.abort.sqid = cpu_to_le16(nvmeq->qid);
|
|
|
|
|
2016-02-10 22:51:15 +07:00
|
|
|
dev_warn(nvmeq->dev->ctrl.device,
|
|
|
|
"I/O %d QID %d timeout, aborting\n",
|
|
|
|
req->tag, nvmeq->qid);
|
2015-11-16 16:39:48 +07:00
|
|
|
|
|
|
|
abort_req = nvme_alloc_request(dev->ctrl.admin_q, &cmd,
|
2016-06-13 21:45:23 +07:00
|
|
|
BLK_MQ_REQ_NOWAIT, NVME_QID_ANY);
|
2015-11-16 16:39:48 +07:00
|
|
|
if (IS_ERR(abort_req)) {
|
|
|
|
atomic_inc(&dev->ctrl.abort_limit);
|
|
|
|
return BLK_EH_RESET_TIMER;
|
|
|
|
}
|
|
|
|
|
|
|
|
abort_req->timeout = ADMIN_TIMEOUT;
|
|
|
|
abort_req->end_io_data = NULL;
|
|
|
|
blk_execute_rq_nowait(abort_req->q, NULL, abort_req, 0, abort_endio);
|
2013-12-11 03:10:38 +07:00
|
|
|
|
2015-10-22 19:03:35 +07:00
|
|
|
/*
|
|
|
|
* The aborted req will be completed on receiving the abort req.
|
|
|
|
* We enable the timer again. If hit twice, it'll cause a device reset,
|
|
|
|
* as the device then is in a faulty state.
|
|
|
|
*/
|
|
|
|
return BLK_EH_RESET_TIMER;
|
2013-12-11 03:10:38 +07:00
|
|
|
}
|
|
|
|
|
2014-11-04 22:20:14 +07:00
|
|
|
static void nvme_free_queue(struct nvme_queue *nvmeq)
|
|
|
|
{
|
2012-08-04 00:55:56 +07:00
|
|
|
dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
|
|
|
|
(void *)nvmeq->cqes, nvmeq->cq_dma_addr);
|
2015-07-20 23:14:09 +07:00
|
|
|
if (nvmeq->sq_cmds)
|
|
|
|
dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
|
2012-08-04 00:55:56 +07:00
|
|
|
nvmeq->sq_cmds, nvmeq->sq_dma_addr);
|
|
|
|
}
|
|
|
|
|
2013-12-17 01:50:00 +07:00
|
|
|
static void nvme_free_queues(struct nvme_dev *dev, int lowest)
|
2013-07-16 04:02:20 +07:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
2017-04-24 14:58:29 +07:00
|
|
|
for (i = dev->ctrl.queue_count - 1; i >= lowest; i--) {
|
|
|
|
dev->ctrl.queue_count--;
|
2018-01-14 17:39:01 +07:00
|
|
|
nvme_free_queue(&dev->queues[i]);
|
2015-01-15 11:01:58 +07:00
|
|
|
}
|
2013-07-16 04:02:20 +07:00
|
|
|
}
|
|
|
|
|
2013-12-11 03:10:40 +07:00
|
|
|
/**
|
|
|
|
* nvme_suspend_queue - put queue into suspended state
|
|
|
|
* @nvmeq - queue to suspend
|
|
|
|
*/
|
|
|
|
static int nvme_suspend_queue(struct nvme_queue *nvmeq)
|
2011-01-21 00:50:14 +07:00
|
|
|
{
|
2014-12-23 02:59:04 +07:00
|
|
|
int vector;
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2018-05-17 23:31:51 +07:00
|
|
|
spin_lock_irq(&nvmeq->cq_lock);
|
2014-12-23 02:59:04 +07:00
|
|
|
if (nvmeq->cq_vector == -1) {
|
2018-05-17 23:31:51 +07:00
|
|
|
spin_unlock_irq(&nvmeq->cq_lock);
|
2014-12-23 02:59:04 +07:00
|
|
|
return 1;
|
|
|
|
}
|
2017-04-13 14:06:43 +07:00
|
|
|
vector = nvmeq->cq_vector;
|
2014-03-24 23:46:25 +07:00
|
|
|
nvmeq->dev->online_queues--;
|
2014-12-23 02:59:04 +07:00
|
|
|
nvmeq->cq_vector = -1;
|
2018-05-17 23:31:51 +07:00
|
|
|
spin_unlock_irq(&nvmeq->cq_lock);
|
2012-08-08 02:56:23 +07:00
|
|
|
|
2018-05-17 23:31:49 +07:00
|
|
|
/*
|
|
|
|
* Ensure that nvme_queue_rq() sees it ->cq_vector == -1 without
|
|
|
|
* having to grab the lock.
|
|
|
|
*/
|
|
|
|
mb();
|
2012-08-08 02:56:23 +07:00
|
|
|
|
2015-11-26 16:06:56 +07:00
|
|
|
if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
|
2017-07-02 19:53:27 +07:00
|
|
|
blk_mq_quiesce_queue(nvmeq->dev->ctrl.admin_q);
|
2015-03-27 02:49:33 +07:00
|
|
|
|
2017-04-13 14:06:43 +07:00
|
|
|
pci_free_irq(to_pci_dev(nvmeq->dev->dev), vector, nvmeq);
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2013-12-11 03:10:40 +07:00
|
|
|
return 0;
|
|
|
|
}
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2016-01-13 04:41:18 +07:00
|
|
|
static void nvme_disable_admin_queue(struct nvme_dev *dev, bool shutdown)
|
2013-12-11 03:10:40 +07:00
|
|
|
{
|
2018-01-14 17:39:01 +07:00
|
|
|
struct nvme_queue *nvmeq = &dev->queues[0];
|
2018-05-17 23:31:50 +07:00
|
|
|
u16 start, end;
|
2013-12-11 03:10:40 +07:00
|
|
|
|
2016-01-13 04:41:18 +07:00
|
|
|
if (shutdown)
|
|
|
|
nvme_shutdown_ctrl(&dev->ctrl);
|
|
|
|
else
|
2017-06-28 02:16:38 +07:00
|
|
|
nvme_disable_ctrl(&dev->ctrl, dev->ctrl.cap);
|
2015-02-20 00:34:48 +07:00
|
|
|
|
2018-05-17 23:31:51 +07:00
|
|
|
spin_lock_irq(&nvmeq->cq_lock);
|
2018-05-17 23:31:50 +07:00
|
|
|
nvme_process_cq(nvmeq, &start, &end, -1);
|
2018-05-17 23:31:51 +07:00
|
|
|
spin_unlock_irq(&nvmeq->cq_lock);
|
2018-05-17 23:31:50 +07:00
|
|
|
|
|
|
|
nvme_complete_cqes(nvmeq, start, end);
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
|
|
|
|
2015-07-20 23:14:09 +07:00
|
|
|
static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
|
|
|
|
int entry_size)
|
|
|
|
{
|
|
|
|
int q_depth = dev->q_depth;
|
2015-11-28 21:03:49 +07:00
|
|
|
unsigned q_size_aligned = roundup(q_depth * entry_size,
|
|
|
|
dev->ctrl.page_size);
|
2015-07-20 23:14:09 +07:00
|
|
|
|
|
|
|
if (q_size_aligned * nr_io_queues > dev->cmb_size) {
|
2015-07-22 04:08:13 +07:00
|
|
|
u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues);
|
2015-11-28 21:03:49 +07:00
|
|
|
mem_per_q = round_down(mem_per_q, dev->ctrl.page_size);
|
2015-07-22 04:08:13 +07:00
|
|
|
q_depth = div_u64(mem_per_q, entry_size);
|
2015-07-20 23:14:09 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Ensure the reduced q_depth is above some threshold where it
|
|
|
|
* would be better to map queues in system memory with the
|
|
|
|
* original depth
|
|
|
|
*/
|
|
|
|
if (q_depth < 64)
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
return q_depth;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq,
|
|
|
|
int qid, int depth)
|
|
|
|
{
|
2018-02-13 19:44:44 +07:00
|
|
|
/* CMB SQEs will be mapped before creation */
|
|
|
|
if (qid && dev->cmb && use_cmb_sqes && (dev->cmbsz & NVME_CMBSZ_SQS))
|
|
|
|
return 0;
|
2015-07-20 23:14:09 +07:00
|
|
|
|
2018-02-13 19:44:44 +07:00
|
|
|
nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(depth),
|
|
|
|
&nvmeq->sq_dma_addr, GFP_KERNEL);
|
|
|
|
if (!nvmeq->sq_cmds)
|
|
|
|
return -ENOMEM;
|
2015-07-20 23:14:09 +07:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-04-12 22:16:09 +07:00
|
|
|
static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth)
|
2011-01-21 00:50:14 +07:00
|
|
|
{
|
2018-01-14 17:39:01 +07:00
|
|
|
struct nvme_queue *nvmeq = &dev->queues[qid];
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2018-01-23 23:16:19 +07:00
|
|
|
if (dev->ctrl.queue_count > qid)
|
|
|
|
return 0;
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2015-05-22 16:12:39 +07:00
|
|
|
nvmeq->cqes = dma_zalloc_coherent(dev->dev, CQ_SIZE(depth),
|
2014-06-16 03:37:33 +07:00
|
|
|
&nvmeq->cq_dma_addr, GFP_KERNEL);
|
2011-01-21 00:50:14 +07:00
|
|
|
if (!nvmeq->cqes)
|
|
|
|
goto free_nvmeq;
|
|
|
|
|
2015-07-20 23:14:09 +07:00
|
|
|
if (nvme_alloc_sq_cmds(dev, nvmeq, qid, depth))
|
2011-01-21 00:50:14 +07:00
|
|
|
goto free_cqdma;
|
|
|
|
|
2015-05-22 16:12:39 +07:00
|
|
|
nvmeq->q_dmadev = dev->dev;
|
2011-02-10 21:56:01 +07:00
|
|
|
nvmeq->dev = dev;
|
2018-05-17 23:31:51 +07:00
|
|
|
spin_lock_init(&nvmeq->sq_lock);
|
|
|
|
spin_lock_init(&nvmeq->cq_lock);
|
2011-01-21 00:50:14 +07:00
|
|
|
nvmeq->cq_head = 0;
|
2011-01-21 01:24:06 +07:00
|
|
|
nvmeq->cq_phase = 1;
|
2013-09-10 10:25:37 +07:00
|
|
|
nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
|
2011-01-21 00:50:14 +07:00
|
|
|
nvmeq->q_depth = depth;
|
2013-12-11 03:10:38 +07:00
|
|
|
nvmeq->qid = qid;
|
2015-07-01 00:22:52 +07:00
|
|
|
nvmeq->cq_vector = -1;
|
2017-04-24 14:58:29 +07:00
|
|
|
dev->ctrl.queue_count++;
|
2015-05-28 01:26:23 +07:00
|
|
|
|
2018-01-14 17:39:01 +07:00
|
|
|
return 0;
|
2011-01-21 00:50:14 +07:00
|
|
|
|
|
|
|
free_cqdma:
|
2015-05-22 16:12:39 +07:00
|
|
|
dma_free_coherent(dev->dev, CQ_SIZE(depth), (void *)nvmeq->cqes,
|
2011-01-21 00:50:14 +07:00
|
|
|
nvmeq->cq_dma_addr);
|
|
|
|
free_nvmeq:
|
2018-01-14 17:39:01 +07:00
|
|
|
return -ENOMEM;
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
|
|
|
|
2016-09-14 21:18:57 +07:00
|
|
|
static int queue_request_irq(struct nvme_queue *nvmeq)
|
2011-01-20 21:10:15 +07:00
|
|
|
{
|
2017-04-13 14:06:43 +07:00
|
|
|
struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
|
|
|
|
int nr = nvmeq->dev->ctrl.instance;
|
|
|
|
|
|
|
|
if (use_threaded_interrupts) {
|
|
|
|
return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq_check,
|
|
|
|
nvme_irq, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
|
|
|
|
} else {
|
|
|
|
return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq,
|
|
|
|
NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
|
|
|
|
}
|
2011-01-20 21:10:15 +07:00
|
|
|
}
|
|
|
|
|
2013-07-16 04:02:20 +07:00
|
|
|
static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
|
2011-01-21 00:50:14 +07:00
|
|
|
{
|
2013-07-16 04:02:20 +07:00
|
|
|
struct nvme_dev *dev = nvmeq->dev;
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2018-05-17 23:31:51 +07:00
|
|
|
spin_lock_irq(&nvmeq->cq_lock);
|
2013-07-16 04:02:20 +07:00
|
|
|
nvmeq->sq_tail = 0;
|
|
|
|
nvmeq->cq_head = 0;
|
|
|
|
nvmeq->cq_phase = 1;
|
2013-09-10 10:25:37 +07:00
|
|
|
nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
|
2013-07-16 04:02:20 +07:00
|
|
|
memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
|
2017-04-10 22:51:07 +07:00
|
|
|
nvme_dbbuf_init(dev, nvmeq, qid);
|
2014-03-24 23:46:25 +07:00
|
|
|
dev->online_queues++;
|
2018-05-17 23:31:51 +07:00
|
|
|
spin_unlock_irq(&nvmeq->cq_lock);
|
2013-07-16 04:02:20 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
|
|
|
|
{
|
|
|
|
struct nvme_dev *dev = nvmeq->dev;
|
|
|
|
int result;
|
2018-05-24 16:51:33 +07:00
|
|
|
s16 vector;
|
2011-02-01 20:39:04 +07:00
|
|
|
|
2018-02-13 19:44:44 +07:00
|
|
|
if (dev->cmb && use_cmb_sqes && (dev->cmbsz & NVME_CMBSZ_SQS)) {
|
|
|
|
unsigned offset = (qid - 1) * roundup(SQ_SIZE(nvmeq->q_depth),
|
|
|
|
dev->ctrl.page_size);
|
|
|
|
nvmeq->sq_dma_addr = dev->cmb_bus_addr + offset;
|
|
|
|
nvmeq->sq_cmds_io = dev->cmb + offset;
|
|
|
|
}
|
|
|
|
|
2018-04-12 22:16:10 +07:00
|
|
|
/*
|
|
|
|
* A queue's vector matches the queue identifier unless the controller
|
|
|
|
* has only one vector available.
|
|
|
|
*/
|
2018-05-24 16:51:33 +07:00
|
|
|
vector = dev->num_vecs == 1 ? 0 : qid;
|
|
|
|
result = adapter_alloc_cq(dev, qid, nvmeq, vector);
|
2018-06-06 21:13:06 +07:00
|
|
|
if (result)
|
|
|
|
return result;
|
2011-01-21 00:50:14 +07:00
|
|
|
|
|
|
|
result = adapter_alloc_sq(dev, qid, nvmeq);
|
|
|
|
if (result < 0)
|
2018-06-06 21:13:06 +07:00
|
|
|
return result;
|
|
|
|
else if (result)
|
2011-01-21 00:50:14 +07:00
|
|
|
goto release_cq;
|
|
|
|
|
2018-05-24 16:51:33 +07:00
|
|
|
/*
|
|
|
|
* Set cq_vector after alloc cq/sq, otherwise nvme_suspend_queue will
|
|
|
|
* invoke free_irq for it and cause a 'Trying to free already-free IRQ
|
|
|
|
* xxx' warning if the create CQ/SQ command times out.
|
|
|
|
*/
|
|
|
|
nvmeq->cq_vector = vector;
|
2017-09-15 00:54:39 +07:00
|
|
|
nvme_init_queue(nvmeq, qid);
|
2016-09-14 21:18:57 +07:00
|
|
|
result = queue_request_irq(nvmeq);
|
2011-01-21 00:50:14 +07:00
|
|
|
if (result < 0)
|
|
|
|
goto release_sq;
|
|
|
|
|
2013-07-16 04:02:20 +07:00
|
|
|
return result;
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2018-05-24 16:51:33 +07:00
|
|
|
release_sq:
|
|
|
|
nvmeq->cq_vector = -1;
|
2018-02-15 18:13:41 +07:00
|
|
|
dev->online_queues--;
|
2011-01-21 00:50:14 +07:00
|
|
|
adapter_delete_sq(dev, qid);
|
2018-05-24 16:51:33 +07:00
|
|
|
release_cq:
|
2011-01-21 00:50:14 +07:00
|
|
|
adapter_delete_cq(dev, qid);
|
2013-07-16 04:02:20 +07:00
|
|
|
return result;
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
|
|
|
|
2017-03-31 03:39:16 +07:00
|
|
|
static const struct blk_mq_ops nvme_mq_admin_ops = {
|
2015-05-22 16:12:46 +07:00
|
|
|
.queue_rq = nvme_queue_rq,
|
2017-03-30 18:41:32 +07:00
|
|
|
.complete = nvme_pci_complete_rq,
|
2014-11-04 22:20:14 +07:00
|
|
|
.init_hctx = nvme_admin_init_hctx,
|
2015-06-08 23:08:13 +07:00
|
|
|
.exit_hctx = nvme_admin_exit_hctx,
|
2017-06-13 14:15:18 +07:00
|
|
|
.init_request = nvme_init_request,
|
2014-11-04 22:20:14 +07:00
|
|
|
.timeout = nvme_timeout,
|
|
|
|
};
|
|
|
|
|
2017-03-31 03:39:16 +07:00
|
|
|
static const struct blk_mq_ops nvme_mq_ops = {
|
2014-11-04 22:20:14 +07:00
|
|
|
.queue_rq = nvme_queue_rq,
|
2017-03-30 18:41:32 +07:00
|
|
|
.complete = nvme_pci_complete_rq,
|
2014-11-04 22:20:14 +07:00
|
|
|
.init_hctx = nvme_init_hctx,
|
|
|
|
.init_request = nvme_init_request,
|
2016-09-14 21:18:57 +07:00
|
|
|
.map_queues = nvme_pci_map_queues,
|
2014-11-04 22:20:14 +07:00
|
|
|
.timeout = nvme_timeout,
|
2015-11-04 10:37:26 +07:00
|
|
|
.poll = nvme_poll,
|
2014-11-04 22:20:14 +07:00
|
|
|
};
|
|
|
|
|
2015-01-08 08:55:49 +07:00
|
|
|
static void nvme_dev_remove_admin(struct nvme_dev *dev)
|
|
|
|
{
|
2015-11-26 16:06:56 +07:00
|
|
|
if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) {
|
2016-02-24 23:15:56 +07:00
|
|
|
/*
|
|
|
|
* If the controller was reset during removal, it's possible
|
|
|
|
* user requests may be waiting on a stopped queue. Start the
|
|
|
|
* queue to flush these to completion.
|
|
|
|
*/
|
2017-07-02 19:53:27 +07:00
|
|
|
blk_mq_unquiesce_queue(dev->ctrl.admin_q);
|
2015-11-26 16:06:56 +07:00
|
|
|
blk_cleanup_queue(dev->ctrl.admin_q);
|
2015-01-08 08:55:49 +07:00
|
|
|
blk_mq_free_tag_set(&dev->admin_tagset);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-11-04 22:20:14 +07:00
|
|
|
static int nvme_alloc_admin_tags(struct nvme_dev *dev)
|
|
|
|
{
|
2015-11-26 16:06:56 +07:00
|
|
|
if (!dev->ctrl.admin_q) {
|
2014-11-04 22:20:14 +07:00
|
|
|
dev->admin_tagset.ops = &nvme_mq_admin_ops;
|
|
|
|
dev->admin_tagset.nr_hw_queues = 1;
|
2016-01-04 23:10:55 +07:00
|
|
|
|
2017-11-08 05:13:10 +07:00
|
|
|
dev->admin_tagset.queue_depth = NVME_AQ_MQ_TAG_DEPTH;
|
2014-11-04 22:20:14 +07:00
|
|
|
dev->admin_tagset.timeout = ADMIN_TIMEOUT;
|
2015-05-22 16:12:39 +07:00
|
|
|
dev->admin_tagset.numa_node = dev_to_node(dev->dev);
|
2017-10-17 08:24:20 +07:00
|
|
|
dev->admin_tagset.cmd_size = nvme_pci_cmd_size(dev, false);
|
2017-01-14 04:43:58 +07:00
|
|
|
dev->admin_tagset.flags = BLK_MQ_F_NO_SCHED;
|
2014-11-04 22:20:14 +07:00
|
|
|
dev->admin_tagset.driver_data = dev;
|
|
|
|
|
|
|
|
if (blk_mq_alloc_tag_set(&dev->admin_tagset))
|
|
|
|
return -ENOMEM;
|
2017-07-10 13:22:29 +07:00
|
|
|
dev->ctrl.admin_tagset = &dev->admin_tagset;
|
2014-11-04 22:20:14 +07:00
|
|
|
|
2015-11-26 16:06:56 +07:00
|
|
|
dev->ctrl.admin_q = blk_mq_init_queue(&dev->admin_tagset);
|
|
|
|
if (IS_ERR(dev->ctrl.admin_q)) {
|
2014-11-04 22:20:14 +07:00
|
|
|
blk_mq_free_tag_set(&dev->admin_tagset);
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
2015-11-26 16:06:56 +07:00
|
|
|
if (!blk_get_queue(dev->ctrl.admin_q)) {
|
2015-01-08 08:55:49 +07:00
|
|
|
nvme_dev_remove_admin(dev);
|
2015-11-26 16:06:56 +07:00
|
|
|
dev->ctrl.admin_q = NULL;
|
2015-01-08 08:55:49 +07:00
|
|
|
return -ENODEV;
|
|
|
|
}
|
2015-01-08 08:55:50 +07:00
|
|
|
} else
|
2017-07-02 19:53:27 +07:00
|
|
|
blk_mq_unquiesce_queue(dev->ctrl.admin_q);
|
2014-11-04 22:20:14 +07:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-05-24 15:39:55 +07:00
|
|
|
static unsigned long db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
|
|
|
|
{
|
|
|
|
return NVME_REG_DBS + ((nr_io_queues + 1) * 8 * dev->db_stride);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int nvme_remap_bar(struct nvme_dev *dev, unsigned long size)
|
|
|
|
{
|
|
|
|
struct pci_dev *pdev = to_pci_dev(dev->dev);
|
|
|
|
|
|
|
|
if (size <= dev->bar_mapped_size)
|
|
|
|
return 0;
|
|
|
|
if (size > pci_resource_len(pdev, 0))
|
|
|
|
return -ENOMEM;
|
|
|
|
if (dev->bar)
|
|
|
|
iounmap(dev->bar);
|
|
|
|
dev->bar = ioremap(pci_resource_start(pdev, 0), size);
|
|
|
|
if (!dev->bar) {
|
|
|
|
dev->bar_mapped_size = 0;
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
dev->bar_mapped_size = size;
|
|
|
|
dev->dbs = dev->bar + NVME_REG_DBS;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-05-01 04:27:17 +07:00
|
|
|
static int nvme_pci_configure_admin_queue(struct nvme_dev *dev)
|
2011-01-21 00:50:14 +07:00
|
|
|
{
|
2013-05-04 17:43:16 +07:00
|
|
|
int result;
|
2011-01-21 00:50:14 +07:00
|
|
|
u32 aqa;
|
|
|
|
struct nvme_queue *nvmeq;
|
|
|
|
|
2017-05-24 15:39:55 +07:00
|
|
|
result = nvme_remap_bar(dev, db_bar_size(dev, 0));
|
|
|
|
if (result < 0)
|
|
|
|
return result;
|
|
|
|
|
2016-10-19 22:51:05 +07:00
|
|
|
dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1, 0) ?
|
2017-06-28 02:16:38 +07:00
|
|
|
NVME_CAP_NSSRC(dev->ctrl.cap) : 0;
|
2015-08-11 04:20:40 +07:00
|
|
|
|
2015-11-20 14:58:10 +07:00
|
|
|
if (dev->subsystem &&
|
|
|
|
(readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO))
|
|
|
|
writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS);
|
2015-08-11 04:20:40 +07:00
|
|
|
|
2017-06-28 02:16:38 +07:00
|
|
|
result = nvme_disable_ctrl(&dev->ctrl, dev->ctrl.cap);
|
2013-05-04 17:43:16 +07:00
|
|
|
if (result < 0)
|
|
|
|
return result;
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2018-04-12 22:16:09 +07:00
|
|
|
result = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
|
2018-01-14 17:39:01 +07:00
|
|
|
if (result)
|
|
|
|
return result;
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2018-01-14 17:39:01 +07:00
|
|
|
nvmeq = &dev->queues[0];
|
2011-01-21 00:50:14 +07:00
|
|
|
aqa = nvmeq->q_depth - 1;
|
|
|
|
aqa |= aqa << 16;
|
|
|
|
|
2015-11-20 14:58:10 +07:00
|
|
|
writel(aqa, dev->bar + NVME_REG_AQA);
|
|
|
|
lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ);
|
|
|
|
lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ);
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2017-06-28 02:16:38 +07:00
|
|
|
result = nvme_enable_ctrl(&dev->ctrl, dev->ctrl.cap);
|
2013-05-02 02:07:51 +07:00
|
|
|
if (result)
|
2016-11-16 03:56:26 +07:00
|
|
|
return result;
|
2014-11-04 22:20:14 +07:00
|
|
|
|
2014-12-23 02:59:04 +07:00
|
|
|
nvmeq->cq_vector = 0;
|
2017-09-15 00:54:39 +07:00
|
|
|
nvme_init_queue(nvmeq, 0);
|
2016-09-14 21:18:57 +07:00
|
|
|
result = queue_request_irq(nvmeq);
|
2015-07-01 00:22:52 +07:00
|
|
|
if (result) {
|
|
|
|
nvmeq->cq_vector = -1;
|
2016-11-16 03:56:26 +07:00
|
|
|
return result;
|
2015-07-01 00:22:52 +07:00
|
|
|
}
|
2013-05-02 02:07:51 +07:00
|
|
|
|
2011-01-21 00:50:14 +07:00
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2015-11-26 17:46:39 +07:00
|
|
|
static int nvme_create_io_queues(struct nvme_dev *dev)
|
2014-03-24 23:46:25 +07:00
|
|
|
{
|
2015-12-18 07:08:15 +07:00
|
|
|
unsigned i, max;
|
2015-11-26 17:46:39 +07:00
|
|
|
int ret = 0;
|
2014-03-24 23:46:25 +07:00
|
|
|
|
2017-04-24 14:58:29 +07:00
|
|
|
for (i = dev->ctrl.queue_count; i <= dev->max_qid; i++) {
|
2018-04-12 22:16:09 +07:00
|
|
|
if (nvme_alloc_queue(dev, i, dev->q_depth)) {
|
2015-11-26 17:46:39 +07:00
|
|
|
ret = -ENOMEM;
|
2014-03-24 23:46:25 +07:00
|
|
|
break;
|
2015-11-26 17:46:39 +07:00
|
|
|
}
|
|
|
|
}
|
2014-03-24 23:46:25 +07:00
|
|
|
|
2017-04-24 14:58:29 +07:00
|
|
|
max = min(dev->max_qid, dev->ctrl.queue_count - 1);
|
2015-12-18 07:08:15 +07:00
|
|
|
for (i = dev->online_queues; i <= max; i++) {
|
2018-01-14 17:39:01 +07:00
|
|
|
ret = nvme_create_queue(&dev->queues[i], i);
|
2016-11-16 03:56:26 +07:00
|
|
|
if (ret)
|
2014-03-24 23:46:25 +07:00
|
|
|
break;
|
2014-04-11 22:58:45 +07:00
|
|
|
}
|
2015-11-26 17:46:39 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Ignore failing Create SQ/CQ commands, we can continue with less
|
2018-01-14 14:14:27 +07:00
|
|
|
* than the desired amount of queues, and even a controller without
|
|
|
|
* I/O queues can still be used to issue admin commands. This might
|
2015-11-26 17:46:39 +07:00
|
|
|
* be useful to upgrade a buggy firmware for example.
|
|
|
|
*/
|
|
|
|
return ret >= 0 ? 0 : ret;
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
|
|
|
|
2016-10-06 09:01:12 +07:00
|
|
|
static ssize_t nvme_cmb_show(struct device *dev,
|
|
|
|
struct device_attribute *attr,
|
|
|
|
char *buf)
|
|
|
|
{
|
|
|
|
struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
|
|
|
|
|
2016-12-17 01:54:50 +07:00
|
|
|
return scnprintf(buf, PAGE_SIZE, "cmbloc : x%08x\ncmbsz : x%08x\n",
|
2016-10-06 09:01:12 +07:00
|
|
|
ndev->cmbloc, ndev->cmbsz);
|
|
|
|
}
|
|
|
|
static DEVICE_ATTR(cmb, S_IRUGO, nvme_cmb_show, NULL);
|
|
|
|
|
2017-12-20 20:50:00 +07:00
|
|
|
static u64 nvme_cmb_size_unit(struct nvme_dev *dev)
|
2015-07-20 23:14:09 +07:00
|
|
|
{
|
2017-12-20 20:50:00 +07:00
|
|
|
u8 szu = (dev->cmbsz >> NVME_CMBSZ_SZU_SHIFT) & NVME_CMBSZ_SZU_MASK;
|
|
|
|
|
|
|
|
return 1ULL << (12 + 4 * szu);
|
|
|
|
}
|
|
|
|
|
|
|
|
static u32 nvme_cmb_size(struct nvme_dev *dev)
|
|
|
|
{
|
|
|
|
return (dev->cmbsz >> NVME_CMBSZ_SZ_SHIFT) & NVME_CMBSZ_SZ_MASK;
|
|
|
|
}
|
|
|
|
|
2017-12-20 20:25:11 +07:00
|
|
|
static void nvme_map_cmb(struct nvme_dev *dev)
|
2015-07-20 23:14:09 +07:00
|
|
|
{
|
2017-12-20 20:50:00 +07:00
|
|
|
u64 size, offset;
|
2015-07-20 23:14:09 +07:00
|
|
|
resource_size_t bar_size;
|
|
|
|
struct pci_dev *pdev = to_pci_dev(dev->dev);
|
2017-10-01 14:37:35 +07:00
|
|
|
int bar;
|
2015-07-20 23:14:09 +07:00
|
|
|
|
2015-11-20 14:58:10 +07:00
|
|
|
dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ);
|
2017-12-20 20:25:11 +07:00
|
|
|
if (!dev->cmbsz)
|
|
|
|
return;
|
2016-10-06 09:01:12 +07:00
|
|
|
dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC);
|
2015-07-20 23:14:09 +07:00
|
|
|
|
2016-10-06 09:01:12 +07:00
|
|
|
if (!use_cmb_sqes)
|
2017-12-20 20:25:11 +07:00
|
|
|
return;
|
2015-07-20 23:14:09 +07:00
|
|
|
|
2017-12-20 20:50:00 +07:00
|
|
|
size = nvme_cmb_size_unit(dev) * nvme_cmb_size(dev);
|
|
|
|
offset = nvme_cmb_size_unit(dev) * NVME_CMB_OFST(dev->cmbloc);
|
2017-10-01 14:37:35 +07:00
|
|
|
bar = NVME_CMB_BIR(dev->cmbloc);
|
|
|
|
bar_size = pci_resource_len(pdev, bar);
|
2015-07-20 23:14:09 +07:00
|
|
|
|
|
|
|
if (offset > bar_size)
|
2017-12-20 20:25:11 +07:00
|
|
|
return;
|
2015-07-20 23:14:09 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Controllers may support a CMB size larger than their BAR,
|
|
|
|
* for example, due to being behind a bridge. Reduce the CMB to
|
|
|
|
* the reported size of the BAR
|
|
|
|
*/
|
|
|
|
if (size > bar_size - offset)
|
|
|
|
size = bar_size - offset;
|
|
|
|
|
2017-12-20 20:25:11 +07:00
|
|
|
dev->cmb = ioremap_wc(pci_resource_start(pdev, bar) + offset, size);
|
|
|
|
if (!dev->cmb)
|
|
|
|
return;
|
2017-10-01 14:37:35 +07:00
|
|
|
dev->cmb_bus_addr = pci_bus_address(pdev, bar) + offset;
|
2015-07-20 23:14:09 +07:00
|
|
|
dev->cmb_size = size;
|
2017-12-20 20:25:11 +07:00
|
|
|
|
|
|
|
if (sysfs_add_file_to_group(&dev->ctrl.device->kobj,
|
|
|
|
&dev_attr_cmb.attr, NULL))
|
|
|
|
dev_warn(dev->ctrl.device,
|
|
|
|
"failed to add sysfs attribute for CMB\n");
|
2015-07-20 23:14:09 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void nvme_release_cmb(struct nvme_dev *dev)
|
|
|
|
{
|
|
|
|
if (dev->cmb) {
|
|
|
|
iounmap(dev->cmb);
|
|
|
|
dev->cmb = NULL;
|
2017-07-30 05:45:08 +07:00
|
|
|
sysfs_remove_file_from_group(&dev->ctrl.device->kobj,
|
|
|
|
&dev_attr_cmb.attr, NULL);
|
|
|
|
dev->cmbsz = 0;
|
2015-07-20 23:14:09 +07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-05-12 22:02:58 +07:00
|
|
|
static int nvme_set_host_mem(struct nvme_dev *dev, u32 bits)
|
|
|
|
{
|
2017-08-28 15:47:18 +07:00
|
|
|
u64 dma_addr = dev->host_mem_descs_dma;
|
2017-05-12 22:02:58 +07:00
|
|
|
struct nvme_command c;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
memset(&c, 0, sizeof(c));
|
|
|
|
c.features.opcode = nvme_admin_set_features;
|
|
|
|
c.features.fid = cpu_to_le32(NVME_FEAT_HOST_MEM_BUF);
|
|
|
|
c.features.dword11 = cpu_to_le32(bits);
|
|
|
|
c.features.dword12 = cpu_to_le32(dev->host_mem_size >>
|
|
|
|
ilog2(dev->ctrl.page_size));
|
|
|
|
c.features.dword13 = cpu_to_le32(lower_32_bits(dma_addr));
|
|
|
|
c.features.dword14 = cpu_to_le32(upper_32_bits(dma_addr));
|
|
|
|
c.features.dword15 = cpu_to_le32(dev->nr_host_mem_descs);
|
|
|
|
|
|
|
|
ret = nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
|
|
|
|
if (ret) {
|
|
|
|
dev_warn(dev->ctrl.device,
|
|
|
|
"failed to set host mem (err %d, flags %#x).\n",
|
|
|
|
ret, bits);
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void nvme_free_host_mem(struct nvme_dev *dev)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < dev->nr_host_mem_descs; i++) {
|
|
|
|
struct nvme_host_mem_buf_desc *desc = &dev->host_mem_descs[i];
|
|
|
|
size_t size = le32_to_cpu(desc->size) * dev->ctrl.page_size;
|
|
|
|
|
|
|
|
dma_free_coherent(dev->dev, size, dev->host_mem_desc_bufs[i],
|
|
|
|
le64_to_cpu(desc->addr));
|
|
|
|
}
|
|
|
|
|
|
|
|
kfree(dev->host_mem_desc_bufs);
|
|
|
|
dev->host_mem_desc_bufs = NULL;
|
2017-08-28 15:47:18 +07:00
|
|
|
dma_free_coherent(dev->dev,
|
|
|
|
dev->nr_host_mem_descs * sizeof(*dev->host_mem_descs),
|
|
|
|
dev->host_mem_descs, dev->host_mem_descs_dma);
|
2017-05-12 22:02:58 +07:00
|
|
|
dev->host_mem_descs = NULL;
|
2017-11-25 01:03:00 +07:00
|
|
|
dev->nr_host_mem_descs = 0;
|
2017-05-12 22:02:58 +07:00
|
|
|
}
|
|
|
|
|
2017-09-11 23:08:43 +07:00
|
|
|
static int __nvme_alloc_host_mem(struct nvme_dev *dev, u64 preferred,
|
|
|
|
u32 chunk_size)
|
2013-07-16 04:02:24 +07:00
|
|
|
{
|
2017-05-12 22:02:58 +07:00
|
|
|
struct nvme_host_mem_buf_desc *descs;
|
2017-09-11 23:08:43 +07:00
|
|
|
u32 max_entries, len;
|
2017-08-28 15:47:18 +07:00
|
|
|
dma_addr_t descs_dma;
|
2017-07-06 16:26:52 +07:00
|
|
|
int i = 0;
|
2017-05-12 22:02:58 +07:00
|
|
|
void **bufs;
|
2017-12-05 03:23:54 +07:00
|
|
|
u64 size, tmp;
|
2017-05-12 22:02:58 +07:00
|
|
|
|
|
|
|
tmp = (preferred + chunk_size - 1);
|
|
|
|
do_div(tmp, chunk_size);
|
|
|
|
max_entries = tmp;
|
2017-09-11 23:09:28 +07:00
|
|
|
|
|
|
|
if (dev->ctrl.hmmaxd && dev->ctrl.hmmaxd < max_entries)
|
|
|
|
max_entries = dev->ctrl.hmmaxd;
|
|
|
|
|
2017-08-28 15:47:18 +07:00
|
|
|
descs = dma_zalloc_coherent(dev->dev, max_entries * sizeof(*descs),
|
|
|
|
&descs_dma, GFP_KERNEL);
|
2017-05-12 22:02:58 +07:00
|
|
|
if (!descs)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
bufs = kcalloc(max_entries, sizeof(*bufs), GFP_KERNEL);
|
|
|
|
if (!bufs)
|
|
|
|
goto out_free_descs;
|
|
|
|
|
2017-11-16 23:34:24 +07:00
|
|
|
for (size = 0; size < preferred && i < max_entries; size += len) {
|
2017-05-12 22:02:58 +07:00
|
|
|
dma_addr_t dma_addr;
|
|
|
|
|
2017-07-25 22:39:07 +07:00
|
|
|
len = min_t(u64, chunk_size, preferred - size);
|
2017-05-12 22:02:58 +07:00
|
|
|
bufs[i] = dma_alloc_attrs(dev->dev, len, &dma_addr, GFP_KERNEL,
|
|
|
|
DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN);
|
|
|
|
if (!bufs[i])
|
|
|
|
break;
|
|
|
|
|
|
|
|
descs[i].addr = cpu_to_le64(dma_addr);
|
|
|
|
descs[i].size = cpu_to_le32(len / dev->ctrl.page_size);
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
|
2017-09-11 23:08:43 +07:00
|
|
|
if (!size)
|
2017-05-12 22:02:58 +07:00
|
|
|
goto out_free_bufs;
|
|
|
|
|
|
|
|
dev->nr_host_mem_descs = i;
|
|
|
|
dev->host_mem_size = size;
|
|
|
|
dev->host_mem_descs = descs;
|
2017-08-28 15:47:18 +07:00
|
|
|
dev->host_mem_descs_dma = descs_dma;
|
2017-05-12 22:02:58 +07:00
|
|
|
dev->host_mem_desc_bufs = bufs;
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
out_free_bufs:
|
|
|
|
while (--i >= 0) {
|
|
|
|
size_t size = le32_to_cpu(descs[i].size) * dev->ctrl.page_size;
|
|
|
|
|
|
|
|
dma_free_coherent(dev->dev, size, bufs[i],
|
|
|
|
le64_to_cpu(descs[i].addr));
|
|
|
|
}
|
|
|
|
|
|
|
|
kfree(bufs);
|
|
|
|
out_free_descs:
|
2017-08-28 15:47:18 +07:00
|
|
|
dma_free_coherent(dev->dev, max_entries * sizeof(*descs), descs,
|
|
|
|
descs_dma);
|
2017-05-12 22:02:58 +07:00
|
|
|
out:
|
|
|
|
dev->host_mem_descs = NULL;
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
2017-09-11 23:08:43 +07:00
|
|
|
static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred)
|
|
|
|
{
|
|
|
|
u32 chunk_size;
|
|
|
|
|
|
|
|
/* start big and work our way down */
|
2017-09-06 17:15:31 +07:00
|
|
|
for (chunk_size = min_t(u64, preferred, PAGE_SIZE * MAX_ORDER_NR_PAGES);
|
2017-09-11 23:09:28 +07:00
|
|
|
chunk_size >= max_t(u32, dev->ctrl.hmminds * 4096, PAGE_SIZE * 2);
|
2017-09-11 23:08:43 +07:00
|
|
|
chunk_size /= 2) {
|
|
|
|
if (!__nvme_alloc_host_mem(dev, preferred, chunk_size)) {
|
|
|
|
if (!min || dev->host_mem_size >= min)
|
|
|
|
return 0;
|
|
|
|
nvme_free_host_mem(dev);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
2017-09-06 17:19:57 +07:00
|
|
|
static int nvme_setup_host_mem(struct nvme_dev *dev)
|
2017-05-12 22:02:58 +07:00
|
|
|
{
|
|
|
|
u64 max = (u64)max_host_mem_size_mb * SZ_1M;
|
|
|
|
u64 preferred = (u64)dev->ctrl.hmpre * 4096;
|
|
|
|
u64 min = (u64)dev->ctrl.hmmin * 4096;
|
|
|
|
u32 enable_bits = NVME_HOST_MEM_ENABLE;
|
2017-12-05 03:23:54 +07:00
|
|
|
int ret;
|
2017-05-12 22:02:58 +07:00
|
|
|
|
|
|
|
preferred = min(preferred, max);
|
|
|
|
if (min > max) {
|
|
|
|
dev_warn(dev->ctrl.device,
|
|
|
|
"min host memory (%lld MiB) above limit (%d MiB).\n",
|
|
|
|
min >> ilog2(SZ_1M), max_host_mem_size_mb);
|
|
|
|
nvme_free_host_mem(dev);
|
2017-09-06 17:19:57 +07:00
|
|
|
return 0;
|
2017-05-12 22:02:58 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If we already have a buffer allocated check if we can reuse it.
|
|
|
|
*/
|
|
|
|
if (dev->host_mem_descs) {
|
|
|
|
if (dev->host_mem_size >= min)
|
|
|
|
enable_bits |= NVME_HOST_MEM_RETURN;
|
|
|
|
else
|
|
|
|
nvme_free_host_mem(dev);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!dev->host_mem_descs) {
|
2017-09-11 23:08:43 +07:00
|
|
|
if (nvme_alloc_host_mem(dev, min, preferred)) {
|
|
|
|
dev_warn(dev->ctrl.device,
|
|
|
|
"failed to allocate host memory buffer.\n");
|
2017-09-06 17:19:57 +07:00
|
|
|
return 0; /* controller must work without HMB */
|
2017-09-11 23:08:43 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
dev_info(dev->ctrl.device,
|
|
|
|
"allocated %lld MiB host memory buffer.\n",
|
|
|
|
dev->host_mem_size >> ilog2(SZ_1M));
|
2017-05-12 22:02:58 +07:00
|
|
|
}
|
|
|
|
|
2017-09-06 17:19:57 +07:00
|
|
|
ret = nvme_set_host_mem(dev, enable_bits);
|
|
|
|
if (ret)
|
2017-05-12 22:02:58 +07:00
|
|
|
nvme_free_host_mem(dev);
|
2017-09-06 17:19:57 +07:00
|
|
|
return ret;
|
2013-07-16 04:02:24 +07:00
|
|
|
}
|
|
|
|
|
2012-12-22 06:13:49 +07:00
|
|
|
static int nvme_setup_io_queues(struct nvme_dev *dev)
|
2011-01-21 00:50:14 +07:00
|
|
|
{
|
2018-01-14 17:39:01 +07:00
|
|
|
struct nvme_queue *adminq = &dev->queues[0];
|
2015-05-22 16:12:39 +07:00
|
|
|
struct pci_dev *pdev = to_pci_dev(dev->dev);
|
2017-05-24 15:39:55 +07:00
|
|
|
int result, nr_io_queues;
|
|
|
|
unsigned long size;
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2018-04-12 22:16:10 +07:00
|
|
|
struct irq_affinity affd = {
|
|
|
|
.pre_vectors = 1
|
|
|
|
};
|
|
|
|
|
nvme: pci: pass max vectors as num_possible_cpus() to pci_alloc_irq_vectors
84676c1f21 ("genirq/affinity: assign vectors to all possible CPUs")
has switched to do irq vectors spread among all possible CPUs, so
pass num_possible_cpus() as max vecotrs to be assigned.
For example, in a 8 cores system, 0~3 online, 4~8 offline/not present,
see 'lscpu':
[ming@box]$lscpu
Architecture: x86_64
CPU op-mode(s): 32-bit, 64-bit
Byte Order: Little Endian
CPU(s): 4
On-line CPU(s) list: 0-3
Thread(s) per core: 1
Core(s) per socket: 2
Socket(s): 2
NUMA node(s): 2
...
NUMA node0 CPU(s): 0-3
NUMA node1 CPU(s):
...
1) before this patch, follows the allocated vectors and their affinity:
irq 47, cpu list 0,4
irq 48, cpu list 1,6
irq 49, cpu list 2,5
irq 50, cpu list 3,7
2) after this patch, follows the allocated vectors and their affinity:
irq 43, cpu list 0
irq 44, cpu list 1
irq 45, cpu list 2
irq 46, cpu list 3
irq 47, cpu list 4
irq 48, cpu list 6
irq 49, cpu list 5
irq 50, cpu list 7
Cc: Keith Busch <keith.busch@intel.com>
Cc: Sagi Grimberg <sagi@grimberg.me>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Christoph Hellwig <hch@lst.de>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Keith Busch <keith.busch@intel.com>
2018-02-06 19:17:42 +07:00
|
|
|
nr_io_queues = num_possible_cpus();
|
2015-11-26 17:09:06 +07:00
|
|
|
result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues);
|
|
|
|
if (result < 0)
|
2011-01-21 01:01:49 +07:00
|
|
|
return result;
|
2015-11-26 17:09:06 +07:00
|
|
|
|
2016-06-07 04:20:50 +07:00
|
|
|
if (nr_io_queues == 0)
|
2016-04-09 05:09:10 +07:00
|
|
|
return 0;
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2017-12-20 20:50:00 +07:00
|
|
|
if (dev->cmb && (dev->cmbsz & NVME_CMBSZ_SQS)) {
|
2015-07-20 23:14:09 +07:00
|
|
|
result = nvme_cmb_qdepth(dev, nr_io_queues,
|
|
|
|
sizeof(struct nvme_command));
|
|
|
|
if (result > 0)
|
|
|
|
dev->q_depth = result;
|
|
|
|
else
|
|
|
|
nvme_release_cmb(dev);
|
|
|
|
}
|
|
|
|
|
2017-05-24 15:39:55 +07:00
|
|
|
do {
|
|
|
|
size = db_bar_size(dev, nr_io_queues);
|
|
|
|
result = nvme_remap_bar(dev, size);
|
|
|
|
if (!result)
|
|
|
|
break;
|
|
|
|
if (!--nr_io_queues)
|
|
|
|
return -ENOMEM;
|
|
|
|
} while (1);
|
|
|
|
adminq->q_db = dev->dbs;
|
2011-10-21 04:00:41 +07:00
|
|
|
|
2013-07-16 04:02:24 +07:00
|
|
|
/* Deregister the admin queue's interrupt */
|
2017-04-13 14:06:43 +07:00
|
|
|
pci_free_irq(pdev, 0, adminq);
|
2013-07-16 04:02:24 +07:00
|
|
|
|
2014-11-14 23:49:26 +07:00
|
|
|
/*
|
|
|
|
* If we enable msix early due to not intx, disable it again before
|
|
|
|
* setting up the full range we need.
|
|
|
|
*/
|
2016-09-14 21:18:57 +07:00
|
|
|
pci_free_irq_vectors(pdev);
|
2018-04-12 22:16:10 +07:00
|
|
|
result = pci_alloc_irq_vectors_affinity(pdev, 1, nr_io_queues + 1,
|
|
|
|
PCI_IRQ_ALL_TYPES | PCI_IRQ_AFFINITY, &affd);
|
|
|
|
if (result <= 0)
|
2016-09-14 21:18:57 +07:00
|
|
|
return -EIO;
|
2018-04-12 22:16:10 +07:00
|
|
|
dev->num_vecs = result;
|
|
|
|
dev->max_qid = max(result - 1, 1);
|
2013-05-12 05:19:31 +07:00
|
|
|
|
2013-06-20 21:53:48 +07:00
|
|
|
/*
|
|
|
|
* Should investigate if there's a performance win from allocating
|
|
|
|
* more queues than interrupt vectors; it might allow the submission
|
|
|
|
* path to scale better, even if the receive path is limited by the
|
|
|
|
* number of interrupts.
|
|
|
|
*/
|
|
|
|
|
2016-09-14 21:18:57 +07:00
|
|
|
result = queue_request_irq(adminq);
|
2015-07-01 00:22:52 +07:00
|
|
|
if (result) {
|
|
|
|
adminq->cq_vector = -1;
|
2016-11-16 03:56:26 +07:00
|
|
|
return result;
|
2015-07-01 00:22:52 +07:00
|
|
|
}
|
2015-11-26 17:46:39 +07:00
|
|
|
return nvme_create_io_queues(dev);
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
|
|
|
|
2017-06-03 14:38:04 +07:00
|
|
|
static void nvme_del_queue_end(struct request *req, blk_status_t error)
|
2015-06-02 03:28:14 +07:00
|
|
|
{
|
2016-01-13 04:41:17 +07:00
|
|
|
struct nvme_queue *nvmeq = req->end_io_data;
|
2015-12-12 03:14:28 +07:00
|
|
|
|
2016-01-13 04:41:17 +07:00
|
|
|
blk_mq_free_request(req);
|
|
|
|
complete(&nvmeq->dev->ioq_wait);
|
2015-06-02 03:28:14 +07:00
|
|
|
}
|
|
|
|
|
2017-06-03 14:38:04 +07:00
|
|
|
static void nvme_del_cq_end(struct request *req, blk_status_t error)
|
2015-06-02 03:28:14 +07:00
|
|
|
{
|
2016-01-13 04:41:17 +07:00
|
|
|
struct nvme_queue *nvmeq = req->end_io_data;
|
2018-05-17 23:31:50 +07:00
|
|
|
u16 start, end;
|
2015-06-02 03:28:14 +07:00
|
|
|
|
2016-01-13 04:41:17 +07:00
|
|
|
if (!error) {
|
|
|
|
unsigned long flags;
|
|
|
|
|
2018-06-06 21:13:04 +07:00
|
|
|
spin_lock_irqsave(&nvmeq->cq_lock, flags);
|
2018-05-17 23:31:50 +07:00
|
|
|
nvme_process_cq(nvmeq, &start, &end, -1);
|
2018-05-17 23:31:51 +07:00
|
|
|
spin_unlock_irqrestore(&nvmeq->cq_lock, flags);
|
2018-05-17 23:31:50 +07:00
|
|
|
|
|
|
|
nvme_complete_cqes(nvmeq, start, end);
|
2015-06-02 03:28:14 +07:00
|
|
|
}
|
2016-01-13 04:41:17 +07:00
|
|
|
|
|
|
|
nvme_del_queue_end(req, error);
|
2015-06-02 03:28:14 +07:00
|
|
|
}
|
|
|
|
|
2016-01-13 04:41:17 +07:00
|
|
|
static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode)
|
2015-09-03 21:18:17 +07:00
|
|
|
{
|
2016-01-13 04:41:17 +07:00
|
|
|
struct request_queue *q = nvmeq->dev->ctrl.admin_q;
|
|
|
|
struct request *req;
|
|
|
|
struct nvme_command cmd;
|
2015-09-03 21:18:17 +07:00
|
|
|
|
2016-01-13 04:41:17 +07:00
|
|
|
memset(&cmd, 0, sizeof(cmd));
|
|
|
|
cmd.delete_queue.opcode = opcode;
|
|
|
|
cmd.delete_queue.qid = cpu_to_le16(nvmeq->qid);
|
2015-09-03 21:18:17 +07:00
|
|
|
|
2016-06-13 21:45:23 +07:00
|
|
|
req = nvme_alloc_request(q, &cmd, BLK_MQ_REQ_NOWAIT, NVME_QID_ANY);
|
2016-01-13 04:41:17 +07:00
|
|
|
if (IS_ERR(req))
|
|
|
|
return PTR_ERR(req);
|
2015-09-03 21:18:17 +07:00
|
|
|
|
2016-01-13 04:41:17 +07:00
|
|
|
req->timeout = ADMIN_TIMEOUT;
|
|
|
|
req->end_io_data = nvmeq;
|
|
|
|
|
|
|
|
blk_execute_rq_nowait(q, NULL, req, false,
|
|
|
|
opcode == nvme_admin_delete_cq ?
|
|
|
|
nvme_del_cq_end : nvme_del_queue_end);
|
|
|
|
return 0;
|
2015-09-03 21:18:17 +07:00
|
|
|
}
|
|
|
|
|
2018-01-25 04:55:12 +07:00
|
|
|
static void nvme_disable_io_queues(struct nvme_dev *dev)
|
2015-06-02 03:28:14 +07:00
|
|
|
{
|
2018-01-25 04:55:12 +07:00
|
|
|
int pass, queues = dev->online_queues - 1;
|
2016-01-13 04:41:17 +07:00
|
|
|
unsigned long timeout;
|
|
|
|
u8 opcode = nvme_admin_delete_sq;
|
2015-06-02 03:28:14 +07:00
|
|
|
|
2016-01-13 04:41:17 +07:00
|
|
|
for (pass = 0; pass < 2; pass++) {
|
2016-05-07 00:50:52 +07:00
|
|
|
int sent = 0, i = queues;
|
2016-01-13 04:41:17 +07:00
|
|
|
|
|
|
|
reinit_completion(&dev->ioq_wait);
|
|
|
|
retry:
|
|
|
|
timeout = ADMIN_TIMEOUT;
|
2016-08-11 22:35:57 +07:00
|
|
|
for (; i > 0; i--, sent++)
|
2018-01-14 17:39:01 +07:00
|
|
|
if (nvme_delete_queue(&dev->queues[i], opcode))
|
2016-01-13 04:41:17 +07:00
|
|
|
break;
|
2016-08-11 22:35:57 +07:00
|
|
|
|
2016-01-13 04:41:17 +07:00
|
|
|
while (sent--) {
|
|
|
|
timeout = wait_for_completion_io_timeout(&dev->ioq_wait, timeout);
|
|
|
|
if (timeout == 0)
|
|
|
|
return;
|
|
|
|
if (i)
|
|
|
|
goto retry;
|
|
|
|
}
|
|
|
|
opcode = nvme_admin_delete_cq;
|
|
|
|
}
|
2015-06-02 03:28:14 +07:00
|
|
|
}
|
|
|
|
|
2013-04-16 22:22:36 +07:00
|
|
|
/*
|
2018-01-06 07:01:58 +07:00
|
|
|
* return error value only when tagset allocation failed
|
2013-04-16 22:22:36 +07:00
|
|
|
*/
|
2012-12-22 06:13:49 +07:00
|
|
|
static int nvme_dev_add(struct nvme_dev *dev)
|
2011-01-21 00:50:14 +07:00
|
|
|
{
|
2018-01-06 07:01:58 +07:00
|
|
|
int ret;
|
|
|
|
|
2015-11-28 21:39:07 +07:00
|
|
|
if (!dev->ctrl.tagset) {
|
2015-06-08 23:08:15 +07:00
|
|
|
dev->tagset.ops = &nvme_mq_ops;
|
|
|
|
dev->tagset.nr_hw_queues = dev->online_queues - 1;
|
|
|
|
dev->tagset.timeout = NVME_IO_TIMEOUT;
|
|
|
|
dev->tagset.numa_node = dev_to_node(dev->dev);
|
|
|
|
dev->tagset.queue_depth =
|
2014-11-04 22:20:14 +07:00
|
|
|
min_t(int, dev->q_depth, BLK_MQ_MAX_DEPTH) - 1;
|
2017-10-17 08:24:20 +07:00
|
|
|
dev->tagset.cmd_size = nvme_pci_cmd_size(dev, false);
|
|
|
|
if ((dev->ctrl.sgls & ((1 << 0) | (1 << 1))) && sgl_threshold) {
|
|
|
|
dev->tagset.cmd_size = max(dev->tagset.cmd_size,
|
|
|
|
nvme_pci_cmd_size(dev, true));
|
|
|
|
}
|
2015-06-08 23:08:15 +07:00
|
|
|
dev->tagset.flags = BLK_MQ_F_SHOULD_MERGE;
|
|
|
|
dev->tagset.driver_data = dev;
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2018-01-06 07:01:58 +07:00
|
|
|
ret = blk_mq_alloc_tag_set(&dev->tagset);
|
|
|
|
if (ret) {
|
|
|
|
dev_warn(dev->ctrl.device,
|
|
|
|
"IO queues tagset allocation failed %d\n", ret);
|
|
|
|
return ret;
|
|
|
|
}
|
2015-11-28 21:39:07 +07:00
|
|
|
dev->ctrl.tagset = &dev->tagset;
|
2017-04-10 22:51:07 +07:00
|
|
|
|
|
|
|
nvme_dbbuf_set(dev);
|
2015-12-18 07:08:15 +07:00
|
|
|
} else {
|
|
|
|
blk_mq_update_nr_hw_queues(&dev->tagset, dev->online_queues - 1);
|
|
|
|
|
|
|
|
/* Free previously allocated queues that are no longer usable */
|
|
|
|
nvme_free_queues(dev, dev->online_queues);
|
2015-06-08 23:08:15 +07:00
|
|
|
}
|
2015-12-18 07:08:15 +07:00
|
|
|
|
NVMe: Metadata format support
Adds support for NVMe metadata formats and exposes block devices for
all namespaces regardless of their format. Namespace formats that are
unusable will have disk capacity set to 0, but a handle to the block
device is created to simplify device management. A namespace is not
usable when the format requires host interleave block and metadata in
single buffer, has no provisioned storage, or has better data but failed
to register with blk integrity.
The namespace has to be scanned in two phases to support separate
metadata formats. The first establishes the sector size and capacity
prior to invoking add_disk. If metadata is required, the capacity will
be temporarilly set to 0 until it can be revalidated and registered with
the integrity extenstions after add_disk completes.
The driver relies on the integrity extensions to provide the metadata
buffer. NVMe requires this be a single physically contiguous region,
so only one integrity segment is allowed per command. If the metadata
is used for T10 PI, the driver provides mappings to save and restore
the reftag physical block translation. The driver provides no-op
functions for generate and verify if metadata is not used for protection
information. This way the setup is always provided by the block layer.
If a request does not supply a required metadata buffer, the command
is failed with bad address. This could only happen if a user manually
disables verify/generate on such a disk. The only exception to where
this is okay is if the controller is capable of stripping/generating
the metadata, which is possible on some types of formats.
The metadata scatter gather list now occupies the spot in the nvme_iod
that used to be used to link retryable IOD's, but we don't do that
anymore, so the field was unused.
Signed-off-by: Keith Busch <keith.busch@intel.com>
2015-02-20 03:39:03 +07:00
|
|
|
return 0;
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
|
|
|
|
2016-02-24 23:15:52 +07:00
|
|
|
static int nvme_pci_enable(struct nvme_dev *dev)
|
2013-07-16 04:02:19 +07:00
|
|
|
{
|
2016-02-24 23:15:52 +07:00
|
|
|
int result = -ENOMEM;
|
2015-05-22 16:12:39 +07:00
|
|
|
struct pci_dev *pdev = to_pci_dev(dev->dev);
|
2013-07-16 04:02:19 +07:00
|
|
|
|
|
|
|
if (pci_enable_device_mem(pdev))
|
|
|
|
return result;
|
|
|
|
|
|
|
|
pci_set_master(pdev);
|
|
|
|
|
2015-05-22 16:12:39 +07:00
|
|
|
if (dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(64)) &&
|
|
|
|
dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(32)))
|
2013-06-27 05:49:11 +07:00
|
|
|
goto disable;
|
2013-07-16 04:02:19 +07:00
|
|
|
|
2015-11-20 14:58:10 +07:00
|
|
|
if (readl(dev->bar + NVME_REG_CSTS) == -1) {
|
2013-12-11 03:10:39 +07:00
|
|
|
result = -ENODEV;
|
2016-02-24 23:15:52 +07:00
|
|
|
goto disable;
|
2013-12-11 03:10:39 +07:00
|
|
|
}
|
2014-11-14 23:49:26 +07:00
|
|
|
|
|
|
|
/*
|
2016-04-09 05:09:10 +07:00
|
|
|
* Some devices and/or platforms don't advertise or work with INTx
|
|
|
|
* interrupts. Pre-enable a single MSIX or MSI vec for setup. We'll
|
|
|
|
* adjust this later.
|
2014-11-14 23:49:26 +07:00
|
|
|
*/
|
2016-09-14 21:18:57 +07:00
|
|
|
result = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES);
|
|
|
|
if (result < 0)
|
|
|
|
return result;
|
2014-11-14 23:49:26 +07:00
|
|
|
|
2017-06-28 02:16:38 +07:00
|
|
|
dev->ctrl.cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
|
2015-11-20 14:58:10 +07:00
|
|
|
|
2017-06-28 02:16:38 +07:00
|
|
|
dev->q_depth = min_t(int, NVME_CAP_MQES(dev->ctrl.cap) + 1,
|
2017-07-10 15:46:59 +07:00
|
|
|
io_queue_depth);
|
2017-06-28 02:16:38 +07:00
|
|
|
dev->db_stride = 1 << NVME_CAP_STRIDE(dev->ctrl.cap);
|
2015-11-20 14:58:10 +07:00
|
|
|
dev->dbs = dev->bar + 4096;
|
2015-12-02 03:23:22 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Temporary fix for the Apple controller found in the MacBook8,1 and
|
|
|
|
* some MacBook7,1 to avoid controller resets and data loss.
|
|
|
|
*/
|
|
|
|
if (pdev->vendor == PCI_VENDOR_ID_APPLE && pdev->device == 0x2001) {
|
|
|
|
dev->q_depth = 2;
|
2017-05-20 20:14:43 +07:00
|
|
|
dev_warn(dev->ctrl.device, "detected Apple NVMe controller, "
|
|
|
|
"set queue depth=%u to work around controller resets\n",
|
2015-12-02 03:23:22 +07:00
|
|
|
dev->q_depth);
|
2017-06-28 09:27:57 +07:00
|
|
|
} else if (pdev->vendor == PCI_VENDOR_ID_SAMSUNG &&
|
|
|
|
(pdev->device == 0xa821 || pdev->device == 0xa822) &&
|
2017-06-28 02:16:38 +07:00
|
|
|
NVME_CAP_MQES(dev->ctrl.cap) == 0) {
|
2017-06-28 09:27:57 +07:00
|
|
|
dev->q_depth = 64;
|
|
|
|
dev_err(dev->ctrl.device, "detected PM1725 NVMe controller, "
|
|
|
|
"set queue depth=%u\n", dev->q_depth);
|
2015-12-02 03:23:22 +07:00
|
|
|
}
|
|
|
|
|
2017-12-20 20:25:11 +07:00
|
|
|
nvme_map_cmb(dev);
|
2016-10-06 09:01:12 +07:00
|
|
|
|
2015-12-08 05:30:31 +07:00
|
|
|
pci_enable_pcie_error_reporting(pdev);
|
|
|
|
pci_save_state(pdev);
|
2013-07-16 04:02:19 +07:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
disable:
|
|
|
|
pci_disable_device(pdev);
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void nvme_dev_unmap(struct nvme_dev *dev)
|
2016-02-24 23:15:52 +07:00
|
|
|
{
|
|
|
|
if (dev->bar)
|
|
|
|
iounmap(dev->bar);
|
2016-06-07 14:44:02 +07:00
|
|
|
pci_release_mem_regions(to_pci_dev(dev->dev));
|
2016-02-24 23:15:52 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static void nvme_pci_disable(struct nvme_dev *dev)
|
2013-07-16 04:02:19 +07:00
|
|
|
{
|
2015-05-22 16:12:39 +07:00
|
|
|
struct pci_dev *pdev = to_pci_dev(dev->dev);
|
|
|
|
|
2017-05-06 03:52:06 +07:00
|
|
|
nvme_release_cmb(dev);
|
2016-09-14 21:18:57 +07:00
|
|
|
pci_free_irq_vectors(pdev);
|
2013-07-16 04:02:19 +07:00
|
|
|
|
2015-12-08 05:30:31 +07:00
|
|
|
if (pci_is_enabled(pdev)) {
|
|
|
|
pci_disable_pcie_error_reporting(pdev);
|
2015-05-22 16:12:39 +07:00
|
|
|
pci_disable_device(pdev);
|
2013-12-11 03:10:40 +07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-01-13 04:41:18 +07:00
|
|
|
static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
|
2011-01-21 00:50:14 +07:00
|
|
|
{
|
2018-01-25 04:55:12 +07:00
|
|
|
int i;
|
2017-03-02 02:22:12 +07:00
|
|
|
bool dead = true;
|
|
|
|
struct pci_dev *pdev = to_pci_dev(dev->dev);
|
2013-07-16 04:02:20 +07:00
|
|
|
|
2015-11-26 18:21:29 +07:00
|
|
|
mutex_lock(&dev->shutdown_lock);
|
2017-03-02 02:22:12 +07:00
|
|
|
if (pci_is_enabled(pdev)) {
|
|
|
|
u32 csts = readl(dev->bar + NVME_REG_CSTS);
|
|
|
|
|
2017-06-28 06:44:05 +07:00
|
|
|
if (dev->ctrl.state == NVME_CTRL_LIVE ||
|
|
|
|
dev->ctrl.state == NVME_CTRL_RESETTING)
|
2017-03-02 02:22:12 +07:00
|
|
|
nvme_start_freeze(&dev->ctrl);
|
|
|
|
dead = !!((csts & NVME_CSTS_CFS) || !(csts & NVME_CSTS_RDY) ||
|
|
|
|
pdev->error_state != pci_channel_io_normal);
|
2015-01-08 08:55:52 +07:00
|
|
|
}
|
2016-08-11 22:35:57 +07:00
|
|
|
|
2017-03-02 02:22:12 +07:00
|
|
|
/*
|
|
|
|
* Give the controller a chance to complete all entered requests if
|
|
|
|
* doing a safe shutdown.
|
|
|
|
*/
|
2017-05-12 22:02:58 +07:00
|
|
|
if (!dead) {
|
|
|
|
if (shutdown)
|
|
|
|
nvme_wait_freeze_timeout(&dev->ctrl, NVME_IO_TIMEOUT);
|
2018-02-12 19:57:24 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
nvme_stop_queues(&dev->ctrl);
|
2017-05-12 22:02:58 +07:00
|
|
|
|
2018-04-12 22:16:08 +07:00
|
|
|
if (!dead && dev->ctrl.queue_count > 0) {
|
2018-01-25 04:55:12 +07:00
|
|
|
nvme_disable_io_queues(dev);
|
2016-01-13 04:41:18 +07:00
|
|
|
nvme_disable_admin_queue(dev, shutdown);
|
2013-12-11 03:10:40 +07:00
|
|
|
}
|
2018-01-25 04:55:12 +07:00
|
|
|
for (i = dev->ctrl.queue_count - 1; i >= 0; i--)
|
|
|
|
nvme_suspend_queue(&dev->queues[i]);
|
|
|
|
|
2016-02-24 23:15:52 +07:00
|
|
|
nvme_pci_disable(dev);
|
2015-02-20 00:34:48 +07:00
|
|
|
|
2016-05-19 04:05:01 +07:00
|
|
|
blk_mq_tagset_busy_iter(&dev->tagset, nvme_cancel_request, &dev->ctrl);
|
|
|
|
blk_mq_tagset_busy_iter(&dev->admin_tagset, nvme_cancel_request, &dev->ctrl);
|
2017-03-02 02:22:12 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The driver will not be starting up queues again if shutting down so
|
|
|
|
* must flush all entered requests to their failed completion to avoid
|
|
|
|
* deadlocking blk-mq hot-cpu notifier.
|
|
|
|
*/
|
|
|
|
if (shutdown)
|
|
|
|
nvme_start_queues(&dev->ctrl);
|
2015-11-26 18:21:29 +07:00
|
|
|
mutex_unlock(&dev->shutdown_lock);
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
|
|
|
|
2011-02-10 21:56:01 +07:00
|
|
|
static int nvme_setup_prp_pools(struct nvme_dev *dev)
|
|
|
|
{
|
2015-05-22 16:12:39 +07:00
|
|
|
dev->prp_page_pool = dma_pool_create("prp list page", dev->dev,
|
2011-02-10 21:56:01 +07:00
|
|
|
PAGE_SIZE, PAGE_SIZE, 0);
|
|
|
|
if (!dev->prp_page_pool)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
2011-02-10 22:30:34 +07:00
|
|
|
/* Optimisation for I/Os between 4k and 128k */
|
2015-05-22 16:12:39 +07:00
|
|
|
dev->prp_small_pool = dma_pool_create("prp list 256", dev->dev,
|
2011-02-10 22:30:34 +07:00
|
|
|
256, 256, 0);
|
|
|
|
if (!dev->prp_small_pool) {
|
|
|
|
dma_pool_destroy(dev->prp_page_pool);
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
2011-02-10 21:56:01 +07:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void nvme_release_prp_pools(struct nvme_dev *dev)
|
|
|
|
{
|
|
|
|
dma_pool_destroy(dev->prp_page_pool);
|
2011-02-10 22:30:34 +07:00
|
|
|
dma_pool_destroy(dev->prp_small_pool);
|
2011-02-10 21:56:01 +07:00
|
|
|
}
|
|
|
|
|
2015-11-26 16:54:19 +07:00
|
|
|
static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
|
2013-02-20 00:17:58 +07:00
|
|
|
{
|
2015-11-26 16:54:19 +07:00
|
|
|
struct nvme_dev *dev = to_nvme_dev(ctrl);
|
2014-02-01 06:53:39 +07:00
|
|
|
|
2017-04-10 22:51:07 +07:00
|
|
|
nvme_dbbuf_dma_free(dev);
|
2015-05-22 16:12:39 +07:00
|
|
|
put_device(dev->dev);
|
2015-06-08 23:08:13 +07:00
|
|
|
if (dev->tagset.tags)
|
|
|
|
blk_mq_free_tag_set(&dev->tagset);
|
2015-11-26 16:06:56 +07:00
|
|
|
if (dev->ctrl.admin_q)
|
|
|
|
blk_put_queue(dev->ctrl.admin_q);
|
2013-02-20 00:17:58 +07:00
|
|
|
kfree(dev->queues);
|
2017-02-23 00:15:07 +07:00
|
|
|
free_opal_dev(dev->ctrl.opal_dev);
|
2018-06-21 22:49:37 +07:00
|
|
|
mempool_destroy(dev->iod_mempool);
|
2013-02-20 00:17:58 +07:00
|
|
|
kfree(dev);
|
|
|
|
}
|
|
|
|
|
2016-02-24 23:15:55 +07:00
|
|
|
static void nvme_remove_dead_ctrl(struct nvme_dev *dev, int status)
|
|
|
|
{
|
Merge branch 'for-4.6/drivers' of git://git.kernel.dk/linux-block
Pull block driver updates from Jens Axboe:
"This is the block driver pull request for this merge window. It sits
on top of for-4.6/core, that was just sent out.
This contains:
- A set of fixes for lightnvm. One from Alan, fixing an overflow,
and the rest from the usual suspects, Javier and Matias.
- A set of fixes for nbd from Markus and Dan, and a fixup from Arnd
for correct usage of the signed 64-bit divider.
- A set of bug fixes for the Micron mtip32xx, from Asai.
- A fix for the brd discard handling from Bart.
- Update the maintainers entry for cciss, since that hardware has
transferred ownership.
- Three bug fixes for bcache from Eric Wheeler.
- Set of fixes for xen-blk{back,front} from Jan and Konrad.
- Removal of the cpqarray driver. It has been disabled in Kconfig
since 2013, and we were initially scheduled to remove it in 3.15.
- Various updates and fixes for NVMe, with the most important being:
- Removal of the per-device NVMe thread, replacing that with a
watchdog timer instead. From Christoph.
- Exposing the namespace WWID through sysfs, from Keith.
- Set of cleanups from Ming Lin.
- Logging the controller device name instead of the underlying
PCI device name, from Sagi.
- And a bunch of fixes and optimizations from the usual suspects
in this area"
* 'for-4.6/drivers' of git://git.kernel.dk/linux-block: (49 commits)
NVMe: Expose ns wwid through single sysfs entry
drivers:block: cpqarray clean up
brd: Fix discard request processing
cpqarray: remove it from the kernel
cciss: update MAINTAINERS
NVMe: Remove unused sq_head read in completion path
bcache: fix cache_set_flush() NULL pointer dereference on OOM
bcache: cleaned up error handling around register_cache()
bcache: fix race of writeback thread starting before complete initialization
NVMe: Create discard zero quirk white list
nbd: use correct div_s64 helper
mtip32xx: remove unneeded variable in mtip_cmd_timeout()
lightnvm: generalize rrpc ppa calculations
lightnvm: remove struct nvm_dev->total_blocks
lightnvm: rename ->nr_pages to ->nr_sects
lightnvm: update closed list outside of intr context
xen/blback: Fit the important information of the thread in 17 characters
lightnvm: fold get bb tbl when using dual/quad plane mode
lightnvm: fix up nonsensical configure overrun checking
xen-blkback: advertise indirect segment support earlier
...
2016-03-19 07:13:31 +07:00
|
|
|
dev_warn(dev->ctrl.device, "Removing after probe failure status: %d\n", status);
|
2016-02-24 23:15:55 +07:00
|
|
|
|
2017-10-18 18:25:42 +07:00
|
|
|
nvme_get_ctrl(&dev->ctrl);
|
2016-02-24 23:15:56 +07:00
|
|
|
nvme_dev_disable(dev, false);
|
2018-06-20 12:42:22 +07:00
|
|
|
nvme_kill_queues(&dev->ctrl);
|
2017-11-09 18:32:07 +07:00
|
|
|
if (!queue_work(nvme_wq, &dev->remove_work))
|
2016-02-24 23:15:55 +07:00
|
|
|
nvme_put_ctrl(&dev->ctrl);
|
|
|
|
}
|
|
|
|
|
2015-11-26 18:42:26 +07:00
|
|
|
static void nvme_reset_work(struct work_struct *work)
|
2013-02-20 00:17:58 +07:00
|
|
|
{
|
2017-06-15 20:41:08 +07:00
|
|
|
struct nvme_dev *dev =
|
|
|
|
container_of(work, struct nvme_dev, ctrl.reset_work);
|
2017-02-04 02:50:32 +07:00
|
|
|
bool was_suspend = !!(dev->ctrl.ctrl_config & NVME_CC_SHN_NORMAL);
|
2016-02-24 23:15:55 +07:00
|
|
|
int result = -ENODEV;
|
2018-01-06 07:01:58 +07:00
|
|
|
enum nvme_ctrl_state new_state = NVME_CTRL_LIVE;
|
2013-02-20 00:17:58 +07:00
|
|
|
|
2017-06-05 18:43:11 +07:00
|
|
|
if (WARN_ON(dev->ctrl.state != NVME_CTRL_RESETTING))
|
2015-11-26 18:42:26 +07:00
|
|
|
goto out;
|
2013-02-20 00:17:58 +07:00
|
|
|
|
2015-11-26 18:42:26 +07:00
|
|
|
/*
|
|
|
|
* If we're called to reset a live controller first shut it down before
|
|
|
|
* moving on.
|
|
|
|
*/
|
2016-02-24 23:15:52 +07:00
|
|
|
if (dev->ctrl.ctrl_config & NVME_CC_ENABLE)
|
2016-01-13 04:41:18 +07:00
|
|
|
nvme_dev_disable(dev, false);
|
2013-02-20 00:17:58 +07:00
|
|
|
|
2018-01-22 21:03:16 +07:00
|
|
|
/*
|
2018-01-31 23:31:24 +07:00
|
|
|
* Introduce CONNECTING state from nvme-fc/rdma transports to mark the
|
2018-01-22 21:03:16 +07:00
|
|
|
* initializing procedure here.
|
|
|
|
*/
|
2018-01-31 23:31:24 +07:00
|
|
|
if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_CONNECTING)) {
|
2018-01-22 21:03:16 +07:00
|
|
|
dev_warn(dev->ctrl.device,
|
2018-01-31 23:31:24 +07:00
|
|
|
"failed to mark controller CONNECTING\n");
|
2018-01-22 21:03:16 +07:00
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2016-02-24 23:15:52 +07:00
|
|
|
result = nvme_pci_enable(dev);
|
2013-07-16 04:02:21 +07:00
|
|
|
if (result)
|
2015-10-03 14:49:23 +07:00
|
|
|
goto out;
|
2013-07-16 04:02:21 +07:00
|
|
|
|
2017-05-01 04:27:17 +07:00
|
|
|
result = nvme_pci_configure_admin_queue(dev);
|
2013-07-16 04:02:21 +07:00
|
|
|
if (result)
|
2016-02-24 23:15:55 +07:00
|
|
|
goto out;
|
2013-07-16 04:02:21 +07:00
|
|
|
|
2015-01-08 08:55:50 +07:00
|
|
|
result = nvme_alloc_admin_tags(dev);
|
|
|
|
if (result)
|
2016-02-24 23:15:55 +07:00
|
|
|
goto out;
|
2014-04-08 06:10:11 +07:00
|
|
|
|
2018-06-21 22:49:37 +07:00
|
|
|
/*
|
|
|
|
* Limit the max command size to prevent iod->sg allocations going
|
|
|
|
* over a single page.
|
|
|
|
*/
|
|
|
|
dev->ctrl.max_hw_sectors = NVME_MAX_KB_SZ << 1;
|
|
|
|
dev->ctrl.max_segments = NVME_MAX_SEGS;
|
|
|
|
|
2015-10-16 12:58:46 +07:00
|
|
|
result = nvme_init_identify(&dev->ctrl);
|
|
|
|
if (result)
|
2016-02-24 23:15:55 +07:00
|
|
|
goto out;
|
2015-10-16 12:58:46 +07:00
|
|
|
|
2017-02-23 00:15:07 +07:00
|
|
|
if (dev->ctrl.oacs & NVME_CTRL_OACS_SEC_SUPP) {
|
|
|
|
if (!dev->ctrl.opal_dev)
|
|
|
|
dev->ctrl.opal_dev =
|
|
|
|
init_opal_dev(&dev->ctrl, &nvme_sec_submit);
|
|
|
|
else if (was_suspend)
|
|
|
|
opal_unlock_from_suspend(dev->ctrl.opal_dev);
|
|
|
|
} else {
|
|
|
|
free_opal_dev(dev->ctrl.opal_dev);
|
|
|
|
dev->ctrl.opal_dev = NULL;
|
2017-02-17 19:59:39 +07:00
|
|
|
}
|
2017-02-04 02:50:32 +07:00
|
|
|
|
2017-04-10 22:51:07 +07:00
|
|
|
if (dev->ctrl.oacs & NVME_CTRL_OACS_DBBUF_SUPP) {
|
|
|
|
result = nvme_dbbuf_dma_alloc(dev);
|
|
|
|
if (result)
|
|
|
|
dev_warn(dev->dev,
|
|
|
|
"unable to allocate dma for dbbuf\n");
|
|
|
|
}
|
|
|
|
|
2017-09-06 17:19:57 +07:00
|
|
|
if (dev->ctrl.hmpre) {
|
|
|
|
result = nvme_setup_host_mem(dev);
|
|
|
|
if (result < 0)
|
|
|
|
goto out;
|
|
|
|
}
|
2017-05-12 22:02:58 +07:00
|
|
|
|
2013-07-16 04:02:21 +07:00
|
|
|
result = nvme_setup_io_queues(dev);
|
2014-06-24 03:25:35 +07:00
|
|
|
if (result)
|
2016-02-24 23:15:55 +07:00
|
|
|
goto out;
|
2013-07-16 04:02:21 +07:00
|
|
|
|
2015-10-02 23:51:31 +07:00
|
|
|
/*
|
|
|
|
* Keep the controller around but remove all namespaces if we don't have
|
|
|
|
* any working I/O queue.
|
|
|
|
*/
|
2015-10-03 14:49:23 +07:00
|
|
|
if (dev->online_queues < 2) {
|
2016-02-10 22:51:15 +07:00
|
|
|
dev_warn(dev->ctrl.device, "IO queues not created\n");
|
2016-04-28 04:51:18 +07:00
|
|
|
nvme_kill_queues(&dev->ctrl);
|
2015-11-28 21:39:07 +07:00
|
|
|
nvme_remove_namespaces(&dev->ctrl);
|
2018-01-06 07:01:58 +07:00
|
|
|
new_state = NVME_CTRL_ADMIN_ONLY;
|
2015-10-03 14:49:23 +07:00
|
|
|
} else {
|
2016-01-04 23:10:57 +07:00
|
|
|
nvme_start_queues(&dev->ctrl);
|
2017-03-02 02:22:12 +07:00
|
|
|
nvme_wait_freeze(&dev->ctrl);
|
2018-01-06 07:01:58 +07:00
|
|
|
/* hit this only when allocate tagset fails */
|
|
|
|
if (nvme_dev_add(dev))
|
|
|
|
new_state = NVME_CTRL_ADMIN_ONLY;
|
2017-03-02 02:22:12 +07:00
|
|
|
nvme_unfreeze(&dev->ctrl);
|
2015-10-03 14:49:23 +07:00
|
|
|
}
|
|
|
|
|
2018-01-06 07:01:58 +07:00
|
|
|
/*
|
|
|
|
* If only admin queue live, keep it to do further investigation or
|
|
|
|
* recovery.
|
|
|
|
*/
|
|
|
|
if (!nvme_change_ctrl_state(&dev->ctrl, new_state)) {
|
|
|
|
dev_warn(dev->ctrl.device,
|
|
|
|
"failed to mark controller state %d\n", new_state);
|
2016-04-26 18:51:57 +07:00
|
|
|
goto out;
|
|
|
|
}
|
2016-04-26 18:51:58 +07:00
|
|
|
|
2017-07-02 14:56:43 +07:00
|
|
|
nvme_start_ctrl(&dev->ctrl);
|
2015-10-03 14:49:23 +07:00
|
|
|
return;
|
2013-07-16 04:02:21 +07:00
|
|
|
|
2015-10-03 14:49:23 +07:00
|
|
|
out:
|
2016-02-24 23:15:55 +07:00
|
|
|
nvme_remove_dead_ctrl(dev, result);
|
2013-07-16 04:02:21 +07:00
|
|
|
}
|
|
|
|
|
2015-11-26 18:35:49 +07:00
|
|
|
static void nvme_remove_dead_ctrl_work(struct work_struct *work)
|
2013-12-11 03:10:36 +07:00
|
|
|
{
|
2015-11-26 18:35:49 +07:00
|
|
|
struct nvme_dev *dev = container_of(work, struct nvme_dev, remove_work);
|
2015-05-22 16:12:39 +07:00
|
|
|
struct pci_dev *pdev = to_pci_dev(dev->dev);
|
2013-12-11 03:10:36 +07:00
|
|
|
|
|
|
|
if (pci_get_drvdata(pdev))
|
2016-03-29 05:03:21 +07:00
|
|
|
device_release_driver(&pdev->dev);
|
2015-11-26 16:54:19 +07:00
|
|
|
nvme_put_ctrl(&dev->ctrl);
|
2013-12-11 03:10:36 +07:00
|
|
|
}
|
|
|
|
|
2015-11-26 16:06:56 +07:00
|
|
|
static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
|
2014-03-07 22:24:49 +07:00
|
|
|
{
|
2015-11-26 16:06:56 +07:00
|
|
|
*val = readl(to_nvme_dev(ctrl)->bar + off);
|
2015-10-02 23:49:23 +07:00
|
|
|
return 0;
|
2014-03-07 22:24:49 +07:00
|
|
|
}
|
|
|
|
|
2015-11-28 21:03:49 +07:00
|
|
|
static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
|
2015-06-05 23:30:08 +07:00
|
|
|
{
|
2015-11-28 21:03:49 +07:00
|
|
|
writel(val, to_nvme_dev(ctrl)->bar + off);
|
|
|
|
return 0;
|
|
|
|
}
|
2015-06-05 23:30:08 +07:00
|
|
|
|
2015-11-28 21:37:52 +07:00
|
|
|
static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
|
|
|
|
{
|
|
|
|
*val = readq(to_nvme_dev(ctrl)->bar + off);
|
|
|
|
return 0;
|
2015-06-05 23:30:08 +07:00
|
|
|
}
|
|
|
|
|
2018-03-09 04:50:32 +07:00
|
|
|
static int nvme_pci_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
|
|
|
|
{
|
|
|
|
struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev);
|
|
|
|
|
|
|
|
return snprintf(buf, size, "%s", dev_name(&pdev->dev));
|
|
|
|
}
|
|
|
|
|
2015-11-26 16:06:56 +07:00
|
|
|
static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
|
2016-06-13 21:45:24 +07:00
|
|
|
.name = "pcie",
|
2016-02-11 01:03:29 +07:00
|
|
|
.module = THIS_MODULE,
|
2017-05-20 20:14:45 +07:00
|
|
|
.flags = NVME_F_METADATA_SUPPORTED,
|
2015-11-26 16:06:56 +07:00
|
|
|
.reg_read32 = nvme_pci_reg_read32,
|
2015-11-28 21:03:49 +07:00
|
|
|
.reg_write32 = nvme_pci_reg_write32,
|
2015-11-28 21:37:52 +07:00
|
|
|
.reg_read64 = nvme_pci_reg_read64,
|
2015-11-26 16:54:19 +07:00
|
|
|
.free_ctrl = nvme_pci_free_ctrl,
|
2016-04-26 18:52:00 +07:00
|
|
|
.submit_async_event = nvme_pci_submit_async_event,
|
2018-03-09 04:50:32 +07:00
|
|
|
.get_address = nvme_pci_get_address,
|
2015-11-26 16:06:56 +07:00
|
|
|
};
|
2015-06-05 23:30:08 +07:00
|
|
|
|
2016-02-24 23:15:52 +07:00
|
|
|
static int nvme_dev_map(struct nvme_dev *dev)
|
|
|
|
{
|
|
|
|
struct pci_dev *pdev = to_pci_dev(dev->dev);
|
|
|
|
|
2016-06-07 14:44:02 +07:00
|
|
|
if (pci_request_mem_regions(pdev, "nvme"))
|
2016-02-24 23:15:52 +07:00
|
|
|
return -ENODEV;
|
|
|
|
|
2017-05-24 15:39:55 +07:00
|
|
|
if (nvme_remap_bar(dev, NVME_REG_DBS + 4096))
|
2016-02-24 23:15:52 +07:00
|
|
|
goto release;
|
|
|
|
|
2016-12-19 21:18:24 +07:00
|
|
|
return 0;
|
2016-02-24 23:15:52 +07:00
|
|
|
release:
|
2016-12-19 21:18:24 +07:00
|
|
|
pci_release_mem_regions(pdev);
|
|
|
|
return -ENODEV;
|
2016-02-24 23:15:52 +07:00
|
|
|
}
|
|
|
|
|
2017-11-09 13:12:03 +07:00
|
|
|
static unsigned long check_vendor_combination_bug(struct pci_dev *pdev)
|
2017-04-21 03:37:55 +07:00
|
|
|
{
|
|
|
|
if (pdev->vendor == 0x144d && pdev->device == 0xa802) {
|
|
|
|
/*
|
|
|
|
* Several Samsung devices seem to drop off the PCIe bus
|
|
|
|
* randomly when APST is on and uses the deepest sleep state.
|
|
|
|
* This has been observed on a Samsung "SM951 NVMe SAMSUNG
|
|
|
|
* 256GB", a "PM951 NVMe SAMSUNG 512GB", and a "Samsung SSD
|
|
|
|
* 950 PRO 256GB", but it seems to be restricted to two Dell
|
|
|
|
* laptops.
|
|
|
|
*/
|
|
|
|
if (dmi_match(DMI_SYS_VENDOR, "Dell Inc.") &&
|
|
|
|
(dmi_match(DMI_PRODUCT_NAME, "XPS 15 9550") ||
|
|
|
|
dmi_match(DMI_PRODUCT_NAME, "Precision 5510")))
|
|
|
|
return NVME_QUIRK_NO_DEEPEST_PS;
|
2017-11-09 13:12:03 +07:00
|
|
|
} else if (pdev->vendor == 0x144d && pdev->device == 0xa804) {
|
|
|
|
/*
|
|
|
|
* Samsung SSD 960 EVO drops off the PCIe bus after system
|
2018-03-12 01:51:56 +07:00
|
|
|
* suspend on a Ryzen board, ASUS PRIME B350M-A, as well as
|
|
|
|
* within few minutes after bootup on a Coffee Lake board -
|
|
|
|
* ASUS PRIME Z370-A
|
2017-11-09 13:12:03 +07:00
|
|
|
*/
|
|
|
|
if (dmi_match(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC.") &&
|
2018-03-12 01:51:56 +07:00
|
|
|
(dmi_match(DMI_BOARD_NAME, "PRIME B350M-A") ||
|
|
|
|
dmi_match(DMI_BOARD_NAME, "PRIME Z370-A")))
|
2017-11-09 13:12:03 +07:00
|
|
|
return NVME_QUIRK_NO_APST;
|
2017-04-21 03:37:55 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-04-28 02:42:52 +07:00
|
|
|
static void nvme_async_probe(void *data, async_cookie_t cookie)
|
|
|
|
{
|
|
|
|
struct nvme_dev *dev = data;
|
2018-05-07 21:30:24 +07:00
|
|
|
|
2018-04-28 02:42:52 +07:00
|
|
|
nvme_reset_ctrl_sync(&dev->ctrl);
|
|
|
|
flush_work(&dev->ctrl.scan_work);
|
2018-05-07 21:30:24 +07:00
|
|
|
nvme_put_ctrl(&dev->ctrl);
|
2018-04-28 02:42:52 +07:00
|
|
|
}
|
|
|
|
|
2012-12-22 06:13:49 +07:00
|
|
|
static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
|
2011-01-21 00:50:14 +07:00
|
|
|
{
|
2014-11-04 22:20:14 +07:00
|
|
|
int node, result = -ENOMEM;
|
2011-01-21 00:50:14 +07:00
|
|
|
struct nvme_dev *dev;
|
2017-04-21 03:37:55 +07:00
|
|
|
unsigned long quirks = id->driver_data;
|
2018-06-21 22:49:37 +07:00
|
|
|
size_t alloc_size;
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2014-11-04 22:20:14 +07:00
|
|
|
node = dev_to_node(&pdev->dev);
|
|
|
|
if (node == NUMA_NO_NODE)
|
2016-06-20 07:33:17 +07:00
|
|
|
set_dev_node(&pdev->dev, first_memory_node);
|
2014-11-04 22:20:14 +07:00
|
|
|
|
|
|
|
dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
|
2011-01-21 00:50:14 +07:00
|
|
|
if (!dev)
|
|
|
|
return -ENOMEM;
|
2018-01-14 17:39:01 +07:00
|
|
|
|
|
|
|
dev->queues = kcalloc_node(num_possible_cpus() + 1,
|
|
|
|
sizeof(struct nvme_queue), GFP_KERNEL, node);
|
2011-01-21 00:50:14 +07:00
|
|
|
if (!dev->queues)
|
|
|
|
goto free;
|
|
|
|
|
2015-05-22 16:12:39 +07:00
|
|
|
dev->dev = get_device(&pdev->dev);
|
2013-12-11 03:10:36 +07:00
|
|
|
pci_set_drvdata(pdev, dev);
|
2015-11-26 16:06:56 +07:00
|
|
|
|
2016-02-24 23:15:52 +07:00
|
|
|
result = nvme_dev_map(dev);
|
|
|
|
if (result)
|
2017-07-16 15:39:03 +07:00
|
|
|
goto put_pci;
|
2016-02-24 23:15:52 +07:00
|
|
|
|
2017-06-15 20:41:08 +07:00
|
|
|
INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work);
|
2015-11-26 18:35:49 +07:00
|
|
|
INIT_WORK(&dev->remove_work, nvme_remove_dead_ctrl_work);
|
2015-11-26 18:21:29 +07:00
|
|
|
mutex_init(&dev->shutdown_lock);
|
2016-01-13 04:41:17 +07:00
|
|
|
init_completion(&dev->ioq_wait);
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2011-02-10 21:56:01 +07:00
|
|
|
result = nvme_setup_prp_pools(dev);
|
|
|
|
if (result)
|
2017-07-16 15:39:03 +07:00
|
|
|
goto unmap;
|
2015-06-05 23:30:08 +07:00
|
|
|
|
2017-11-09 13:12:03 +07:00
|
|
|
quirks |= check_vendor_combination_bug(pdev);
|
2017-04-21 03:37:55 +07:00
|
|
|
|
2015-11-28 21:40:19 +07:00
|
|
|
result = nvme_init_ctrl(&dev->ctrl, &pdev->dev, &nvme_pci_ctrl_ops,
|
2017-04-21 03:37:55 +07:00
|
|
|
quirks);
|
2015-06-05 23:30:08 +07:00
|
|
|
if (result)
|
2015-02-13 05:33:00 +07:00
|
|
|
goto release_pools;
|
2011-02-16 04:28:20 +07:00
|
|
|
|
2018-06-21 22:49:37 +07:00
|
|
|
/*
|
|
|
|
* Double check that our mempool alloc size will cover the biggest
|
|
|
|
* command we support.
|
|
|
|
*/
|
|
|
|
alloc_size = nvme_pci_iod_alloc_size(dev, NVME_MAX_KB_SZ,
|
|
|
|
NVME_MAX_SEGS, true);
|
|
|
|
WARN_ON_ONCE(alloc_size > PAGE_SIZE);
|
|
|
|
|
|
|
|
dev->iod_mempool = mempool_create_node(1, mempool_kmalloc,
|
|
|
|
mempool_kfree,
|
|
|
|
(void *) alloc_size,
|
|
|
|
GFP_KERNEL, node);
|
|
|
|
if (!dev->iod_mempool) {
|
|
|
|
result = -ENOMEM;
|
|
|
|
goto release_pools;
|
|
|
|
}
|
|
|
|
|
2016-02-10 22:51:15 +07:00
|
|
|
dev_info(dev->ctrl.device, "pci function %s\n", dev_name(&pdev->dev));
|
|
|
|
|
2018-05-07 21:30:24 +07:00
|
|
|
nvme_get_ctrl(&dev->ctrl);
|
2018-04-28 02:42:52 +07:00
|
|
|
async_schedule(nvme_async_probe, dev);
|
2017-12-31 19:01:19 +07:00
|
|
|
|
2011-01-21 00:50:14 +07:00
|
|
|
return 0;
|
|
|
|
|
2013-07-16 04:02:19 +07:00
|
|
|
release_pools:
|
2011-02-10 21:56:01 +07:00
|
|
|
nvme_release_prp_pools(dev);
|
2017-07-16 15:39:03 +07:00
|
|
|
unmap:
|
|
|
|
nvme_dev_unmap(dev);
|
2014-08-20 08:15:59 +07:00
|
|
|
put_pci:
|
2015-05-22 16:12:39 +07:00
|
|
|
put_device(dev->dev);
|
2011-01-21 00:50:14 +07:00
|
|
|
free:
|
|
|
|
kfree(dev->queues);
|
|
|
|
kfree(dev);
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2017-06-01 18:10:38 +07:00
|
|
|
static void nvme_reset_prepare(struct pci_dev *pdev)
|
2014-05-02 23:40:43 +07:00
|
|
|
{
|
2014-06-24 05:03:21 +07:00
|
|
|
struct nvme_dev *dev = pci_get_drvdata(pdev);
|
2017-07-09 05:51:57 +07:00
|
|
|
nvme_dev_disable(dev, false);
|
2017-06-01 18:10:38 +07:00
|
|
|
}
|
2014-05-02 23:40:43 +07:00
|
|
|
|
2017-06-01 18:10:38 +07:00
|
|
|
static void nvme_reset_done(struct pci_dev *pdev)
|
|
|
|
{
|
2017-07-09 05:51:57 +07:00
|
|
|
struct nvme_dev *dev = pci_get_drvdata(pdev);
|
2018-01-14 17:39:00 +07:00
|
|
|
nvme_reset_ctrl_sync(&dev->ctrl);
|
2014-05-02 23:40:43 +07:00
|
|
|
}
|
|
|
|
|
2014-01-27 23:29:40 +07:00
|
|
|
static void nvme_shutdown(struct pci_dev *pdev)
|
|
|
|
{
|
|
|
|
struct nvme_dev *dev = pci_get_drvdata(pdev);
|
2016-01-13 04:41:18 +07:00
|
|
|
nvme_dev_disable(dev, true);
|
2014-01-27 23:29:40 +07:00
|
|
|
}
|
|
|
|
|
2016-02-24 23:15:55 +07:00
|
|
|
/*
|
|
|
|
* The driver's remove may be called on a device in a partially initialized
|
|
|
|
* state. This function must not have any dependencies on the device state in
|
|
|
|
* order to proceed.
|
|
|
|
*/
|
2012-12-22 06:13:49 +07:00
|
|
|
static void nvme_remove(struct pci_dev *pdev)
|
2011-01-21 00:50:14 +07:00
|
|
|
{
|
|
|
|
struct nvme_dev *dev = pci_get_drvdata(pdev);
|
2013-12-11 03:10:36 +07:00
|
|
|
|
2016-04-26 18:51:57 +07:00
|
|
|
nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
|
|
|
|
|
2017-06-15 20:41:08 +07:00
|
|
|
cancel_work_sync(&dev->ctrl.reset_work);
|
2013-12-11 03:10:36 +07:00
|
|
|
pci_set_drvdata(pdev, NULL);
|
2016-05-12 21:37:14 +07:00
|
|
|
|
2017-02-11 06:15:49 +07:00
|
|
|
if (!pci_device_is_present(pdev)) {
|
2016-05-12 21:37:14 +07:00
|
|
|
nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD);
|
2018-06-06 21:13:08 +07:00
|
|
|
nvme_dev_disable(dev, true);
|
2017-02-11 06:15:49 +07:00
|
|
|
}
|
2016-05-12 21:37:14 +07:00
|
|
|
|
2017-06-15 20:41:08 +07:00
|
|
|
flush_work(&dev->ctrl.reset_work);
|
2017-07-02 14:56:43 +07:00
|
|
|
nvme_stop_ctrl(&dev->ctrl);
|
|
|
|
nvme_remove_namespaces(&dev->ctrl);
|
2016-01-13 04:41:18 +07:00
|
|
|
nvme_dev_disable(dev, true);
|
2017-05-12 22:02:58 +07:00
|
|
|
nvme_free_host_mem(dev);
|
2014-11-04 22:20:14 +07:00
|
|
|
nvme_dev_remove_admin(dev);
|
2013-12-17 01:50:00 +07:00
|
|
|
nvme_free_queues(dev, 0);
|
2017-07-02 14:56:43 +07:00
|
|
|
nvme_uninit_ctrl(&dev->ctrl);
|
2013-12-11 03:10:36 +07:00
|
|
|
nvme_release_prp_pools(dev);
|
2016-02-24 23:15:52 +07:00
|
|
|
nvme_dev_unmap(dev);
|
2015-11-26 16:54:19 +07:00
|
|
|
nvme_put_ctrl(&dev->ctrl);
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
|
|
|
|
2014-02-13 09:19:14 +07:00
|
|
|
#ifdef CONFIG_PM_SLEEP
|
2013-07-16 04:02:23 +07:00
|
|
|
static int nvme_suspend(struct device *dev)
|
|
|
|
{
|
|
|
|
struct pci_dev *pdev = to_pci_dev(dev);
|
|
|
|
struct nvme_dev *ndev = pci_get_drvdata(pdev);
|
|
|
|
|
2016-01-13 04:41:18 +07:00
|
|
|
nvme_dev_disable(ndev, true);
|
2013-07-16 04:02:23 +07:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int nvme_resume(struct device *dev)
|
|
|
|
{
|
|
|
|
struct pci_dev *pdev = to_pci_dev(dev);
|
|
|
|
struct nvme_dev *ndev = pci_get_drvdata(pdev);
|
|
|
|
|
2017-06-15 20:41:08 +07:00
|
|
|
nvme_reset_ctrl(&ndev->ctrl);
|
2013-12-11 03:10:36 +07:00
|
|
|
return 0;
|
2013-07-16 04:02:23 +07:00
|
|
|
}
|
2014-02-13 09:19:14 +07:00
|
|
|
#endif
|
2013-07-16 04:02:23 +07:00
|
|
|
|
|
|
|
static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);
|
2011-01-21 00:50:14 +07:00
|
|
|
|
2015-12-08 05:30:31 +07:00
|
|
|
static pci_ers_result_t nvme_error_detected(struct pci_dev *pdev,
|
|
|
|
pci_channel_state_t state)
|
|
|
|
{
|
|
|
|
struct nvme_dev *dev = pci_get_drvdata(pdev);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* A frozen channel requires a reset. When detected, this method will
|
|
|
|
* shutdown the controller to quiesce. The controller will be restarted
|
|
|
|
* after the slot reset through driver's slot_reset callback.
|
|
|
|
*/
|
|
|
|
switch (state) {
|
|
|
|
case pci_channel_io_normal:
|
|
|
|
return PCI_ERS_RESULT_CAN_RECOVER;
|
|
|
|
case pci_channel_io_frozen:
|
2016-04-05 04:07:41 +07:00
|
|
|
dev_warn(dev->ctrl.device,
|
|
|
|
"frozen state error detected, reset controller\n");
|
2016-01-13 04:41:18 +07:00
|
|
|
nvme_dev_disable(dev, false);
|
2015-12-08 05:30:31 +07:00
|
|
|
return PCI_ERS_RESULT_NEED_RESET;
|
|
|
|
case pci_channel_io_perm_failure:
|
2016-04-05 04:07:41 +07:00
|
|
|
dev_warn(dev->ctrl.device,
|
|
|
|
"failure state error detected, request disconnect\n");
|
2015-12-08 05:30:31 +07:00
|
|
|
return PCI_ERS_RESULT_DISCONNECT;
|
|
|
|
}
|
|
|
|
return PCI_ERS_RESULT_NEED_RESET;
|
|
|
|
}
|
|
|
|
|
|
|
|
static pci_ers_result_t nvme_slot_reset(struct pci_dev *pdev)
|
|
|
|
{
|
|
|
|
struct nvme_dev *dev = pci_get_drvdata(pdev);
|
|
|
|
|
2016-02-10 22:51:15 +07:00
|
|
|
dev_info(dev->ctrl.device, "restart after slot reset\n");
|
2015-12-08 05:30:31 +07:00
|
|
|
pci_restore_state(pdev);
|
2017-06-15 20:41:08 +07:00
|
|
|
nvme_reset_ctrl(&dev->ctrl);
|
2015-12-08 05:30:31 +07:00
|
|
|
return PCI_ERS_RESULT_RECOVERED;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void nvme_error_resume(struct pci_dev *pdev)
|
|
|
|
{
|
2018-05-25 05:16:04 +07:00
|
|
|
struct nvme_dev *dev = pci_get_drvdata(pdev);
|
|
|
|
|
|
|
|
flush_work(&dev->ctrl.reset_work);
|
2015-12-08 05:30:31 +07:00
|
|
|
pci_cleanup_aer_uncorrect_error_status(pdev);
|
|
|
|
}
|
|
|
|
|
2012-09-07 23:33:17 +07:00
|
|
|
static const struct pci_error_handlers nvme_err_handler = {
|
2011-01-21 00:50:14 +07:00
|
|
|
.error_detected = nvme_error_detected,
|
|
|
|
.slot_reset = nvme_slot_reset,
|
|
|
|
.resume = nvme_error_resume,
|
2017-06-01 18:10:38 +07:00
|
|
|
.reset_prepare = nvme_reset_prepare,
|
|
|
|
.reset_done = nvme_reset_done,
|
2011-01-21 00:50:14 +07:00
|
|
|
};
|
|
|
|
|
2014-03-24 21:11:22 +07:00
|
|
|
static const struct pci_device_id nvme_id_table[] = {
|
2015-11-26 16:07:41 +07:00
|
|
|
{ PCI_VDEVICE(INTEL, 0x0953),
|
2016-03-05 03:15:17 +07:00
|
|
|
.driver_data = NVME_QUIRK_STRIPE_SIZE |
|
2017-04-06 00:21:13 +07:00
|
|
|
NVME_QUIRK_DEALLOCATE_ZEROES, },
|
2016-05-03 04:14:24 +07:00
|
|
|
{ PCI_VDEVICE(INTEL, 0x0a53),
|
|
|
|
.driver_data = NVME_QUIRK_STRIPE_SIZE |
|
2017-04-06 00:21:13 +07:00
|
|
|
NVME_QUIRK_DEALLOCATE_ZEROES, },
|
2016-05-03 04:14:24 +07:00
|
|
|
{ PCI_VDEVICE(INTEL, 0x0a54),
|
|
|
|
.driver_data = NVME_QUIRK_STRIPE_SIZE |
|
2017-04-06 00:21:13 +07:00
|
|
|
NVME_QUIRK_DEALLOCATE_ZEROES, },
|
2017-07-11 01:39:59 +07:00
|
|
|
{ PCI_VDEVICE(INTEL, 0x0a55),
|
|
|
|
.driver_data = NVME_QUIRK_STRIPE_SIZE |
|
|
|
|
NVME_QUIRK_DEALLOCATE_ZEROES, },
|
2017-05-25 05:06:31 +07:00
|
|
|
{ PCI_VDEVICE(INTEL, 0xf1a5), /* Intel 600P/P3100 */
|
2018-05-08 23:25:15 +07:00
|
|
|
.driver_data = NVME_QUIRK_NO_DEEPEST_PS |
|
|
|
|
NVME_QUIRK_MEDIUM_PRIO_SQ },
|
2015-10-23 04:45:06 +07:00
|
|
|
{ PCI_VDEVICE(INTEL, 0x5845), /* Qemu emulated controller */
|
|
|
|
.driver_data = NVME_QUIRK_IDENTIFY_CNS, },
|
2018-04-13 02:25:25 +07:00
|
|
|
{ PCI_DEVICE(0x1bb1, 0x0100), /* Seagate Nytro Flash Storage */
|
|
|
|
.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
|
2016-06-15 04:22:41 +07:00
|
|
|
{ PCI_DEVICE(0x1c58, 0x0003), /* HGST adapter */
|
|
|
|
.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
|
2017-11-21 23:44:37 +07:00
|
|
|
{ PCI_DEVICE(0x1c58, 0x0023), /* WDC SN200 adapter */
|
|
|
|
.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
|
2016-09-08 23:12:11 +07:00
|
|
|
{ PCI_DEVICE(0x1c5f, 0x0540), /* Memblaze Pblaze4 adapter */
|
|
|
|
.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
|
2017-06-28 09:27:57 +07:00
|
|
|
{ PCI_DEVICE(0x144d, 0xa821), /* Samsung PM1725 */
|
|
|
|
.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
|
|
|
|
{ PCI_DEVICE(0x144d, 0xa822), /* Samsung PM1725a */
|
|
|
|
.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
|
2017-09-06 16:45:24 +07:00
|
|
|
{ PCI_DEVICE(0x1d1d, 0x1f1f), /* LighNVM qemu device */
|
|
|
|
.driver_data = NVME_QUIRK_LIGHTNVM, },
|
|
|
|
{ PCI_DEVICE(0x1d1d, 0x2807), /* CNEX WL */
|
|
|
|
.driver_data = NVME_QUIRK_LIGHTNVM, },
|
2018-04-27 03:59:19 +07:00
|
|
|
{ PCI_DEVICE(0x1d1d, 0x2601), /* CNEX Granby */
|
|
|
|
.driver_data = NVME_QUIRK_LIGHTNVM, },
|
2011-01-21 00:50:14 +07:00
|
|
|
{ PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
|
2015-11-04 06:49:45 +07:00
|
|
|
{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) },
|
2017-02-23 05:17:29 +07:00
|
|
|
{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2003) },
|
2011-01-21 00:50:14 +07:00
|
|
|
{ 0, }
|
|
|
|
};
|
|
|
|
MODULE_DEVICE_TABLE(pci, nvme_id_table);
|
|
|
|
|
|
|
|
static struct pci_driver nvme_driver = {
|
|
|
|
.name = "nvme",
|
|
|
|
.id_table = nvme_id_table,
|
|
|
|
.probe = nvme_probe,
|
2012-12-22 06:13:49 +07:00
|
|
|
.remove = nvme_remove,
|
2014-01-27 23:29:40 +07:00
|
|
|
.shutdown = nvme_shutdown,
|
2013-07-16 04:02:23 +07:00
|
|
|
.driver = {
|
|
|
|
.pm = &nvme_dev_pm_ops,
|
|
|
|
},
|
2018-04-25 04:47:27 +07:00
|
|
|
.sriov_configure = pci_sriov_configure_simple,
|
2011-01-21 00:50:14 +07:00
|
|
|
.err_handler = &nvme_err_handler,
|
|
|
|
};
|
|
|
|
|
|
|
|
static int __init nvme_init(void)
|
|
|
|
{
|
2017-06-08 01:31:55 +07:00
|
|
|
return pci_register_driver(&nvme_driver);
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static void __exit nvme_exit(void)
|
|
|
|
{
|
|
|
|
pci_unregister_driver(&nvme_driver);
|
2017-11-09 18:32:07 +07:00
|
|
|
flush_workqueue(nvme_wq);
|
2014-05-10 09:42:26 +07:00
|
|
|
_nvme_check_size();
|
2011-01-21 00:50:14 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
|
|
|
|
MODULE_LICENSE("GPL");
|
2014-11-22 05:16:32 +07:00
|
|
|
MODULE_VERSION("1.0");
|
2011-01-21 00:50:14 +07:00
|
|
|
module_init(nvme_init);
|
|
|
|
module_exit(nvme_exit);
|