2015-10-03 20:46:41 +07:00
|
|
|
/*
|
|
|
|
* Copyright (c) 2011-2014, Intel Corporation.
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify it
|
|
|
|
* under the terms and conditions of the GNU General Public License,
|
|
|
|
* version 2, as published by the Free Software Foundation.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope it will be useful, but WITHOUT
|
|
|
|
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
|
|
* more details.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef _NVME_H
|
|
|
|
#define _NVME_H
|
|
|
|
|
|
|
|
#include <linux/nvme.h>
|
2017-10-18 21:59:25 +07:00
|
|
|
#include <linux/cdev.h>
|
2015-10-03 20:46:41 +07:00
|
|
|
#include <linux/pci.h>
|
|
|
|
#include <linux/kref.h>
|
|
|
|
#include <linux/blk-mq.h>
|
2016-09-16 19:25:07 +07:00
|
|
|
#include <linux/lightnvm.h>
|
2017-02-04 02:50:32 +07:00
|
|
|
#include <linux/sed-opal.h>
|
2015-10-03 20:46:41 +07:00
|
|
|
|
2017-09-07 07:23:56 +07:00
|
|
|
extern unsigned int nvme_io_timeout;
|
2015-10-03 20:46:41 +07:00
|
|
|
#define NVME_IO_TIMEOUT (nvme_io_timeout * HZ)
|
|
|
|
|
2017-09-07 07:23:56 +07:00
|
|
|
extern unsigned int admin_timeout;
|
2015-11-26 15:08:36 +07:00
|
|
|
#define ADMIN_TIMEOUT (admin_timeout * HZ)
|
|
|
|
|
2016-06-13 21:45:28 +07:00
|
|
|
#define NVME_DEFAULT_KATO 5
|
|
|
|
#define NVME_KATO_GRACE 10
|
|
|
|
|
2017-06-08 01:31:55 +07:00
|
|
|
extern struct workqueue_struct *nvme_wq;
|
2018-01-14 17:39:02 +07:00
|
|
|
extern struct workqueue_struct *nvme_reset_wq;
|
|
|
|
extern struct workqueue_struct *nvme_delete_wq;
|
2017-06-08 01:31:55 +07:00
|
|
|
|
2015-10-29 15:57:29 +07:00
|
|
|
enum {
|
|
|
|
NVME_NS_LBA = 0,
|
|
|
|
NVME_NS_LIGHTNVM = 1,
|
|
|
|
};
|
|
|
|
|
2015-10-03 20:46:41 +07:00
|
|
|
/*
|
2015-11-26 16:07:41 +07:00
|
|
|
* List of workarounds for devices that required behavior not specified in
|
|
|
|
* the standard.
|
2015-10-03 20:46:41 +07:00
|
|
|
*/
|
2015-11-26 16:07:41 +07:00
|
|
|
enum nvme_quirks {
|
|
|
|
/*
|
|
|
|
* Prefers I/O aligned to a stripe size specified in a vendor
|
|
|
|
* specific Identify field.
|
|
|
|
*/
|
|
|
|
NVME_QUIRK_STRIPE_SIZE = (1 << 0),
|
2015-10-23 04:45:06 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The controller doesn't handle Identify value others than 0 or 1
|
|
|
|
* correctly.
|
|
|
|
*/
|
|
|
|
NVME_QUIRK_IDENTIFY_CNS = (1 << 1),
|
2016-03-05 03:15:17 +07:00
|
|
|
|
|
|
|
/*
|
2017-04-06 00:21:13 +07:00
|
|
|
* The controller deterministically returns O's on reads to
|
|
|
|
* logical blocks that deallocate was called on.
|
2016-03-05 03:15:17 +07:00
|
|
|
*/
|
2017-04-06 00:21:13 +07:00
|
|
|
NVME_QUIRK_DEALLOCATE_ZEROES = (1 << 2),
|
2016-06-15 04:22:41 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The controller needs a delay before starts checking the device
|
|
|
|
* readiness, which is done by reading the NVME_CSTS_RDY bit.
|
|
|
|
*/
|
|
|
|
NVME_QUIRK_DELAY_BEFORE_CHK_RDY = (1 << 3),
|
nvme: Enable autonomous power state transitions
NVMe devices can advertise multiple power states. These states can
be either "operational" (the device is fully functional but possibly
slow) or "non-operational" (the device is asleep until woken up).
Some devices can automatically enter a non-operational state when
idle for a specified amount of time and then automatically wake back
up when needed.
The hardware configuration is a table. For each state, an entry in
the table indicates the next deeper non-operational state, if any,
to autonomously transition to and the idle time required before
transitioning.
This patch teaches the driver to program APST so that each successive
non-operational state will be entered after an idle time equal to 100%
of the total latency (entry plus exit) associated with that state.
The maximum acceptable latency is controlled using dev_pm_qos
(e.g. power/pm_qos_latency_tolerance_us in sysfs); non-operational
states with total latency greater than this value will not be used.
As a special case, setting the latency tolerance to 0 will disable
APST entirely. On hardware without APST support, the sysfs file will
not be exposed.
The latency tolerance for newly-probed devices is set by the module
parameter nvme_core.default_ps_max_latency_us.
In theory, the device can expose "default" APST table, but this
doesn't seem to function correctly on my device (Samsung 950), nor
does it seem particularly useful. There is also an optional
mechanism by which a configuration can be "saved" so it will be
automatically loaded on reset. This can be configured from
userspace, but it doesn't seem useful to support in the driver.
On my laptop, enabling APST seems to save nearly 1W.
The hardware tables can be decoded in userspace with nvme-cli.
'nvme id-ctrl /dev/nvmeN' will show the power state table and
'nvme get-feature -f 0x0c -H /dev/nvme0' will show the current APST
configuration.
This feature is quirked off on a known-buggy Samsung device.
Signed-off-by: Andy Lutomirski <luto@kernel.org>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Sagi Grimberg <sagi@grimberg.me>
Signed-off-by: Jens Axboe <axboe@fb.com>
2017-02-08 01:08:45 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* APST should not be used.
|
|
|
|
*/
|
|
|
|
NVME_QUIRK_NO_APST = (1 << 4),
|
2017-04-21 03:37:55 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The deepest sleep state should not be used.
|
|
|
|
*/
|
|
|
|
NVME_QUIRK_NO_DEEPEST_PS = (1 << 5),
|
2017-09-06 16:45:24 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Supports the LighNVM command set if indicated in vs[1].
|
|
|
|
*/
|
|
|
|
NVME_QUIRK_LIGHTNVM = (1 << 6),
|
2015-11-26 16:07:41 +07:00
|
|
|
};
|
|
|
|
|
2016-11-10 22:32:33 +07:00
|
|
|
/*
|
|
|
|
* Common request structure for NVMe passthrough. All drivers must have
|
|
|
|
* this structure as the first member of their request-private data.
|
|
|
|
*/
|
|
|
|
struct nvme_request {
|
|
|
|
struct nvme_command *cmd;
|
|
|
|
union nvme_result result;
|
2017-04-06 00:18:11 +07:00
|
|
|
u8 retries;
|
2017-04-20 21:02:57 +07:00
|
|
|
u8 flags;
|
|
|
|
u16 status;
|
|
|
|
};
|
|
|
|
|
2017-11-02 18:59:30 +07:00
|
|
|
/*
|
|
|
|
* Mark a bio as coming in through the mpath node.
|
|
|
|
*/
|
|
|
|
#define REQ_NVME_MPATH REQ_DRV
|
|
|
|
|
2017-04-20 21:02:57 +07:00
|
|
|
enum {
|
|
|
|
NVME_REQ_CANCELLED = (1 << 0),
|
2016-11-10 22:32:33 +07:00
|
|
|
};
|
|
|
|
|
|
|
|
static inline struct nvme_request *nvme_req(struct request *req)
|
|
|
|
{
|
|
|
|
return blk_mq_rq_to_pdu(req);
|
|
|
|
}
|
|
|
|
|
2016-06-15 04:22:41 +07:00
|
|
|
/* The below value is the specific amount of delay needed before checking
|
|
|
|
* readiness in case of the PCI_DEVICE(0x1c58, 0x0003), which needs the
|
|
|
|
* NVME_QUIRK_DELAY_BEFORE_CHK_RDY quirk enabled. The value (in ms) was
|
|
|
|
* found empirically.
|
|
|
|
*/
|
2017-11-21 23:44:37 +07:00
|
|
|
#define NVME_QUIRK_DELAY_AMOUNT 2300
|
2016-06-15 04:22:41 +07:00
|
|
|
|
2016-04-26 18:51:57 +07:00
|
|
|
enum nvme_ctrl_state {
|
|
|
|
NVME_CTRL_NEW,
|
|
|
|
NVME_CTRL_LIVE,
|
2018-01-06 07:01:58 +07:00
|
|
|
NVME_CTRL_ADMIN_ONLY, /* Only admin queue live */
|
2016-04-26 18:51:57 +07:00
|
|
|
NVME_CTRL_RESETTING,
|
2016-07-06 19:55:49 +07:00
|
|
|
NVME_CTRL_RECONNECTING,
|
2016-04-26 18:51:57 +07:00
|
|
|
NVME_CTRL_DELETING,
|
2016-05-12 21:37:14 +07:00
|
|
|
NVME_CTRL_DEAD,
|
2016-04-26 18:51:57 +07:00
|
|
|
};
|
|
|
|
|
2015-11-26 16:06:56 +07:00
|
|
|
struct nvme_ctrl {
|
2016-04-26 18:51:57 +07:00
|
|
|
enum nvme_ctrl_state state;
|
2017-02-23 03:32:36 +07:00
|
|
|
bool identified;
|
2016-04-26 18:51:57 +07:00
|
|
|
spinlock_t lock;
|
2015-11-26 16:06:56 +07:00
|
|
|
const struct nvme_ctrl_ops *ops;
|
2015-10-03 20:46:41 +07:00
|
|
|
struct request_queue *admin_q;
|
2016-06-13 21:45:26 +07:00
|
|
|
struct request_queue *connect_q;
|
2015-10-03 20:46:41 +07:00
|
|
|
struct device *dev;
|
|
|
|
int instance;
|
2015-11-28 21:39:07 +07:00
|
|
|
struct blk_mq_tag_set *tagset;
|
2017-07-10 13:22:29 +07:00
|
|
|
struct blk_mq_tag_set *admin_tagset;
|
2015-10-03 20:46:41 +07:00
|
|
|
struct list_head namespaces;
|
2015-12-24 21:27:00 +07:00
|
|
|
struct mutex namespaces_mutex;
|
2017-10-18 18:25:42 +07:00
|
|
|
struct device ctrl_device;
|
2015-11-28 21:39:07 +07:00
|
|
|
struct device *device; /* char device */
|
2017-10-18 21:59:25 +07:00
|
|
|
struct cdev cdev;
|
2017-06-15 20:41:08 +07:00
|
|
|
struct work_struct reset_work;
|
2017-10-29 15:44:29 +07:00
|
|
|
struct work_struct delete_work;
|
2015-11-26 16:06:56 +07:00
|
|
|
|
2017-11-09 19:48:55 +07:00
|
|
|
struct nvme_subsystem *subsys;
|
|
|
|
struct list_head subsys_entry;
|
|
|
|
|
2017-02-17 19:59:39 +07:00
|
|
|
struct opal_dev *opal_dev;
|
2017-02-04 02:50:32 +07:00
|
|
|
|
2015-10-03 20:46:41 +07:00
|
|
|
char name[12];
|
2016-04-17 01:57:58 +07:00
|
|
|
u16 cntlid;
|
2015-11-28 21:03:49 +07:00
|
|
|
|
|
|
|
u32 ctrl_config;
|
2017-07-12 17:40:40 +07:00
|
|
|
u16 mtfa;
|
2017-04-24 14:58:29 +07:00
|
|
|
u32 queue_count;
|
2015-11-28 21:03:49 +07:00
|
|
|
|
2017-06-28 02:16:38 +07:00
|
|
|
u64 cap;
|
2015-11-28 21:03:49 +07:00
|
|
|
u32 page_size;
|
2015-10-03 20:46:41 +07:00
|
|
|
u32 max_hw_sectors;
|
|
|
|
u16 oncs;
|
2017-02-17 19:59:40 +07:00
|
|
|
u16 oacs;
|
2017-06-28 01:03:06 +07:00
|
|
|
u16 nssa;
|
|
|
|
u16 nr_streams;
|
2015-11-20 15:36:44 +07:00
|
|
|
atomic_t abort_limit;
|
2015-10-03 20:46:41 +07:00
|
|
|
u8 vwc;
|
2015-11-28 21:40:19 +07:00
|
|
|
u32 vs;
|
2016-06-13 21:45:26 +07:00
|
|
|
u32 sgls;
|
2016-06-13 21:45:28 +07:00
|
|
|
u16 kas;
|
nvme: Enable autonomous power state transitions
NVMe devices can advertise multiple power states. These states can
be either "operational" (the device is fully functional but possibly
slow) or "non-operational" (the device is asleep until woken up).
Some devices can automatically enter a non-operational state when
idle for a specified amount of time and then automatically wake back
up when needed.
The hardware configuration is a table. For each state, an entry in
the table indicates the next deeper non-operational state, if any,
to autonomously transition to and the idle time required before
transitioning.
This patch teaches the driver to program APST so that each successive
non-operational state will be entered after an idle time equal to 100%
of the total latency (entry plus exit) associated with that state.
The maximum acceptable latency is controlled using dev_pm_qos
(e.g. power/pm_qos_latency_tolerance_us in sysfs); non-operational
states with total latency greater than this value will not be used.
As a special case, setting the latency tolerance to 0 will disable
APST entirely. On hardware without APST support, the sysfs file will
not be exposed.
The latency tolerance for newly-probed devices is set by the module
parameter nvme_core.default_ps_max_latency_us.
In theory, the device can expose "default" APST table, but this
doesn't seem to function correctly on my device (Samsung 950), nor
does it seem particularly useful. There is also an optional
mechanism by which a configuration can be "saved" so it will be
automatically loaded on reset. This can be configured from
userspace, but it doesn't seem useful to support in the driver.
On my laptop, enabling APST seems to save nearly 1W.
The hardware tables can be decoded in userspace with nvme-cli.
'nvme id-ctrl /dev/nvmeN' will show the power state table and
'nvme get-feature -f 0x0c -H /dev/nvme0' will show the current APST
configuration.
This feature is quirked off on a known-buggy Samsung device.
Signed-off-by: Andy Lutomirski <luto@kernel.org>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Sagi Grimberg <sagi@grimberg.me>
Signed-off-by: Jens Axboe <axboe@fb.com>
2017-02-08 01:08:45 +07:00
|
|
|
u8 npss;
|
|
|
|
u8 apsta;
|
2017-11-08 05:13:14 +07:00
|
|
|
u32 aen_result;
|
2017-08-26 06:14:50 +07:00
|
|
|
unsigned int shutdown_timeout;
|
2016-06-13 21:45:28 +07:00
|
|
|
unsigned int kato;
|
2015-11-28 21:40:19 +07:00
|
|
|
bool subsystem;
|
2015-11-26 16:07:41 +07:00
|
|
|
unsigned long quirks;
|
nvme: Enable autonomous power state transitions
NVMe devices can advertise multiple power states. These states can
be either "operational" (the device is fully functional but possibly
slow) or "non-operational" (the device is asleep until woken up).
Some devices can automatically enter a non-operational state when
idle for a specified amount of time and then automatically wake back
up when needed.
The hardware configuration is a table. For each state, an entry in
the table indicates the next deeper non-operational state, if any,
to autonomously transition to and the idle time required before
transitioning.
This patch teaches the driver to program APST so that each successive
non-operational state will be entered after an idle time equal to 100%
of the total latency (entry plus exit) associated with that state.
The maximum acceptable latency is controlled using dev_pm_qos
(e.g. power/pm_qos_latency_tolerance_us in sysfs); non-operational
states with total latency greater than this value will not be used.
As a special case, setting the latency tolerance to 0 will disable
APST entirely. On hardware without APST support, the sysfs file will
not be exposed.
The latency tolerance for newly-probed devices is set by the module
parameter nvme_core.default_ps_max_latency_us.
In theory, the device can expose "default" APST table, but this
doesn't seem to function correctly on my device (Samsung 950), nor
does it seem particularly useful. There is also an optional
mechanism by which a configuration can be "saved" so it will be
automatically loaded on reset. This can be configured from
userspace, but it doesn't seem useful to support in the driver.
On my laptop, enabling APST seems to save nearly 1W.
The hardware tables can be decoded in userspace with nvme-cli.
'nvme id-ctrl /dev/nvmeN' will show the power state table and
'nvme get-feature -f 0x0c -H /dev/nvme0' will show the current APST
configuration.
This feature is quirked off on a known-buggy Samsung device.
Signed-off-by: Andy Lutomirski <luto@kernel.org>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Sagi Grimberg <sagi@grimberg.me>
Signed-off-by: Jens Axboe <axboe@fb.com>
2017-02-08 01:08:45 +07:00
|
|
|
struct nvme_id_power_state psd[32];
|
2017-11-08 00:28:32 +07:00
|
|
|
struct nvme_effects_log *effects;
|
2016-04-26 18:51:59 +07:00
|
|
|
struct work_struct scan_work;
|
2016-04-26 18:52:00 +07:00
|
|
|
struct work_struct async_event_work;
|
2016-06-13 21:45:28 +07:00
|
|
|
struct delayed_work ka_work;
|
2017-07-12 17:40:40 +07:00
|
|
|
struct work_struct fw_act_work;
|
2016-06-13 21:45:26 +07:00
|
|
|
|
nvme: Enable autonomous power state transitions
NVMe devices can advertise multiple power states. These states can
be either "operational" (the device is fully functional but possibly
slow) or "non-operational" (the device is asleep until woken up).
Some devices can automatically enter a non-operational state when
idle for a specified amount of time and then automatically wake back
up when needed.
The hardware configuration is a table. For each state, an entry in
the table indicates the next deeper non-operational state, if any,
to autonomously transition to and the idle time required before
transitioning.
This patch teaches the driver to program APST so that each successive
non-operational state will be entered after an idle time equal to 100%
of the total latency (entry plus exit) associated with that state.
The maximum acceptable latency is controlled using dev_pm_qos
(e.g. power/pm_qos_latency_tolerance_us in sysfs); non-operational
states with total latency greater than this value will not be used.
As a special case, setting the latency tolerance to 0 will disable
APST entirely. On hardware without APST support, the sysfs file will
not be exposed.
The latency tolerance for newly-probed devices is set by the module
parameter nvme_core.default_ps_max_latency_us.
In theory, the device can expose "default" APST table, but this
doesn't seem to function correctly on my device (Samsung 950), nor
does it seem particularly useful. There is also an optional
mechanism by which a configuration can be "saved" so it will be
automatically loaded on reset. This can be configured from
userspace, but it doesn't seem useful to support in the driver.
On my laptop, enabling APST seems to save nearly 1W.
The hardware tables can be decoded in userspace with nvme-cli.
'nvme id-ctrl /dev/nvmeN' will show the power state table and
'nvme get-feature -f 0x0c -H /dev/nvme0' will show the current APST
configuration.
This feature is quirked off on a known-buggy Samsung device.
Signed-off-by: Andy Lutomirski <luto@kernel.org>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Sagi Grimberg <sagi@grimberg.me>
Signed-off-by: Jens Axboe <axboe@fb.com>
2017-02-08 01:08:45 +07:00
|
|
|
/* Power saving configuration */
|
|
|
|
u64 ps_max_latency_us;
|
2017-06-27 03:39:54 +07:00
|
|
|
bool apst_enabled;
|
nvme: Enable autonomous power state transitions
NVMe devices can advertise multiple power states. These states can
be either "operational" (the device is fully functional but possibly
slow) or "non-operational" (the device is asleep until woken up).
Some devices can automatically enter a non-operational state when
idle for a specified amount of time and then automatically wake back
up when needed.
The hardware configuration is a table. For each state, an entry in
the table indicates the next deeper non-operational state, if any,
to autonomously transition to and the idle time required before
transitioning.
This patch teaches the driver to program APST so that each successive
non-operational state will be entered after an idle time equal to 100%
of the total latency (entry plus exit) associated with that state.
The maximum acceptable latency is controlled using dev_pm_qos
(e.g. power/pm_qos_latency_tolerance_us in sysfs); non-operational
states with total latency greater than this value will not be used.
As a special case, setting the latency tolerance to 0 will disable
APST entirely. On hardware without APST support, the sysfs file will
not be exposed.
The latency tolerance for newly-probed devices is set by the module
parameter nvme_core.default_ps_max_latency_us.
In theory, the device can expose "default" APST table, but this
doesn't seem to function correctly on my device (Samsung 950), nor
does it seem particularly useful. There is also an optional
mechanism by which a configuration can be "saved" so it will be
automatically loaded on reset. This can be configured from
userspace, but it doesn't seem useful to support in the driver.
On my laptop, enabling APST seems to save nearly 1W.
The hardware tables can be decoded in userspace with nvme-cli.
'nvme id-ctrl /dev/nvmeN' will show the power state table and
'nvme get-feature -f 0x0c -H /dev/nvme0' will show the current APST
configuration.
This feature is quirked off on a known-buggy Samsung device.
Signed-off-by: Andy Lutomirski <luto@kernel.org>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Sagi Grimberg <sagi@grimberg.me>
Signed-off-by: Jens Axboe <axboe@fb.com>
2017-02-08 01:08:45 +07:00
|
|
|
|
2017-09-11 23:09:28 +07:00
|
|
|
/* PCIe only: */
|
2017-05-12 22:16:10 +07:00
|
|
|
u32 hmpre;
|
|
|
|
u32 hmmin;
|
2017-09-11 23:09:28 +07:00
|
|
|
u32 hmminds;
|
|
|
|
u16 hmmaxd;
|
2017-05-12 22:16:10 +07:00
|
|
|
|
2016-06-13 21:45:26 +07:00
|
|
|
/* Fabrics only */
|
|
|
|
u16 sqsize;
|
|
|
|
u32 ioccsz;
|
|
|
|
u32 iorcsz;
|
|
|
|
u16 icdoff;
|
|
|
|
u16 maxcmd;
|
2017-05-04 17:33:15 +07:00
|
|
|
int nr_reconnects;
|
2016-06-13 21:45:26 +07:00
|
|
|
struct nvmf_ctrl_options *opts;
|
2015-10-03 20:46:41 +07:00
|
|
|
};
|
|
|
|
|
2017-11-09 19:48:55 +07:00
|
|
|
struct nvme_subsystem {
|
|
|
|
int instance;
|
|
|
|
struct device dev;
|
|
|
|
/*
|
|
|
|
* Because we unregister the device on the last put we need
|
|
|
|
* a separate refcount.
|
|
|
|
*/
|
|
|
|
struct kref ref;
|
|
|
|
struct list_head entry;
|
|
|
|
struct mutex lock;
|
|
|
|
struct list_head ctrls;
|
2017-11-09 19:50:43 +07:00
|
|
|
struct list_head nsheads;
|
2017-11-09 19:48:55 +07:00
|
|
|
char subnqn[NVMF_NQN_SIZE];
|
|
|
|
char serial[20];
|
|
|
|
char model[40];
|
|
|
|
char firmware_rev[8];
|
|
|
|
u8 cmic;
|
|
|
|
u16 vendor_id;
|
2017-11-09 19:50:43 +07:00
|
|
|
struct ida ns_ida;
|
2017-11-09 19:48:55 +07:00
|
|
|
};
|
|
|
|
|
2017-11-09 19:50:16 +07:00
|
|
|
/*
|
|
|
|
* Container structure for uniqueue namespace identifiers.
|
|
|
|
*/
|
|
|
|
struct nvme_ns_ids {
|
|
|
|
u8 eui64[8];
|
|
|
|
u8 nguid[16];
|
|
|
|
uuid_t uuid;
|
|
|
|
};
|
|
|
|
|
2017-11-09 19:50:43 +07:00
|
|
|
/*
|
|
|
|
* Anchor structure for namespaces. There is one for each namespace in a
|
|
|
|
* NVMe subsystem that any of our controllers can see, and the namespace
|
|
|
|
* structure for each controller is chained of it. For private namespaces
|
|
|
|
* there is a 1:1 relation to our namespace structures, that is ->list
|
|
|
|
* only ever has a single entry for private namespaces.
|
|
|
|
*/
|
|
|
|
struct nvme_ns_head {
|
2017-11-02 18:59:30 +07:00
|
|
|
#ifdef CONFIG_NVME_MULTIPATH
|
|
|
|
struct gendisk *disk;
|
|
|
|
struct nvme_ns __rcu *current_path;
|
|
|
|
struct bio_list requeue_list;
|
|
|
|
spinlock_t requeue_lock;
|
|
|
|
struct work_struct requeue_work;
|
|
|
|
#endif
|
2017-11-09 19:50:43 +07:00
|
|
|
struct list_head list;
|
|
|
|
struct srcu_struct srcu;
|
|
|
|
struct nvme_subsystem *subsys;
|
|
|
|
unsigned ns_id;
|
|
|
|
struct nvme_ns_ids ids;
|
|
|
|
struct list_head entry;
|
|
|
|
struct kref ref;
|
|
|
|
int instance;
|
|
|
|
};
|
|
|
|
|
2015-10-03 20:46:41 +07:00
|
|
|
struct nvme_ns {
|
|
|
|
struct list_head list;
|
|
|
|
|
2015-11-26 16:06:56 +07:00
|
|
|
struct nvme_ctrl *ctrl;
|
2015-10-03 20:46:41 +07:00
|
|
|
struct request_queue *queue;
|
|
|
|
struct gendisk *disk;
|
2017-11-09 19:50:43 +07:00
|
|
|
struct list_head siblings;
|
2016-09-16 19:25:07 +07:00
|
|
|
struct nvm_dev *ndev;
|
2015-10-03 20:46:41 +07:00
|
|
|
struct kref kref;
|
2017-11-09 19:50:43 +07:00
|
|
|
struct nvme_ns_head *head;
|
2015-10-03 20:46:41 +07:00
|
|
|
|
|
|
|
int lba_shift;
|
|
|
|
u16 ms;
|
2017-06-28 01:03:06 +07:00
|
|
|
u16 sgs;
|
|
|
|
u32 sws;
|
2015-10-03 20:46:41 +07:00
|
|
|
bool ext;
|
|
|
|
u8 pi_type;
|
2016-02-24 23:15:54 +07:00
|
|
|
unsigned long flags;
|
|
|
|
#define NVME_NS_REMOVING 0
|
2016-02-24 23:15:56 +07:00
|
|
|
#define NVME_NS_DEAD 1
|
2017-08-16 20:47:37 +07:00
|
|
|
u16 noiob;
|
2015-10-03 20:46:41 +07:00
|
|
|
};
|
|
|
|
|
2015-11-26 16:06:56 +07:00
|
|
|
struct nvme_ctrl_ops {
|
2016-06-13 21:45:24 +07:00
|
|
|
const char *name;
|
2016-02-11 01:03:29 +07:00
|
|
|
struct module *module;
|
2017-05-20 20:14:44 +07:00
|
|
|
unsigned int flags;
|
|
|
|
#define NVME_F_FABRICS (1 << 0)
|
2017-05-20 20:14:45 +07:00
|
|
|
#define NVME_F_METADATA_SUPPORTED (1 << 1)
|
2015-11-26 16:06:56 +07:00
|
|
|
int (*reg_read32)(struct nvme_ctrl *ctrl, u32 off, u32 *val);
|
2015-11-28 21:03:49 +07:00
|
|
|
int (*reg_write32)(struct nvme_ctrl *ctrl, u32 off, u32 val);
|
2015-11-28 21:37:52 +07:00
|
|
|
int (*reg_read64)(struct nvme_ctrl *ctrl, u32 off, u64 *val);
|
2015-11-26 16:54:19 +07:00
|
|
|
void (*free_ctrl)(struct nvme_ctrl *ctrl);
|
2017-11-08 05:13:12 +07:00
|
|
|
void (*submit_async_event)(struct nvme_ctrl *ctrl);
|
2017-10-29 15:44:29 +07:00
|
|
|
void (*delete_ctrl)(struct nvme_ctrl *ctrl);
|
2016-06-13 21:45:24 +07:00
|
|
|
int (*get_address)(struct nvme_ctrl *ctrl, char *buf, int size);
|
2017-10-11 16:53:07 +07:00
|
|
|
int (*reinit_request)(void *data, struct request *rq);
|
2015-10-03 20:46:41 +07:00
|
|
|
};
|
|
|
|
|
2015-11-26 16:06:56 +07:00
|
|
|
static inline bool nvme_ctrl_ready(struct nvme_ctrl *ctrl)
|
|
|
|
{
|
|
|
|
u32 val = 0;
|
|
|
|
|
|
|
|
if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &val))
|
|
|
|
return false;
|
|
|
|
return val & NVME_CSTS_RDY;
|
|
|
|
}
|
|
|
|
|
2015-11-28 21:40:19 +07:00
|
|
|
static inline int nvme_reset_subsystem(struct nvme_ctrl *ctrl)
|
|
|
|
{
|
|
|
|
if (!ctrl->subsystem)
|
|
|
|
return -ENOTTY;
|
|
|
|
return ctrl->ops->reg_write32(ctrl, NVME_REG_NSSR, 0x4E564D65);
|
|
|
|
}
|
|
|
|
|
2015-10-03 20:46:41 +07:00
|
|
|
static inline u64 nvme_block_nr(struct nvme_ns *ns, sector_t sector)
|
|
|
|
{
|
|
|
|
return (sector >> (ns->lba_shift - 9));
|
|
|
|
}
|
|
|
|
|
2016-04-26 04:33:20 +07:00
|
|
|
static inline void nvme_cleanup_cmd(struct request *req)
|
|
|
|
{
|
2016-12-09 05:20:32 +07:00
|
|
|
if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
|
|
|
|
kfree(page_address(req->special_vec.bv_page) +
|
|
|
|
req->special_vec.bv_offset);
|
|
|
|
}
|
2016-04-26 04:33:20 +07:00
|
|
|
}
|
|
|
|
|
2017-04-20 21:02:57 +07:00
|
|
|
static inline void nvme_end_request(struct request *req, __le16 status,
|
|
|
|
union nvme_result result)
|
2015-10-16 12:58:39 +07:00
|
|
|
{
|
2017-04-20 21:02:57 +07:00
|
|
|
struct nvme_request *rq = nvme_req(req);
|
2015-10-16 12:58:39 +07:00
|
|
|
|
2017-04-20 21:02:57 +07:00
|
|
|
rq->status = le16_to_cpu(status) >> 1;
|
|
|
|
rq->result = result;
|
2017-04-20 21:03:09 +07:00
|
|
|
blk_mq_complete_request(req);
|
2015-11-28 21:41:58 +07:00
|
|
|
}
|
|
|
|
|
2017-10-18 18:25:42 +07:00
|
|
|
static inline void nvme_get_ctrl(struct nvme_ctrl *ctrl)
|
|
|
|
{
|
|
|
|
get_device(ctrl->device);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void nvme_put_ctrl(struct nvme_ctrl *ctrl)
|
|
|
|
{
|
|
|
|
put_device(ctrl->device);
|
|
|
|
}
|
|
|
|
|
2017-03-30 18:41:32 +07:00
|
|
|
void nvme_complete_rq(struct request *req);
|
2016-05-19 04:05:02 +07:00
|
|
|
void nvme_cancel_request(struct request *req, void *data, bool reserved);
|
2016-04-26 18:51:57 +07:00
|
|
|
bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
|
|
|
|
enum nvme_ctrl_state new_state);
|
2015-11-28 21:03:49 +07:00
|
|
|
int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
|
|
|
|
int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
|
|
|
|
int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl);
|
2015-11-28 21:40:19 +07:00
|
|
|
int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
|
|
|
|
const struct nvme_ctrl_ops *ops, unsigned long quirks);
|
2015-11-28 21:41:02 +07:00
|
|
|
void nvme_uninit_ctrl(struct nvme_ctrl *ctrl);
|
2017-07-02 14:56:43 +07:00
|
|
|
void nvme_start_ctrl(struct nvme_ctrl *ctrl);
|
|
|
|
void nvme_stop_ctrl(struct nvme_ctrl *ctrl);
|
2015-11-26 16:54:19 +07:00
|
|
|
void nvme_put_ctrl(struct nvme_ctrl *ctrl);
|
2015-11-28 21:37:52 +07:00
|
|
|
int nvme_init_identify(struct nvme_ctrl *ctrl);
|
2015-11-28 21:39:07 +07:00
|
|
|
|
2016-04-26 18:51:59 +07:00
|
|
|
void nvme_queue_scan(struct nvme_ctrl *ctrl);
|
2015-11-28 21:39:07 +07:00
|
|
|
void nvme_remove_namespaces(struct nvme_ctrl *ctrl);
|
2015-11-26 16:54:19 +07:00
|
|
|
|
2017-02-17 19:59:39 +07:00
|
|
|
int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
|
|
|
|
bool send);
|
2017-02-04 02:50:32 +07:00
|
|
|
|
2016-11-10 22:32:34 +07:00
|
|
|
void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
|
|
|
|
union nvme_result *res);
|
2016-04-26 18:52:00 +07:00
|
|
|
|
2016-01-04 23:10:57 +07:00
|
|
|
void nvme_stop_queues(struct nvme_ctrl *ctrl);
|
|
|
|
void nvme_start_queues(struct nvme_ctrl *ctrl);
|
2016-02-24 23:15:56 +07:00
|
|
|
void nvme_kill_queues(struct nvme_ctrl *ctrl);
|
2017-03-02 02:22:12 +07:00
|
|
|
void nvme_unfreeze(struct nvme_ctrl *ctrl);
|
|
|
|
void nvme_wait_freeze(struct nvme_ctrl *ctrl);
|
|
|
|
void nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout);
|
|
|
|
void nvme_start_freeze(struct nvme_ctrl *ctrl);
|
2017-10-11 16:53:07 +07:00
|
|
|
int nvme_reinit_tagset(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set);
|
2015-12-24 21:26:59 +07:00
|
|
|
|
2016-06-13 21:45:23 +07:00
|
|
|
#define NVME_QID_ANY -1
|
2015-11-20 15:00:02 +07:00
|
|
|
struct request *nvme_alloc_request(struct request_queue *q,
|
2017-11-10 01:49:59 +07:00
|
|
|
struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid);
|
2017-06-03 14:38:05 +07:00
|
|
|
blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
|
2016-04-13 02:10:14 +07:00
|
|
|
struct nvme_command *cmd);
|
2015-10-03 20:46:41 +07:00
|
|
|
int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
|
|
|
|
void *buf, unsigned bufflen);
|
|
|
|
int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
|
2016-11-10 22:32:33 +07:00
|
|
|
union nvme_result *result, void *buffer, unsigned bufflen,
|
2017-11-10 01:49:59 +07:00
|
|
|
unsigned timeout, int qid, int at_head,
|
|
|
|
blk_mq_req_flags_t flags);
|
2015-11-26 17:09:06 +07:00
|
|
|
int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count);
|
2016-06-13 21:45:28 +07:00
|
|
|
void nvme_start_keep_alive(struct nvme_ctrl *ctrl);
|
|
|
|
void nvme_stop_keep_alive(struct nvme_ctrl *ctrl);
|
2017-06-15 20:41:08 +07:00
|
|
|
int nvme_reset_ctrl(struct nvme_ctrl *ctrl);
|
2018-01-14 17:39:00 +07:00
|
|
|
int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl);
|
2017-10-29 15:44:29 +07:00
|
|
|
int nvme_delete_ctrl(struct nvme_ctrl *ctrl);
|
|
|
|
int nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl);
|
2015-10-03 20:46:41 +07:00
|
|
|
|
2017-11-09 19:51:03 +07:00
|
|
|
extern const struct attribute_group nvme_ns_id_attr_group;
|
2017-11-02 18:59:30 +07:00
|
|
|
extern const struct block_device_operations nvme_ns_head_ops;
|
|
|
|
|
|
|
|
#ifdef CONFIG_NVME_MULTIPATH
|
|
|
|
void nvme_failover_req(struct request *req);
|
2018-01-10 02:04:15 +07:00
|
|
|
bool nvme_req_needs_failover(struct request *req, blk_status_t error);
|
2017-11-02 18:59:30 +07:00
|
|
|
void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl);
|
|
|
|
int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl,struct nvme_ns_head *head);
|
|
|
|
void nvme_mpath_add_disk(struct nvme_ns_head *head);
|
2017-11-09 23:57:06 +07:00
|
|
|
void nvme_mpath_add_disk_links(struct nvme_ns *ns);
|
2017-11-02 18:59:30 +07:00
|
|
|
void nvme_mpath_remove_disk(struct nvme_ns_head *head);
|
2017-11-09 23:57:06 +07:00
|
|
|
void nvme_mpath_remove_disk_links(struct nvme_ns *ns);
|
2017-11-02 18:59:30 +07:00
|
|
|
|
|
|
|
static inline void nvme_mpath_clear_current_path(struct nvme_ns *ns)
|
|
|
|
{
|
|
|
|
struct nvme_ns_head *head = ns->head;
|
|
|
|
|
|
|
|
if (head && ns == srcu_dereference(head->current_path, &head->srcu))
|
|
|
|
rcu_assign_pointer(head->current_path, NULL);
|
|
|
|
}
|
|
|
|
struct nvme_ns *nvme_find_path(struct nvme_ns_head *head);
|
2017-12-21 20:07:27 +07:00
|
|
|
|
|
|
|
static inline void nvme_mpath_check_last_path(struct nvme_ns *ns)
|
|
|
|
{
|
|
|
|
struct nvme_ns_head *head = ns->head;
|
|
|
|
|
|
|
|
if (head->disk && list_empty(&head->list))
|
|
|
|
kblockd_schedule_work(&head->requeue_work);
|
|
|
|
}
|
|
|
|
|
2017-11-02 18:59:30 +07:00
|
|
|
#else
|
|
|
|
static inline void nvme_failover_req(struct request *req)
|
|
|
|
{
|
|
|
|
}
|
2018-01-10 02:04:15 +07:00
|
|
|
static inline bool nvme_req_needs_failover(struct request *req,
|
|
|
|
blk_status_t error)
|
2017-11-02 18:59:30 +07:00
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
static inline void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
static inline int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl,
|
|
|
|
struct nvme_ns_head *head)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
static inline void nvme_mpath_add_disk(struct nvme_ns_head *head)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
static inline void nvme_mpath_remove_disk(struct nvme_ns_head *head)
|
|
|
|
{
|
|
|
|
}
|
2017-11-09 23:57:06 +07:00
|
|
|
static inline void nvme_mpath_add_disk_links(struct nvme_ns *ns)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
static inline void nvme_mpath_remove_disk_links(struct nvme_ns *ns)
|
|
|
|
{
|
|
|
|
}
|
2017-11-02 18:59:30 +07:00
|
|
|
static inline void nvme_mpath_clear_current_path(struct nvme_ns *ns)
|
2017-12-21 20:07:27 +07:00
|
|
|
{
|
|
|
|
}
|
|
|
|
static inline void nvme_mpath_check_last_path(struct nvme_ns *ns)
|
2017-11-02 18:59:30 +07:00
|
|
|
{
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_NVME_MULTIPATH */
|
|
|
|
|
2015-11-28 22:49:22 +07:00
|
|
|
#ifdef CONFIG_NVM
|
2016-11-29 04:38:53 +07:00
|
|
|
int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, int node);
|
2016-09-16 19:25:07 +07:00
|
|
|
void nvme_nvm_unregister(struct nvme_ns *ns);
|
2016-11-29 04:38:53 +07:00
|
|
|
int nvme_nvm_register_sysfs(struct nvme_ns *ns);
|
|
|
|
void nvme_nvm_unregister_sysfs(struct nvme_ns *ns);
|
2017-01-31 19:17:16 +07:00
|
|
|
int nvme_nvm_ioctl(struct nvme_ns *ns, unsigned int cmd, unsigned long arg);
|
2015-11-28 22:49:22 +07:00
|
|
|
#else
|
2016-09-16 19:25:07 +07:00
|
|
|
static inline int nvme_nvm_register(struct nvme_ns *ns, char *disk_name,
|
2016-11-29 04:38:53 +07:00
|
|
|
int node)
|
2015-11-28 22:49:22 +07:00
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-09-16 19:25:07 +07:00
|
|
|
static inline void nvme_nvm_unregister(struct nvme_ns *ns) {};
|
2016-11-29 04:38:53 +07:00
|
|
|
static inline int nvme_nvm_register_sysfs(struct nvme_ns *ns)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
static inline void nvme_nvm_unregister_sysfs(struct nvme_ns *ns) {};
|
2017-01-31 19:17:16 +07:00
|
|
|
static inline int nvme_nvm_ioctl(struct nvme_ns *ns, unsigned int cmd,
|
|
|
|
unsigned long arg)
|
|
|
|
{
|
|
|
|
return -ENOTTY;
|
|
|
|
}
|
2016-11-29 04:38:53 +07:00
|
|
|
#endif /* CONFIG_NVM */
|
|
|
|
|
2016-09-16 19:25:08 +07:00
|
|
|
static inline struct nvme_ns *nvme_get_ns_from_dev(struct device *dev)
|
|
|
|
{
|
|
|
|
return dev_to_disk(dev)->private_data;
|
|
|
|
}
|
2015-10-29 15:57:29 +07:00
|
|
|
|
2015-11-28 21:39:07 +07:00
|
|
|
int __init nvme_core_init(void);
|
|
|
|
void nvme_core_exit(void);
|
|
|
|
|
2015-10-03 20:46:41 +07:00
|
|
|
#endif /* _NVME_H */
|