projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
nvme: submit internal commands through the block layer
[deliverable/linux.git] / drivers / block / nvme-core.c
CommitLineData
b60503ba
MW		 1/*
2 * NVM Express device driver
6eb0d698 3 * Copyright (c) 2011-2014, Intel Corporation.
b60503ba
MW		 4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
b60503ba
MW		 13 */
14
15#include <linux/nvme.h>
8de05535 16#include <linux/bitops.h>
b60503ba 17#include <linux/blkdev.h>
a4aea562 18#include <linux/blk-mq.h>
42f61420 19#include <linux/cpu.h>
fd63e9ce 20#include <linux/delay.h>
b60503ba
MW		 21#include <linux/errno.h>
22#include <linux/fs.h>
23#include <linux/genhd.h>
4cc09e2d 24#include <linux/hdreg.h>
5aff9382 25#include <linux/idr.h>
b60503ba
MW		 26#include <linux/init.h>
27#include <linux/interrupt.h>
28#include <linux/io.h>
29#include <linux/kdev_t.h>
1fa6aead 30#include <linux/kthread.h>
b60503ba
MW		 31#include <linux/kernel.h>
32#include <linux/mm.h>
33#include <linux/module.h>
34#include <linux/moduleparam.h>
35#include <linux/pci.h>
be7b6275 36#include <linux/poison.h>
c3bfe717 37#include <linux/ptrace.h>
b60503ba
MW		 38#include <linux/sched.h>
39#include <linux/slab.h>
e1e5e564 40#include <linux/t10-pi.h>
b60503ba 41#include <linux/types.h>
5d0f6131 42#include <scsi/sg.h>
797a796a
HM		 43#include <asm-generic/io-64-nonatomic-lo-hi.h>
44
b3fffdef 45#define NVME_MINORS (1U << MINORBITS)
9d43cf64 46#define NVME_Q_DEPTH 1024
d31af0a3 47#define NVME_AQ_DEPTH 256
b60503ba
MW		 48#define SQ_SIZE(depth) (depth * sizeof(struct nvme_command))
49#define CQ_SIZE(depth) (depth * sizeof(struct nvme_completion))
9d43cf64 50#define ADMIN_TIMEOUT (admin_timeout * HZ)
2484f407 51#define SHUTDOWN_TIMEOUT (shutdown_timeout * HZ)
9d43cf64
KB		 52
53static unsigned char admin_timeout = 60;
54module_param(admin_timeout, byte, 0644);
55MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
b60503ba 56
bd67608a
MW		 57unsigned char nvme_io_timeout = 30;
58module_param_named(io_timeout, nvme_io_timeout, byte, 0644);
b355084a 59MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
b60503ba 60
2484f407
DM		 61static unsigned char shutdown_timeout = 5;
62module_param(shutdown_timeout, byte, 0644);
63MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
64
b60503ba
MW		 65static int nvme_major;
66module_param(nvme_major, int, 0);
67
b3fffdef
KB		 68static int nvme_char_major;
69module_param(nvme_char_major, int, 0);
70
58ffacb5
MW		 71static int use_threaded_interrupts;
72module_param(use_threaded_interrupts, int, 0);
73
1fa6aead
MW		 74static DEFINE_SPINLOCK(dev_list_lock);
75static LIST_HEAD(dev_list);
76static struct task_struct *nvme_thread;
9a6b9458 77static struct workqueue_struct *nvme_workq;
b9afca3e 78static wait_queue_head_t nvme_kthread_wait;
1fa6aead 79
b3fffdef
KB		 80static struct class *nvme_class;
81
d4b4ff8e 82static void nvme_reset_failed_dev(struct work_struct *ws);
a4aea562 83static int nvme_process_cq(struct nvme_queue *nvmeq);
d4b4ff8e 84
4d115420
KB		 85struct async_cmd_info {
86 struct kthread_work work;
87 struct kthread_worker *worker;
a4aea562 88 struct request *req;
4d115420
KB		 89 u32 result;
90 int status;
91 void *ctx;
92};
1fa6aead 93
b60503ba
MW		 94/*
95 * An NVM Express queue. Each device has at least two (one for admin
96 * commands and one for I/O commands).
97 */
98struct nvme_queue {
99 struct device *q_dmadev;
091b6092 100 struct nvme_dev *dev;
3193f07b 101 char irqname[24]; /* nvme4294967295-65535\0 */
b60503ba
MW		 102 spinlock_t q_lock;
103 struct nvme_command *sq_cmds;
104 volatile struct nvme_completion *cqes;
105 dma_addr_t sq_dma_addr;
106 dma_addr_t cq_dma_addr;
b60503ba
MW		 107 u32 __iomem *q_db;
108 u16 q_depth;
6222d172 109 s16 cq_vector;
b60503ba
MW		 110 u16 sq_head;
111 u16 sq_tail;
112 u16 cq_head;
c30341dc 113 u16 qid;
e9539f47
MW		 114 u8 cq_phase;
115 u8 cqe_seen;
4d115420 116 struct async_cmd_info cmdinfo;
a4aea562 117 struct blk_mq_hw_ctx *hctx;
b60503ba
MW		 118};
119
120/*
121 * Check we didin't inadvertently grow the command struct
122 */
123static inline void _nvme_check_size(void)
124{
125 BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
126 BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
127 BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
128 BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
129 BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
f8ebf840 130 BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
c30341dc 131 BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
b60503ba
MW		 132 BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
133 BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096);
134 BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096);
135 BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
6ecec745 136 BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
b60503ba
MW		 137}
138
edd10d33 139typedef void (*nvme_completion_fn)(struct nvme_queue *, void *,
c2f5b650
MW		 140 struct nvme_completion *);
141
e85248e5 142struct nvme_cmd_info {
c2f5b650
MW		 143 nvme_completion_fn fn;
144 void *ctx;
c30341dc 145 int aborted;
a4aea562 146 struct nvme_queue *nvmeq;
ac3dd5bd 147 struct nvme_iod iod[0];
e85248e5
MW		 148};
149
ac3dd5bd
JA		 150/*
151 * Max size of iod being embedded in the request payload
152 */
153#define NVME_INT_PAGES 2
154#define NVME_INT_BYTES(dev) (NVME_INT_PAGES * (dev)->page_size)
fda631ff 155#define NVME_INT_MASK 0x01
ac3dd5bd
JA		 156
157/*
158 * Will slightly overestimate the number of pages needed. This is OK
159 * as it only leads to a small amount of wasted memory for the lifetime of
160 * the I/O.
161 */
162static int nvme_npages(unsigned size, struct nvme_dev *dev)
163{
164 unsigned nprps = DIV_ROUND_UP(size + dev->page_size, dev->page_size);
165 return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
166}
167
168static unsigned int nvme_cmd_size(struct nvme_dev *dev)
169{
170 unsigned int ret = sizeof(struct nvme_cmd_info);
171
172 ret += sizeof(struct nvme_iod);
173 ret += sizeof(__le64 *) * nvme_npages(NVME_INT_BYTES(dev), dev);
174 ret += sizeof(struct scatterlist) * NVME_INT_PAGES;
175
176 return ret;
177}
178
a4aea562
MB		 179static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
180 unsigned int hctx_idx)
e85248e5 181{
a4aea562
MB		 182 struct nvme_dev *dev = data;
183 struct nvme_queue *nvmeq = dev->queues[0];
184
185 WARN_ON(nvmeq->hctx);
186 nvmeq->hctx = hctx;
187 hctx->driver_data = nvmeq;
188 return 0;
e85248e5
MW		 189}
190
a4aea562
MB		 191static int nvme_admin_init_request(void *data, struct request *req,
192 unsigned int hctx_idx, unsigned int rq_idx,
193 unsigned int numa_node)
22404274 194{
a4aea562
MB		 195 struct nvme_dev *dev = data;
196 struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
197 struct nvme_queue *nvmeq = dev->queues[0];
198
199 BUG_ON(!nvmeq);
200 cmd->nvmeq = nvmeq;
201 return 0;
22404274
KB		 202}
203
2c30540b
JA		 204static void nvme_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
205{
206 struct nvme_queue *nvmeq = hctx->driver_data;
207
208 nvmeq->hctx = NULL;
209}
210
a4aea562
MB		 211static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
212 unsigned int hctx_idx)
b60503ba 213{
a4aea562
MB		 214 struct nvme_dev *dev = data;
215 struct nvme_queue *nvmeq = dev->queues[
216 (hctx_idx % dev->queue_count) + 1];
b60503ba 217
a4aea562
MB		 218 if (!nvmeq->hctx)
219 nvmeq->hctx = hctx;
220
221 /* nvmeq queues are shared between namespaces. We assume here that
222 * blk-mq map the tags so they match up with the nvme queue tags. */
223 WARN_ON(nvmeq->hctx->tags != hctx->tags);
b60503ba 224
a4aea562
MB		 225 hctx->driver_data = nvmeq;
226 return 0;
b60503ba
MW		 227}
228
a4aea562
MB		 229static int nvme_init_request(void *data, struct request *req,
230 unsigned int hctx_idx, unsigned int rq_idx,
231 unsigned int numa_node)
b60503ba 232{
a4aea562
MB		 233 struct nvme_dev *dev = data;
234 struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
235 struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1];
236
237 BUG_ON(!nvmeq);
238 cmd->nvmeq = nvmeq;
239 return 0;
240}
241
242static void nvme_set_info(struct nvme_cmd_info *cmd, void *ctx,
243 nvme_completion_fn handler)
244{
245 cmd->fn = handler;
246 cmd->ctx = ctx;
247 cmd->aborted = 0;
c917dfe5 248 blk_mq_start_request(blk_mq_rq_from_pdu(cmd));
b60503ba
MW		 249}
250
ac3dd5bd
JA		 251static void *iod_get_private(struct nvme_iod *iod)
252{
253 return (void *) (iod->private & ~0x1UL);
254}
255
256/*
257 * If bit 0 is set, the iod is embedded in the request payload.
258 */
259static bool iod_should_kfree(struct nvme_iod *iod)
260{
fda631ff 261 return (iod->private & NVME_INT_MASK) == 0;
ac3dd5bd
JA		 262}
263
c2f5b650
MW		 264/* Special values must be less than 0x1000 */
265#define CMD_CTX_BASE ((void *)POISON_POINTER_DELTA)
d2d87034
MW		 266#define CMD_CTX_CANCELLED (0x30C + CMD_CTX_BASE)
267#define CMD_CTX_COMPLETED (0x310 + CMD_CTX_BASE)
268#define CMD_CTX_INVALID (0x314 + CMD_CTX_BASE)
be7b6275 269
edd10d33 270static void special_completion(struct nvme_queue *nvmeq, void *ctx,
c2f5b650
MW		 271 struct nvme_completion *cqe)
272{
273 if (ctx == CMD_CTX_CANCELLED)
274 return;
c2f5b650 275 if (ctx == CMD_CTX_COMPLETED) {
edd10d33 276 dev_warn(nvmeq->q_dmadev,
c2f5b650
MW		 277 "completed id %d twice on queue %d\n",
278 cqe->command_id, le16_to_cpup(&cqe->sq_id));
279 return;
280 }
281 if (ctx == CMD_CTX_INVALID) {
edd10d33 282 dev_warn(nvmeq->q_dmadev,
c2f5b650
MW		 283 "invalid id %d completed on queue %d\n",
284 cqe->command_id, le16_to_cpup(&cqe->sq_id));
285 return;
286 }
edd10d33 287 dev_warn(nvmeq->q_dmadev, "Unknown special completion %p\n", ctx);
c2f5b650
MW		 288}
289
a4aea562 290static void *cancel_cmd_info(struct nvme_cmd_info *cmd, nvme_completion_fn *fn)
b60503ba 291{
c2f5b650 292 void *ctx;
b60503ba 293
859361a2 294 if (fn)
a4aea562
MB		 295 *fn = cmd->fn;
296 ctx = cmd->ctx;
297 cmd->fn = special_completion;
298 cmd->ctx = CMD_CTX_CANCELLED;
c2f5b650 299 return ctx;
b60503ba
MW		 300}
301
a4aea562
MB		 302static void async_req_completion(struct nvme_queue *nvmeq, void *ctx,
303 struct nvme_completion *cqe)
3c0cf138 304{
a4aea562
MB		 305 u32 result = le32_to_cpup(&cqe->result);
306 u16 status = le16_to_cpup(&cqe->status) >> 1;
307
308 if (status == NVME_SC_SUCCESS || status == NVME_SC_ABORT_REQ)
309 ++nvmeq->dev->event_limit;
310 if (status == NVME_SC_SUCCESS)
311 dev_warn(nvmeq->q_dmadev,
312 "async event result %08x\n", result);
b60503ba
MW		 313}
314
a4aea562
MB		 315static void abort_completion(struct nvme_queue *nvmeq, void *ctx,
316 struct nvme_completion *cqe)
5a92e700 317{
a4aea562
MB		 318 struct request *req = ctx;
319
320 u16 status = le16_to_cpup(&cqe->status) >> 1;
321 u32 result = le32_to_cpup(&cqe->result);
a51afb54 322
9d135bb8 323 blk_mq_free_hctx_request(nvmeq->hctx, req);
a51afb54 324
a4aea562
MB		 325 dev_warn(nvmeq->q_dmadev, "Abort status:%x result:%x", status, result);
326 ++nvmeq->dev->abort_limit;
5a92e700
KB		 327}
328
a4aea562
MB		 329static void async_completion(struct nvme_queue *nvmeq, void *ctx,
330 struct nvme_completion *cqe)
b60503ba 331{
a4aea562
MB		 332 struct async_cmd_info *cmdinfo = ctx;
333 cmdinfo->result = le32_to_cpup(&cqe->result);
334 cmdinfo->status = le16_to_cpup(&cqe->status) >> 1;
335 queue_kthread_work(cmdinfo->worker, &cmdinfo->work);
9d135bb8 336 blk_mq_free_hctx_request(nvmeq->hctx, cmdinfo->req);
b60503ba
MW		 337}
338
a4aea562
MB		 339static inline struct nvme_cmd_info *get_cmd_from_tag(struct nvme_queue *nvmeq,
340 unsigned int tag)
b60503ba 341{
a4aea562
MB		 342 struct blk_mq_hw_ctx *hctx = nvmeq->hctx;
343 struct request *req = blk_mq_tag_to_rq(hctx->tags, tag);
a51afb54 344
a4aea562 345 return blk_mq_rq_to_pdu(req);
4f5099af
KB		 346}
347
a4aea562
MB		 348/*
349 * Called with local interrupts disabled and the q_lock held. May not sleep.
350 */
351static void *nvme_finish_cmd(struct nvme_queue *nvmeq, int tag,
352 nvme_completion_fn *fn)
4f5099af 353{
a4aea562
MB		 354 struct nvme_cmd_info *cmd = get_cmd_from_tag(nvmeq, tag);
355 void *ctx;
356 if (tag >= nvmeq->q_depth) {
357 *fn = special_completion;
358 return CMD_CTX_INVALID;
359 }
360 if (fn)
361 *fn = cmd->fn;
362 ctx = cmd->ctx;
363 cmd->fn = special_completion;
364 cmd->ctx = CMD_CTX_COMPLETED;
365 return ctx;
b60503ba
MW		 366}
367
368/**
714a7a22 369 * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
b60503ba
MW		 370 * @nvmeq: The queue to use
371 * @cmd: The command to send
372 *
373 * Safe to use from interrupt context
374 */
a4aea562 375static int __nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
b60503ba 376{
a4aea562
MB		 377 u16 tail = nvmeq->sq_tail;
378
b60503ba 379 memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));
b60503ba
MW		 380 if (++tail == nvmeq->q_depth)
381 tail = 0;
7547881d 382 writel(tail, nvmeq->q_db);
b60503ba 383 nvmeq->sq_tail = tail;
b60503ba
MW		 384
385 return 0;
386}
387
a4aea562
MB		 388static int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
389{
390 unsigned long flags;
391 int ret;
392 spin_lock_irqsave(&nvmeq->q_lock, flags);
393 ret = __nvme_submit_cmd(nvmeq, cmd);
394 spin_unlock_irqrestore(&nvmeq->q_lock, flags);
395 return ret;
396}
397
eca18b23 398static __le64 **iod_list(struct nvme_iod *iod)
e025344c 399{
eca18b23 400 return ((void *)iod) + iod->offset;
e025344c
SMM		 401}
402
ac3dd5bd
JA		 403static inline void iod_init(struct nvme_iod *iod, unsigned nbytes,
404 unsigned nseg, unsigned long private)
eca18b23 405{
ac3dd5bd
JA		 406 iod->private = private;
407 iod->offset = offsetof(struct nvme_iod, sg[nseg]);
408 iod->npages = -1;
409 iod->length = nbytes;
410 iod->nents = 0;
eca18b23 411}
b60503ba 412
eca18b23 413static struct nvme_iod *
ac3dd5bd
JA		 414__nvme_alloc_iod(unsigned nseg, unsigned bytes, struct nvme_dev *dev,
415 unsigned long priv, gfp_t gfp)
b60503ba 416{
eca18b23 417 struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) +
ac3dd5bd 418 sizeof(__le64 *) * nvme_npages(bytes, dev) +
eca18b23
MW		 419 sizeof(struct scatterlist) * nseg, gfp);
420
ac3dd5bd
JA		 421 if (iod)
422 iod_init(iod, bytes, nseg, priv);
eca18b23
MW		 423
424 return iod;
b60503ba
MW		 425}
426
ac3dd5bd
JA		 427static struct nvme_iod *nvme_alloc_iod(struct request *rq, struct nvme_dev *dev,
428 gfp_t gfp)
429{
430 unsigned size = !(rq->cmd_flags & REQ_DISCARD) ? blk_rq_bytes(rq) :
431 sizeof(struct nvme_dsm_range);
ac3dd5bd
JA		 432 struct nvme_iod *iod;
433
434 if (rq->nr_phys_segments <= NVME_INT_PAGES &&
435 size <= NVME_INT_BYTES(dev)) {
436 struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(rq);
437
438 iod = cmd->iod;
ac3dd5bd 439 iod_init(iod, size, rq->nr_phys_segments,
fda631ff 440 (unsigned long) rq | NVME_INT_MASK);
ac3dd5bd
JA		 441 return iod;
442 }
443
444 return __nvme_alloc_iod(rq->nr_phys_segments, size, dev,
445 (unsigned long) rq, gfp);
446}
447
d29ec824 448static void nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod)
b60503ba 449{
1d090624 450 const int last_prp = dev->page_size / 8 - 1;
eca18b23
MW		 451 int i;
452 __le64 **list = iod_list(iod);
453 dma_addr_t prp_dma = iod->first_dma;
454
455 if (iod->npages == 0)
456 dma_pool_free(dev->prp_small_pool, list[0], prp_dma);
457 for (i = 0; i < iod->npages; i++) {
458 __le64 *prp_list = list[i];
459 dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]);
460 dma_pool_free(dev->prp_page_pool, prp_list, prp_dma);
461 prp_dma = next_prp_dma;
462 }
ac3dd5bd
JA		 463
464 if (iod_should_kfree(iod))
465 kfree(iod);
b60503ba
MW		 466}
467
b4ff9c8d
KB		 468static int nvme_error_status(u16 status)
469{
470 switch (status & 0x7ff) {
471 case NVME_SC_SUCCESS:
472 return 0;
473 case NVME_SC_CAP_EXCEEDED:
474 return -ENOSPC;
475 default:
476 return -EIO;
477 }
478}
479
52b68d7e 480#ifdef CONFIG_BLK_DEV_INTEGRITY
e1e5e564
KB		 481static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
482{
483 if (be32_to_cpu(pi->ref_tag) == v)
484 pi->ref_tag = cpu_to_be32(p);
485}
486
487static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
488{
489 if (be32_to_cpu(pi->ref_tag) == p)
490 pi->ref_tag = cpu_to_be32(v);
491}
492
493/**
494 * nvme_dif_remap - remaps ref tags to bip seed and physical lba
495 *
496 * The virtual start sector is the one that was originally submitted by the
497 * block layer. Due to partitioning, MD/DM cloning, etc. the actual physical
498 * start sector may be different. Remap protection information to match the
499 * physical LBA on writes, and back to the original seed on reads.
500 *
501 * Type 0 and 3 do not have a ref tag, so no remapping required.
502 */
503static void nvme_dif_remap(struct request *req,
504 void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
505{
506 struct nvme_ns *ns = req->rq_disk->private_data;
507 struct bio_integrity_payload *bip;
508 struct t10_pi_tuple *pi;
509 void *p, *pmap;
510 u32 i, nlb, ts, phys, virt;
511
512 if (!ns->pi_type || ns->pi_type == NVME_NS_DPS_PI_TYPE3)
513 return;
514
515 bip = bio_integrity(req->bio);
516 if (!bip)
517 return;
518
519 pmap = kmap_atomic(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset;
e1e5e564
KB		 520
521 p = pmap;
522 virt = bip_get_seed(bip);
523 phys = nvme_block_nr(ns, blk_rq_pos(req));
524 nlb = (blk_rq_bytes(req) >> ns->lba_shift);
525 ts = ns->disk->integrity->tuple_size;
526
527 for (i = 0; i < nlb; i++, virt++, phys++) {
528 pi = (struct t10_pi_tuple *)p;
529 dif_swap(phys, virt, pi);
530 p += ts;
531 }
532 kunmap_atomic(pmap);
533}
534
52b68d7e
KB		 535static int nvme_noop_verify(struct blk_integrity_iter *iter)
536{
537 return 0;
538}
539
540static int nvme_noop_generate(struct blk_integrity_iter *iter)
541{
542 return 0;
543}
544
545struct blk_integrity nvme_meta_noop = {
546 .name = "NVME_META_NOOP",
547 .generate_fn = nvme_noop_generate,
548 .verify_fn = nvme_noop_verify,
549};
550
551static void nvme_init_integrity(struct nvme_ns *ns)
552{
553 struct blk_integrity integrity;
554
555 switch (ns->pi_type) {
556 case NVME_NS_DPS_PI_TYPE3:
557 integrity = t10_pi_type3_crc;
558 break;
559 case NVME_NS_DPS_PI_TYPE1:
560 case NVME_NS_DPS_PI_TYPE2:
561 integrity = t10_pi_type1_crc;
562 break;
563 default:
564 integrity = nvme_meta_noop;
565 break;
566 }
567 integrity.tuple_size = ns->ms;
568 blk_integrity_register(ns->disk, &integrity);
569 blk_queue_max_integrity_segments(ns->queue, 1);
570}
571#else /* CONFIG_BLK_DEV_INTEGRITY */
572static void nvme_dif_remap(struct request *req,
573 void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
574{
575}
576static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
577{
578}
579static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
580{
581}
582static void nvme_init_integrity(struct nvme_ns *ns)
583{
584}
585#endif
586
a4aea562 587static void req_completion(struct nvme_queue *nvmeq, void *ctx,
b60503ba
MW		 588 struct nvme_completion *cqe)
589{
eca18b23 590 struct nvme_iod *iod = ctx;
ac3dd5bd 591 struct request *req = iod_get_private(iod);
a4aea562
MB		 592 struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
593
b60503ba
MW		 594 u16 status = le16_to_cpup(&cqe->status) >> 1;
595
edd10d33 596 if (unlikely(status)) {
a4aea562
MB		 597 if (!(status & NVME_SC_DNR || blk_noretry_request(req))
598 && (jiffies - req->start_time) < req->timeout) {
c9d3bf88
KB		 599 unsigned long flags;
600
a4aea562 601 blk_mq_requeue_request(req);
c9d3bf88
KB		 602 spin_lock_irqsave(req->q->queue_lock, flags);
603 if (!blk_queue_stopped(req->q))
604 blk_mq_kick_requeue_list(req->q);
605 spin_unlock_irqrestore(req->q->queue_lock, flags);
edd10d33
KB		 606 return;
607 }
d29ec824
CH		 608 if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
609 req->sense_len = le32_to_cpup(&cqe->result);
610 req->errors = status;
611 } else {
612 req->errors = nvme_error_status(status);
613 }
a4aea562
MB		 614 } else
615 req->errors = 0;
616
617 if (cmd_rq->aborted)
e75ec752 618 dev_warn(nvmeq->dev->dev,
a4aea562
MB		 619 "completing aborted command with status:%04x\n",
620 status);
621
e1e5e564 622 if (iod->nents) {
e75ec752 623 dma_unmap_sg(nvmeq->dev->dev, iod->sg, iod->nents,
a4aea562 624 rq_data_dir(req) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
e1e5e564
KB		 625 if (blk_integrity_rq(req)) {
626 if (!rq_data_dir(req))
627 nvme_dif_remap(req, nvme_dif_complete);
e75ec752 628 dma_unmap_sg(nvmeq->dev->dev, iod->meta_sg, 1,
e1e5e564
KB		 629 rq_data_dir(req) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
630 }
631 }
edd10d33 632 nvme_free_iod(nvmeq->dev, iod);
3291fa57 633
a4aea562 634 blk_mq_complete_request(req);
b60503ba
MW		 635}
636
184d2944 637/* length is in bytes. gfp flags indicates whether we may sleep. */
d29ec824
CH		 638static int nvme_setup_prps(struct nvme_dev *dev, struct nvme_iod *iod,
639 int total_len, gfp_t gfp)
ff22b54f 640{
99802a7a 641 struct dma_pool *pool;
eca18b23
MW		 642 int length = total_len;
643 struct scatterlist *sg = iod->sg;
ff22b54f
MW		 644 int dma_len = sg_dma_len(sg);
645 u64 dma_addr = sg_dma_address(sg);
f137e0f1
MI		 646 u32 page_size = dev->page_size;
647 int offset = dma_addr & (page_size - 1);
e025344c 648 __le64 *prp_list;
eca18b23 649 __le64 **list = iod_list(iod);
e025344c 650 dma_addr_t prp_dma;
eca18b23 651 int nprps, i;
ff22b54f 652
1d090624 653 length -= (page_size - offset);
ff22b54f 654 if (length <= 0)
eca18b23 655 return total_len;
ff22b54f 656
1d090624 657 dma_len -= (page_size - offset);
ff22b54f 658 if (dma_len) {
1d090624 659 dma_addr += (page_size - offset);
ff22b54f
MW		 660 } else {
661 sg = sg_next(sg);
662 dma_addr = sg_dma_address(sg);
663 dma_len = sg_dma_len(sg);
664 }
665
1d090624 666 if (length <= page_size) {
edd10d33 667 iod->first_dma = dma_addr;
eca18b23 668 return total_len;
e025344c
SMM		 669 }
670
1d090624 671 nprps = DIV_ROUND_UP(length, page_size);
99802a7a
MW		 672 if (nprps <= (256 / 8)) {
673 pool = dev->prp_small_pool;
eca18b23 674 iod->npages = 0;
99802a7a
MW		 675 } else {
676 pool = dev->prp_page_pool;
eca18b23 677 iod->npages = 1;
99802a7a
MW		 678 }
679
b77954cb
MW		 680 prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
681 if (!prp_list) {
edd10d33 682 iod->first_dma = dma_addr;
eca18b23 683 iod->npages = -1;
1d090624 684 return (total_len - length) + page_size;
b77954cb 685 }
eca18b23
MW		 686 list[0] = prp_list;
687 iod->first_dma = prp_dma;
e025344c
SMM		 688 i = 0;
689 for (;;) {
1d090624 690 if (i == page_size >> 3) {
e025344c 691 __le64 *old_prp_list = prp_list;
b77954cb 692 prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
eca18b23
MW		 693 if (!prp_list)
694 return total_len - length;
695 list[iod->npages++] = prp_list;
7523d834
MW		 696 prp_list[0] = old_prp_list[i - 1];
697 old_prp_list[i - 1] = cpu_to_le64(prp_dma);
698 i = 1;
e025344c
SMM		 699 }
700 prp_list[i++] = cpu_to_le64(dma_addr);
1d090624
KB		 701 dma_len -= page_size;
702 dma_addr += page_size;
703 length -= page_size;
e025344c
SMM		 704 if (length <= 0)
705 break;
706 if (dma_len > 0)
707 continue;
708 BUG_ON(dma_len < 0);
709 sg = sg_next(sg);
710 dma_addr = sg_dma_address(sg);
711 dma_len = sg_dma_len(sg);
ff22b54f
MW		 712 }
713
eca18b23 714 return total_len;
ff22b54f
MW		 715}
716
d29ec824
CH		 717static void nvme_submit_priv(struct nvme_queue *nvmeq, struct request *req,
718 struct nvme_iod *iod)
719{
720 struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
721
722 memcpy(cmnd, req->cmd, sizeof(struct nvme_command));
723 cmnd->rw.command_id = req->tag;
724 if (req->nr_phys_segments) {
725 cmnd->rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
726 cmnd->rw.prp2 = cpu_to_le64(iod->first_dma);
727 }
728
729 if (++nvmeq->sq_tail == nvmeq->q_depth)
730 nvmeq->sq_tail = 0;
731 writel(nvmeq->sq_tail, nvmeq->q_db);
732}
733
a4aea562
MB		 734/*
735 * We reuse the small pool to allocate the 16-byte range here as it is not
736 * worth having a special pool for these or additional cases to handle freeing
737 * the iod.
738 */
739static void nvme_submit_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns,
740 struct request *req, struct nvme_iod *iod)
0e5e4f0e 741{
edd10d33
KB		 742 struct nvme_dsm_range *range =
743 (struct nvme_dsm_range *)iod_list(iod)[0];
0e5e4f0e
KB		 744 struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
745
0e5e4f0e 746 range->cattr = cpu_to_le32(0);
a4aea562
MB		 747 range->nlb = cpu_to_le32(blk_rq_bytes(req) >> ns->lba_shift);
748 range->slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
0e5e4f0e
KB		 749
750 memset(cmnd, 0, sizeof(*cmnd));
751 cmnd->dsm.opcode = nvme_cmd_dsm;
a4aea562 752 cmnd->dsm.command_id = req->tag;
0e5e4f0e
KB		 753 cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
754 cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma);
755 cmnd->dsm.nr = 0;
756 cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
757
758 if (++nvmeq->sq_tail == nvmeq->q_depth)
759 nvmeq->sq_tail = 0;
760 writel(nvmeq->sq_tail, nvmeq->q_db);
0e5e4f0e
KB		 761}
762
a4aea562 763static void nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns,
00df5cb4
MW		 764 int cmdid)
765{
766 struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
767
768 memset(cmnd, 0, sizeof(*cmnd));
769 cmnd->common.opcode = nvme_cmd_flush;
770 cmnd->common.command_id = cmdid;
771 cmnd->common.nsid = cpu_to_le32(ns->ns_id);
772
773 if (++nvmeq->sq_tail == nvmeq->q_depth)
774 nvmeq->sq_tail = 0;
775 writel(nvmeq->sq_tail, nvmeq->q_db);
00df5cb4
MW		 776}
777
a4aea562
MB		 778static int nvme_submit_iod(struct nvme_queue *nvmeq, struct nvme_iod *iod,
779 struct nvme_ns *ns)
b60503ba 780{
ac3dd5bd 781 struct request *req = iod_get_private(iod);
ff22b54f 782 struct nvme_command *cmnd;
a4aea562
MB		 783 u16 control = 0;
784 u32 dsmgmt = 0;
00df5cb4 785
a4aea562 786 if (req->cmd_flags & REQ_FUA)
b60503ba 787 control |= NVME_RW_FUA;
a4aea562 788 if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
b60503ba
MW		 789 control |= NVME_RW_LR;
790
a4aea562 791 if (req->cmd_flags & REQ_RAHEAD)
b60503ba
MW		 792 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
793
ff22b54f 794 cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
b8deb62c 795 memset(cmnd, 0, sizeof(*cmnd));
b60503ba 796
a4aea562
MB		 797 cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);
798 cmnd->rw.command_id = req->tag;
ff22b54f 799 cmnd->rw.nsid = cpu_to_le32(ns->ns_id);
edd10d33
KB		 800 cmnd->rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
801 cmnd->rw.prp2 = cpu_to_le64(iod->first_dma);
a4aea562
MB		 802 cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
803 cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
e1e5e564
KB		 804
805 if (blk_integrity_rq(req)) {
806 cmnd->rw.metadata = cpu_to_le64(sg_dma_address(iod->meta_sg));
807 switch (ns->pi_type) {
808 case NVME_NS_DPS_PI_TYPE3:
809 control |= NVME_RW_PRINFO_PRCHK_GUARD;
810 break;
811 case NVME_NS_DPS_PI_TYPE1:
812 case NVME_NS_DPS_PI_TYPE2:
813 control |= NVME_RW_PRINFO_PRCHK_GUARD |
814 NVME_RW_PRINFO_PRCHK_REF;
815 cmnd->rw.reftag = cpu_to_le32(
816 nvme_block_nr(ns, blk_rq_pos(req)));
817 break;
818 }
819 } else if (ns->ms)
820 control |= NVME_RW_PRINFO_PRACT;
821
ff22b54f
MW		 822 cmnd->rw.control = cpu_to_le16(control);
823 cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
b60503ba 824
b60503ba
MW		 825 if (++nvmeq->sq_tail == nvmeq->q_depth)
826 nvmeq->sq_tail = 0;
7547881d 827 writel(nvmeq->sq_tail, nvmeq->q_db);
b60503ba 828
1974b1ae 829 return 0;
edd10d33
KB		 830}
831
d29ec824
CH		 832/*
833 * NOTE: ns is NULL when called on the admin queue.
834 */
a4aea562
MB		 835static int nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
836 const struct blk_mq_queue_data *bd)
edd10d33 837{
a4aea562
MB		 838 struct nvme_ns *ns = hctx->queue->queuedata;
839 struct nvme_queue *nvmeq = hctx->driver_data;
d29ec824 840 struct nvme_dev *dev = nvmeq->dev;
a4aea562
MB		 841 struct request *req = bd->rq;
842 struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
edd10d33 843 struct nvme_iod *iod;
a4aea562 844 enum dma_data_direction dma_dir;
edd10d33 845
e1e5e564
KB		 846 /*
847 * If formated with metadata, require the block layer provide a buffer
848 * unless this namespace is formated such that the metadata can be
849 * stripped/generated by the controller with PRACT=1.
850 */
d29ec824 851 if (ns && ns->ms && !blk_integrity_rq(req)) {
e1e5e564
KB		 852 if (!(ns->pi_type && ns->ms == 8)) {
853 req->errors = -EFAULT;
854 blk_mq_complete_request(req);
855 return BLK_MQ_RQ_QUEUE_OK;
856 }
857 }
858
d29ec824 859 iod = nvme_alloc_iod(req, dev, GFP_ATOMIC);
edd10d33 860 if (!iod)
fe54303e 861 return BLK_MQ_RQ_QUEUE_BUSY;
a4aea562 862
a4aea562 863 if (req->cmd_flags & REQ_DISCARD) {
edd10d33
KB		 864 void *range;
865 /*
866 * We reuse the small pool to allocate the 16-byte range here
867 * as it is not worth having a special pool for these or
868 * additional cases to handle freeing the iod.
869 */
d29ec824 870 range = dma_pool_alloc(dev->prp_small_pool, GFP_ATOMIC,
edd10d33 871 &iod->first_dma);
a4aea562 872 if (!range)
fe54303e 873 goto retry_cmd;
edd10d33
KB		 874 iod_list(iod)[0] = (__le64 *)range;
875 iod->npages = 0;
ac3dd5bd 876 } else if (req->nr_phys_segments) {
a4aea562
MB		 877 dma_dir = rq_data_dir(req) ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
878
ac3dd5bd 879 sg_init_table(iod->sg, req->nr_phys_segments);
a4aea562 880 iod->nents = blk_rq_map_sg(req->q, req, iod->sg);
fe54303e
JA		 881 if (!iod->nents)
882 goto error_cmd;
a4aea562
MB		 883
884 if (!dma_map_sg(nvmeq->q_dmadev, iod->sg, iod->nents, dma_dir))
fe54303e 885 goto retry_cmd;
a4aea562 886
fe54303e 887 if (blk_rq_bytes(req) !=
d29ec824
CH		 888 nvme_setup_prps(dev, iod, blk_rq_bytes(req), GFP_ATOMIC)) {
889 dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
fe54303e
JA		 890 goto retry_cmd;
891 }
e1e5e564
KB		 892 if (blk_integrity_rq(req)) {
893 if (blk_rq_count_integrity_sg(req->q, req->bio) != 1)
894 goto error_cmd;
895
896 sg_init_table(iod->meta_sg, 1);
897 if (blk_rq_map_integrity_sg(
898 req->q, req->bio, iod->meta_sg) != 1)
899 goto error_cmd;
900
901 if (rq_data_dir(req))
902 nvme_dif_remap(req, nvme_dif_prep);
903
904 if (!dma_map_sg(nvmeq->q_dmadev, iod->meta_sg, 1, dma_dir))
905 goto error_cmd;
906 }
edd10d33 907 }
1974b1ae 908
9af8785a 909 nvme_set_info(cmd, iod, req_completion);
a4aea562 910 spin_lock_irq(&nvmeq->q_lock);
d29ec824
CH		 911 if (req->cmd_type == REQ_TYPE_DRV_PRIV)
912 nvme_submit_priv(nvmeq, req, iod);
913 else if (req->cmd_flags & REQ_DISCARD)
a4aea562
MB		 914 nvme_submit_discard(nvmeq, ns, req, iod);
915 else if (req->cmd_flags & REQ_FLUSH)
916 nvme_submit_flush(nvmeq, ns, req->tag);
917 else
918 nvme_submit_iod(nvmeq, iod, ns);
919
920 nvme_process_cq(nvmeq);
921 spin_unlock_irq(&nvmeq->q_lock);
922 return BLK_MQ_RQ_QUEUE_OK;
923
fe54303e 924 error_cmd:
d29ec824 925 nvme_free_iod(dev, iod);
fe54303e
JA		 926 return BLK_MQ_RQ_QUEUE_ERROR;
927 retry_cmd:
d29ec824 928 nvme_free_iod(dev, iod);
fe54303e 929 return BLK_MQ_RQ_QUEUE_BUSY;
b60503ba
MW		 930}
931
e9539f47 932static int nvme_process_cq(struct nvme_queue *nvmeq)
b60503ba 933{
82123460 934 u16 head, phase;
b60503ba 935
b60503ba 936 head = nvmeq->cq_head;
82123460 937 phase = nvmeq->cq_phase;
b60503ba
MW		 938
939 for (;;) {
c2f5b650
MW		 940 void *ctx;
941 nvme_completion_fn fn;
b60503ba 942 struct nvme_completion cqe = nvmeq->cqes[head];
82123460 943 if ((le16_to_cpu(cqe.status) & 1) != phase)
b60503ba
MW		 944 break;
945 nvmeq->sq_head = le16_to_cpu(cqe.sq_head);
946 if (++head == nvmeq->q_depth) {
947 head = 0;
82123460 948 phase = !phase;
b60503ba 949 }
a4aea562 950 ctx = nvme_finish_cmd(nvmeq, cqe.command_id, &fn);
edd10d33 951 fn(nvmeq, ctx, &cqe);
b60503ba
MW		 952 }
953
954 /* If the controller ignores the cq head doorbell and continuously
955 * writes to the queue, it is theoretically possible to wrap around
956 * the queue twice and mistakenly return IRQ_NONE. Linux only
957 * requires that 0.1% of your interrupts are handled, so this isn't
958 * a big problem.
959 */
82123460 960 if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
e9539f47 961 return 0;
b60503ba 962
b80d5ccc 963 writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
b60503ba 964 nvmeq->cq_head = head;
82123460 965 nvmeq->cq_phase = phase;
b60503ba 966
e9539f47
MW		 967 nvmeq->cqe_seen = 1;
968 return 1;
b60503ba
MW		 969}
970
971static irqreturn_t nvme_irq(int irq, void *data)
58ffacb5
MW		 972{
973 irqreturn_t result;
974 struct nvme_queue *nvmeq = data;
975 spin_lock(&nvmeq->q_lock);
e9539f47
MW		 976 nvme_process_cq(nvmeq);
977 result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE;
978 nvmeq->cqe_seen = 0;
58ffacb5
MW		 979 spin_unlock(&nvmeq->q_lock);
980 return result;
981}
982
983static irqreturn_t nvme_irq_check(int irq, void *data)
984{
985 struct nvme_queue *nvmeq = data;
986 struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head];
987 if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase)
988 return IRQ_NONE;
989 return IRQ_WAKE_THREAD;
990}
991
b60503ba
MW		 992/*
993 * Returns 0 on success. If the result is negative, it's a Linux error code;
994 * if the result is positive, it's an NVM Express status code
995 */
d29ec824
CH		 996int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
997 void *buffer, void __user *ubuffer, unsigned bufflen,
998 u32 *result, unsigned timeout)
b60503ba 999{
d29ec824
CH		 1000 bool write = cmd->common.opcode & 1;
1001 struct bio *bio = NULL;
f705f837 1002 struct request *req;
d29ec824 1003 int ret;
f705f837 1004
d29ec824 1005 req = blk_mq_alloc_request(q, write, GFP_KERNEL, false);
f705f837
CH		 1006 if (IS_ERR(req))
1007 return PTR_ERR(req);
b60503ba 1008
d29ec824
CH		 1009 req->cmd_type = REQ_TYPE_DRV_PRIV;
1010 req->__data_len = 0;
1011 req->__sector = (sector_t) -1;
1012 req->bio = req->biotail = NULL;
b60503ba 1013
d29ec824 1014 req->timeout = ADMIN_TIMEOUT;
a4aea562 1015
d29ec824
CH		 1016 req->cmd = (unsigned char *)cmd;
1017 req->cmd_len = sizeof(struct nvme_command);
1018 req->sense = NULL;
1019 req->sense_len = 0;
b60503ba 1020
d29ec824
CH		 1021 if (buffer && bufflen) {
1022 ret = blk_rq_map_kern(q, req, buffer, bufflen, __GFP_WAIT);
1023 if (ret)
1024 goto out;
1025 } else if (ubuffer && bufflen) {
1026 ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen, __GFP_WAIT);
1027 if (ret)
1028 goto out;
1029 bio = req->bio;
1030 }
3c0cf138 1031
d29ec824
CH		 1032 blk_execute_rq(req->q, NULL, req, 0);
1033 if (bio)
1034 blk_rq_unmap_user(bio);
b60503ba 1035 if (result)
d29ec824
CH		 1036 *result = req->sense_len;
1037 ret = req->errors;
1038 out:
f705f837 1039 blk_mq_free_request(req);
d29ec824 1040 return ret;
f705f837
CH		 1041}
1042
d29ec824
CH		 1043int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
1044 void *buffer, unsigned bufflen)
f705f837 1045{
d29ec824 1046 return __nvme_submit_sync_cmd(q, cmd, buffer, NULL, bufflen, NULL, 0);
b60503ba
MW		 1047}
1048
a4aea562
MB		 1049static int nvme_submit_async_admin_req(struct nvme_dev *dev)
1050{
1051 struct nvme_queue *nvmeq = dev->queues[0];
1052 struct nvme_command c;
1053 struct nvme_cmd_info *cmd_info;
1054 struct request *req;
1055
1efccc9d 1056 req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_ATOMIC, true);
9f173b33
DC		 1057 if (IS_ERR(req))
1058 return PTR_ERR(req);
a4aea562 1059
c917dfe5 1060 req->cmd_flags |= REQ_NO_TIMEOUT;
a4aea562 1061 cmd_info = blk_mq_rq_to_pdu(req);
1efccc9d 1062 nvme_set_info(cmd_info, NULL, async_req_completion);
a4aea562
MB		 1063
1064 memset(&c, 0, sizeof(c));
1065 c.common.opcode = nvme_admin_async_event;
1066 c.common.command_id = req->tag;
1067
1efccc9d 1068 blk_mq_free_hctx_request(nvmeq->hctx, req);
a4aea562
MB		 1069 return __nvme_submit_cmd(nvmeq, &c);
1070}
1071
1072static int nvme_submit_admin_async_cmd(struct nvme_dev *dev,
4d115420
KB		 1073 struct nvme_command *cmd,
1074 struct async_cmd_info *cmdinfo, unsigned timeout)
1075{
a4aea562
MB		 1076 struct nvme_queue *nvmeq = dev->queues[0];
1077 struct request *req;
1078 struct nvme_cmd_info *cmd_rq;
4d115420 1079
a4aea562 1080 req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_KERNEL, false);
9f173b33
DC		 1081 if (IS_ERR(req))
1082 return PTR_ERR(req);
a4aea562
MB		 1083
1084 req->timeout = timeout;
1085 cmd_rq = blk_mq_rq_to_pdu(req);
1086 cmdinfo->req = req;
1087 nvme_set_info(cmd_rq, cmdinfo, async_completion);
4d115420 1088 cmdinfo->status = -EINTR;
a4aea562
MB		 1089
1090 cmd->common.command_id = req->tag;
1091
4f5099af 1092 return nvme_submit_cmd(nvmeq, cmd);
4d115420
KB		 1093}
1094
b60503ba
MW		 1095static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
1096{
b60503ba
MW		 1097 struct nvme_command c;
1098
1099 memset(&c, 0, sizeof(c));
1100 c.delete_queue.opcode = opcode;
1101 c.delete_queue.qid = cpu_to_le16(id);
1102
d29ec824 1103 return nvme_submit_sync_cmd(dev->admin_q, &c, NULL, 0);
b60503ba
MW		 1104}
1105
1106static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
1107 struct nvme_queue *nvmeq)
1108{
b60503ba
MW		 1109 struct nvme_command c;
1110 int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;
1111
d29ec824
CH		 1112 /*
1113 * Note: we (ab)use the fact the the prp fields survive if no data
1114 * is attached to the request.
1115 */
b60503ba
MW		 1116 memset(&c, 0, sizeof(c));
1117 c.create_cq.opcode = nvme_admin_create_cq;
1118 c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
1119 c.create_cq.cqid = cpu_to_le16(qid);
1120 c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
1121 c.create_cq.cq_flags = cpu_to_le16(flags);
1122 c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);
1123
d29ec824 1124 return nvme_submit_sync_cmd(dev->admin_q, &c, NULL, 0);
b60503ba
MW		 1125}
1126
1127static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
1128 struct nvme_queue *nvmeq)
1129{
b60503ba
MW		 1130 struct nvme_command c;
1131 int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;
1132
d29ec824
CH		 1133 /*
1134 * Note: we (ab)use the fact the the prp fields survive if no data
1135 * is attached to the request.
1136 */
b60503ba
MW		 1137 memset(&c, 0, sizeof(c));
1138 c.create_sq.opcode = nvme_admin_create_sq;
1139 c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
1140 c.create_sq.sqid = cpu_to_le16(qid);
1141 c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
1142 c.create_sq.sq_flags = cpu_to_le16(flags);
1143 c.create_sq.cqid = cpu_to_le16(qid);
1144
d29ec824 1145 return nvme_submit_sync_cmd(dev->admin_q, &c, NULL, 0);
b60503ba
MW		 1146}
1147
1148static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
1149{
1150 return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
1151}
1152
1153static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
1154{
1155 return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
1156}
1157
d29ec824 1158int nvme_identify_ctrl(struct nvme_dev *dev, struct nvme_id_ctrl **id)
bc5fc7e4 1159{
d29ec824
CH		 1160 struct nvme_command c = {
1161 .identify.opcode = nvme_admin_identify,
1162 .identify.cns = cpu_to_le32(1),
1163 };
1164 int error;
bc5fc7e4 1165
d29ec824
CH		 1166 *id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
1167 if (!*id)
1168 return -ENOMEM;
1169
1170 error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
1171 sizeof(struct nvme_id_ctrl));
1172 if (error)
1173 kfree(*id);
1174 return error;
1175}
1176
1177int nvme_identify_ns(struct nvme_dev *dev, unsigned nsid,
1178 struct nvme_id_ns **id)
1179{
1180 struct nvme_command c = {
1181 .identify.opcode = nvme_admin_identify,
1182 .identify.nsid = cpu_to_le32(nsid),
1183 };
1184 int error;
bc5fc7e4 1185
d29ec824
CH		 1186 *id = kmalloc(sizeof(struct nvme_id_ns), GFP_KERNEL);
1187 if (!*id)
1188 return -ENOMEM;
1189
1190 error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
1191 sizeof(struct nvme_id_ns));
1192 if (error)
1193 kfree(*id);
1194 return error;
bc5fc7e4
MW		 1195}
1196
5d0f6131 1197int nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid,
08df1e05 1198 dma_addr_t dma_addr, u32 *result)
bc5fc7e4
MW		 1199{
1200 struct nvme_command c;
1201
1202 memset(&c, 0, sizeof(c));
1203 c.features.opcode = nvme_admin_get_features;
a42cecce 1204 c.features.nsid = cpu_to_le32(nsid);
bc5fc7e4
MW		 1205 c.features.prp1 = cpu_to_le64(dma_addr);
1206 c.features.fid = cpu_to_le32(fid);
bc5fc7e4 1207
d29ec824
CH		 1208 return __nvme_submit_sync_cmd(dev->admin_q, &c, NULL, NULL, 0,
1209 result, 0);
df348139
MW		 1210}
1211
5d0f6131
VV		 1212int nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11,
1213 dma_addr_t dma_addr, u32 *result)
df348139
MW		 1214{
1215 struct nvme_command c;
1216
1217 memset(&c, 0, sizeof(c));
1218 c.features.opcode = nvme_admin_set_features;
1219 c.features.prp1 = cpu_to_le64(dma_addr);
1220 c.features.fid = cpu_to_le32(fid);
1221 c.features.dword11 = cpu_to_le32(dword11);
1222
d29ec824
CH		 1223 return __nvme_submit_sync_cmd(dev->admin_q, &c, NULL, NULL, 0,
1224 result, 0);
1225}
1226
1227int nvme_get_log_page(struct nvme_dev *dev, struct nvme_smart_log **log)
1228{
1229 struct nvme_command c = {
1230 .common.opcode = nvme_admin_get_log_page,
1231 .common.nsid = cpu_to_le32(0xFFFFFFFF),
1232 .common.cdw10[0] = cpu_to_le32(
1233 (((sizeof(struct nvme_smart_log) / 4) - 1) << 16) |
1234 NVME_LOG_SMART),
1235 };
1236 int error;
1237
1238 *log = kmalloc(sizeof(struct nvme_smart_log), GFP_KERNEL);
1239 if (!*log)
1240 return -ENOMEM;
1241
1242 error = nvme_submit_sync_cmd(dev->admin_q, &c, *log,
1243 sizeof(struct nvme_smart_log));
1244 if (error)
1245 kfree(*log);
1246 return error;
bc5fc7e4
MW		 1247}
1248
c30341dc 1249/**
a4aea562 1250 * nvme_abort_req - Attempt aborting a request
c30341dc
KB		 1251 *
1252 * Schedule controller reset if the command was already aborted once before and
1253 * still hasn't been returned to the driver, or if this is the admin queue.
1254 */
a4aea562 1255static void nvme_abort_req(struct request *req)
c30341dc 1256{
a4aea562
MB		 1257 struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
1258 struct nvme_queue *nvmeq = cmd_rq->nvmeq;
c30341dc 1259 struct nvme_dev *dev = nvmeq->dev;
a4aea562
MB		 1260 struct request *abort_req;
1261 struct nvme_cmd_info *abort_cmd;
1262 struct nvme_command cmd;
c30341dc 1263
a4aea562 1264 if (!nvmeq->qid || cmd_rq->aborted) {
7a509a6b
KB		 1265 unsigned long flags;
1266
1267 spin_lock_irqsave(&dev_list_lock, flags);
c30341dc 1268 if (work_busy(&dev->reset_work))
7a509a6b 1269 goto out;
c30341dc 1270 list_del_init(&dev->node);
e75ec752 1271 dev_warn(dev->dev, "I/O %d QID %d timeout, reset controller\n",
a4aea562 1272 req->tag, nvmeq->qid);
9ca97374 1273 dev->reset_workfn = nvme_reset_failed_dev;
c30341dc 1274 queue_work(nvme_workq, &dev->reset_work);
7a509a6b
KB		 1275 out:
1276 spin_unlock_irqrestore(&dev_list_lock, flags);
c30341dc
KB		 1277 return;
1278 }
1279
1280 if (!dev->abort_limit)
1281 return;
1282
a4aea562
MB		 1283 abort_req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_ATOMIC,
1284 false);
9f173b33 1285 if (IS_ERR(abort_req))
c30341dc
KB		 1286 return;
1287
a4aea562
MB		 1288 abort_cmd = blk_mq_rq_to_pdu(abort_req);
1289 nvme_set_info(abort_cmd, abort_req, abort_completion);
1290
c30341dc
KB		 1291 memset(&cmd, 0, sizeof(cmd));
1292 cmd.abort.opcode = nvme_admin_abort_cmd;
a4aea562 1293 cmd.abort.cid = req->tag;
c30341dc 1294 cmd.abort.sqid = cpu_to_le16(nvmeq->qid);
a4aea562 1295 cmd.abort.command_id = abort_req->tag;
c30341dc
KB		 1296
1297 --dev->abort_limit;
a4aea562 1298 cmd_rq->aborted = 1;
c30341dc 1299
a4aea562 1300 dev_warn(nvmeq->q_dmadev, "Aborting I/O %d QID %d\n", req->tag,
c30341dc 1301 nvmeq->qid);
a4aea562
MB		 1302 if (nvme_submit_cmd(dev->queues[0], &cmd) < 0) {
1303 dev_warn(nvmeq->q_dmadev,
1304 "Could not abort I/O %d QID %d",
1305 req->tag, nvmeq->qid);
c87fd540 1306 blk_mq_free_request(abort_req);
a4aea562 1307 }
c30341dc
KB		 1308}
1309
a4aea562
MB		 1310static void nvme_cancel_queue_ios(struct blk_mq_hw_ctx *hctx,
1311 struct request *req, void *data, bool reserved)
a09115b2 1312{
a4aea562
MB		 1313 struct nvme_queue *nvmeq = data;
1314 void *ctx;
1315 nvme_completion_fn fn;
1316 struct nvme_cmd_info *cmd;
cef6a948
KB		 1317 struct nvme_completion cqe;
1318
1319 if (!blk_mq_request_started(req))
1320 return;
a09115b2 1321
a4aea562 1322 cmd = blk_mq_rq_to_pdu(req);
a09115b2 1323
a4aea562
MB		 1324 if (cmd->ctx == CMD_CTX_CANCELLED)
1325 return;
1326
cef6a948
KB		 1327 if (blk_queue_dying(req->q))
1328 cqe.status = cpu_to_le16((NVME_SC_ABORT_REQ | NVME_SC_DNR) << 1);
1329 else
1330 cqe.status = cpu_to_le16(NVME_SC_ABORT_REQ << 1);
1331
1332
a4aea562
MB		 1333 dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d QID %d\n",
1334 req->tag, nvmeq->qid);
1335 ctx = cancel_cmd_info(cmd, &fn);
1336 fn(nvmeq, ctx, &cqe);
a09115b2
MW		 1337}
1338
a4aea562 1339static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
9e866774 1340{
a4aea562
MB		 1341 struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
1342 struct nvme_queue *nvmeq = cmd->nvmeq;
1343
1344 dev_warn(nvmeq->q_dmadev, "Timeout I/O %d QID %d\n", req->tag,
1345 nvmeq->qid);
7a509a6b 1346 spin_lock_irq(&nvmeq->q_lock);
07836e65 1347 nvme_abort_req(req);
7a509a6b 1348 spin_unlock_irq(&nvmeq->q_lock);
a4aea562 1349
07836e65
KB		 1350 /*
1351 * The aborted req will be completed on receiving the abort req.
1352 * We enable the timer again. If hit twice, it'll cause a device reset,
1353 * as the device then is in a faulty state.
1354 */
1355 return BLK_EH_RESET_TIMER;
a4aea562 1356}
22404274 1357
a4aea562
MB		 1358static void nvme_free_queue(struct nvme_queue *nvmeq)
1359{
9e866774
MW		 1360 dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
1361 (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
1362 dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
1363 nvmeq->sq_cmds, nvmeq->sq_dma_addr);
1364 kfree(nvmeq);
1365}
1366
a1a5ef99 1367static void nvme_free_queues(struct nvme_dev *dev, int lowest)
22404274
KB		 1368{
1369 int i;
1370
a1a5ef99 1371 for (i = dev->queue_count - 1; i >= lowest; i--) {
a4aea562 1372 struct nvme_queue *nvmeq = dev->queues[i];
22404274 1373 dev->queue_count--;
a4aea562 1374 dev->queues[i] = NULL;
f435c282 1375 nvme_free_queue(nvmeq);
121c7ad4 1376 }
22404274
KB		 1377}
1378
4d115420
KB		 1379/**
1380 * nvme_suspend_queue - put queue into suspended state
1381 * @nvmeq - queue to suspend
4d115420
KB		 1382 */
1383static int nvme_suspend_queue(struct nvme_queue *nvmeq)
b60503ba 1384{
2b25d981 1385 int vector;
b60503ba 1386
a09115b2 1387 spin_lock_irq(&nvmeq->q_lock);
2b25d981
KB		 1388 if (nvmeq->cq_vector == -1) {
1389 spin_unlock_irq(&nvmeq->q_lock);
1390 return 1;
1391 }
1392 vector = nvmeq->dev->entry[nvmeq->cq_vector].vector;
42f61420 1393 nvmeq->dev->online_queues--;
2b25d981 1394 nvmeq->cq_vector = -1;
a09115b2
MW		 1395 spin_unlock_irq(&nvmeq->q_lock);
1396
6df3dbc8
KB		 1397 if (!nvmeq->qid && nvmeq->dev->admin_q)
1398 blk_mq_freeze_queue_start(nvmeq->dev->admin_q);
1399
aba2080f
MW		 1400 irq_set_affinity_hint(vector, NULL);
1401 free_irq(vector, nvmeq);
b60503ba 1402
4d115420
KB		 1403 return 0;
1404}
b60503ba 1405
4d115420
KB		 1406static void nvme_clear_queue(struct nvme_queue *nvmeq)
1407{
a4aea562
MB		 1408 struct blk_mq_hw_ctx *hctx = nvmeq->hctx;
1409
22404274 1410 spin_lock_irq(&nvmeq->q_lock);
a4aea562
MB		 1411 if (hctx && hctx->tags)
1412 blk_mq_tag_busy_iter(hctx, nvme_cancel_queue_ios, nvmeq);
22404274 1413 spin_unlock_irq(&nvmeq->q_lock);
b60503ba
MW		 1414}
1415
4d115420
KB		 1416static void nvme_disable_queue(struct nvme_dev *dev, int qid)
1417{
a4aea562 1418 struct nvme_queue *nvmeq = dev->queues[qid];
4d115420
KB		 1419
1420 if (!nvmeq)
1421 return;
1422 if (nvme_suspend_queue(nvmeq))
1423 return;
1424
0e53d180
KB		 1425 /* Don't tell the adapter to delete the admin queue.
1426 * Don't tell a removed adapter to delete IO queues. */
1427 if (qid && readl(&dev->bar->csts) != -1) {
b60503ba
MW		 1428 adapter_delete_sq(dev, qid);
1429 adapter_delete_cq(dev, qid);
1430 }
07836e65
KB		 1431
1432 spin_lock_irq(&nvmeq->q_lock);
1433 nvme_process_cq(nvmeq);
1434 spin_unlock_irq(&nvmeq->q_lock);
b60503ba
MW		 1435}
1436
1437static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
2b25d981 1438 int depth)
b60503ba 1439{
a4aea562 1440 struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq), GFP_KERNEL);
b60503ba
MW		 1441 if (!nvmeq)
1442 return NULL;
1443
e75ec752 1444 nvmeq->cqes = dma_zalloc_coherent(dev->dev, CQ_SIZE(depth),
4d51abf9 1445 &nvmeq->cq_dma_addr, GFP_KERNEL);
b60503ba
MW		 1446 if (!nvmeq->cqes)
1447 goto free_nvmeq;
b60503ba 1448
e75ec752 1449 nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(depth),
b60503ba
MW		 1450 &nvmeq->sq_dma_addr, GFP_KERNEL);
1451 if (!nvmeq->sq_cmds)
1452 goto free_cqdma;
1453
e75ec752 1454 nvmeq->q_dmadev = dev->dev;
091b6092 1455 nvmeq->dev = dev;
3193f07b
MW		 1456 snprintf(nvmeq->irqname, sizeof(nvmeq->irqname), "nvme%dq%d",
1457 dev->instance, qid);
b60503ba
MW		 1458 spin_lock_init(&nvmeq->q_lock);
1459 nvmeq->cq_head = 0;
82123460 1460 nvmeq->cq_phase = 1;
b80d5ccc 1461 nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
b60503ba 1462 nvmeq->q_depth = depth;
c30341dc 1463 nvmeq->qid = qid;
22404274 1464 dev->queue_count++;
a4aea562 1465 dev->queues[qid] = nvmeq;
b60503ba
MW		 1466
1467 return nvmeq;
1468
1469 free_cqdma:
e75ec752 1470 dma_free_coherent(dev->dev, CQ_SIZE(depth), (void *)nvmeq->cqes,
b60503ba
MW		 1471 nvmeq->cq_dma_addr);
1472 free_nvmeq:
1473 kfree(nvmeq);
1474 return NULL;
1475}
1476
3001082c
MW		 1477static int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq,
1478 const char *name)
1479{
58ffacb5
MW		 1480 if (use_threaded_interrupts)
1481 return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector,
481e5bad 1482 nvme_irq_check, nvme_irq, IRQF_SHARED,
58ffacb5 1483 name, nvmeq);
3001082c 1484 return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq,
481e5bad 1485 IRQF_SHARED, name, nvmeq);
3001082c
MW		 1486}
1487
22404274 1488static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
b60503ba 1489{
22404274 1490 struct nvme_dev *dev = nvmeq->dev;
b60503ba 1491
7be50e93 1492 spin_lock_irq(&nvmeq->q_lock);
22404274
KB		 1493 nvmeq->sq_tail = 0;
1494 nvmeq->cq_head = 0;
1495 nvmeq->cq_phase = 1;
b80d5ccc 1496 nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
22404274 1497 memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
42f61420 1498 dev->online_queues++;
7be50e93 1499 spin_unlock_irq(&nvmeq->q_lock);
22404274
KB		 1500}
1501
1502static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
1503{
1504 struct nvme_dev *dev = nvmeq->dev;
1505 int result;
3f85d50b 1506
2b25d981 1507 nvmeq->cq_vector = qid - 1;
b60503ba
MW		 1508 result = adapter_alloc_cq(dev, qid, nvmeq);
1509 if (result < 0)
22404274 1510 return result;
b60503ba
MW		 1511
1512 result = adapter_alloc_sq(dev, qid, nvmeq);
1513 if (result < 0)
1514 goto release_cq;
1515
3193f07b 1516 result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
b60503ba
MW		 1517 if (result < 0)
1518 goto release_sq;
1519
22404274 1520 nvme_init_queue(nvmeq, qid);
22404274 1521 return result;
b60503ba
MW		 1522
1523 release_sq:
1524 adapter_delete_sq(dev, qid);
1525 release_cq:
1526 adapter_delete_cq(dev, qid);
22404274 1527 return result;
b60503ba
MW		 1528}
1529
ba47e386
MW		 1530static int nvme_wait_ready(struct nvme_dev *dev, u64 cap, bool enabled)
1531{
1532 unsigned long timeout;
1533 u32 bit = enabled ? NVME_CSTS_RDY : 0;
1534
1535 timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
1536
1537 while ((readl(&dev->bar->csts) & NVME_CSTS_RDY) != bit) {
1538 msleep(100);
1539 if (fatal_signal_pending(current))
1540 return -EINTR;
1541 if (time_after(jiffies, timeout)) {
e75ec752 1542 dev_err(dev->dev,
27e8166c
MW		 1543 "Device not ready; aborting %s\n", enabled ?
1544 "initialisation" : "reset");
ba47e386
MW		 1545 return -ENODEV;
1546 }
1547 }
1548
1549 return 0;
1550}
1551
1552/*
1553 * If the device has been passed off to us in an enabled state, just clear
1554 * the enabled bit. The spec says we should set the 'shutdown notification
1555 * bits', but doing so may cause the device to complete commands to the
1556 * admin queue ... and we don't know what memory that might be pointing at!
1557 */
1558static int nvme_disable_ctrl(struct nvme_dev *dev, u64 cap)
1559{
01079522
DM		 1560 dev->ctrl_config &= ~NVME_CC_SHN_MASK;
1561 dev->ctrl_config &= ~NVME_CC_ENABLE;
1562 writel(dev->ctrl_config, &dev->bar->cc);
44af146a 1563
ba47e386
MW		 1564 return nvme_wait_ready(dev, cap, false);
1565}
1566
1567static int nvme_enable_ctrl(struct nvme_dev *dev, u64 cap)
1568{
01079522
DM		 1569 dev->ctrl_config &= ~NVME_CC_SHN_MASK;
1570 dev->ctrl_config |= NVME_CC_ENABLE;
1571 writel(dev->ctrl_config, &dev->bar->cc);
1572
ba47e386
MW		 1573 return nvme_wait_ready(dev, cap, true);
1574}
1575
1894d8f1
KB		 1576static int nvme_shutdown_ctrl(struct nvme_dev *dev)
1577{
1578 unsigned long timeout;
1894d8f1 1579
01079522
DM		 1580 dev->ctrl_config &= ~NVME_CC_SHN_MASK;
1581 dev->ctrl_config |= NVME_CC_SHN_NORMAL;
1582
1583 writel(dev->ctrl_config, &dev->bar->cc);
1894d8f1 1584
2484f407 1585 timeout = SHUTDOWN_TIMEOUT + jiffies;
1894d8f1
KB		 1586 while ((readl(&dev->bar->csts) & NVME_CSTS_SHST_MASK) !=
1587 NVME_CSTS_SHST_CMPLT) {
1588 msleep(100);
1589 if (fatal_signal_pending(current))
1590 return -EINTR;
1591 if (time_after(jiffies, timeout)) {
e75ec752 1592 dev_err(dev->dev,
1894d8f1
KB		 1593 "Device shutdown incomplete; abort shutdown\n");
1594 return -ENODEV;
1595 }
1596 }
1597
1598 return 0;
1599}
1600
a4aea562 1601static struct blk_mq_ops nvme_mq_admin_ops = {
d29ec824 1602 .queue_rq = nvme_queue_rq,
a4aea562
MB		 1603 .map_queue = blk_mq_map_queue,
1604 .init_hctx = nvme_admin_init_hctx,
2c30540b 1605 .exit_hctx = nvme_exit_hctx,
a4aea562
MB		 1606 .init_request = nvme_admin_init_request,
1607 .timeout = nvme_timeout,
1608};
1609
1610static struct blk_mq_ops nvme_mq_ops = {
1611 .queue_rq = nvme_queue_rq,
1612 .map_queue = blk_mq_map_queue,
1613 .init_hctx = nvme_init_hctx,
2c30540b 1614 .exit_hctx = nvme_exit_hctx,
a4aea562
MB		 1615 .init_request = nvme_init_request,
1616 .timeout = nvme_timeout,
1617};
1618
ea191d2f
KB		 1619static void nvme_dev_remove_admin(struct nvme_dev *dev)
1620{
1621 if (dev->admin_q && !blk_queue_dying(dev->admin_q)) {
1622 blk_cleanup_queue(dev->admin_q);
1623 blk_mq_free_tag_set(&dev->admin_tagset);
1624 }
1625}
1626
a4aea562
MB		 1627static int nvme_alloc_admin_tags(struct nvme_dev *dev)
1628{
1629 if (!dev->admin_q) {
1630 dev->admin_tagset.ops = &nvme_mq_admin_ops;
1631 dev->admin_tagset.nr_hw_queues = 1;
1632 dev->admin_tagset.queue_depth = NVME_AQ_DEPTH - 1;
1efccc9d 1633 dev->admin_tagset.reserved_tags = 1;
a4aea562 1634 dev->admin_tagset.timeout = ADMIN_TIMEOUT;
e75ec752 1635 dev->admin_tagset.numa_node = dev_to_node(dev->dev);
ac3dd5bd 1636 dev->admin_tagset.cmd_size = nvme_cmd_size(dev);
a4aea562
MB		 1637 dev->admin_tagset.driver_data = dev;
1638
1639 if (blk_mq_alloc_tag_set(&dev->admin_tagset))
1640 return -ENOMEM;
1641
1642 dev->admin_q = blk_mq_init_queue(&dev->admin_tagset);
35b489d3 1643 if (IS_ERR(dev->admin_q)) {
a4aea562
MB		 1644 blk_mq_free_tag_set(&dev->admin_tagset);
1645 return -ENOMEM;
1646 }
ea191d2f
KB		 1647 if (!blk_get_queue(dev->admin_q)) {
1648 nvme_dev_remove_admin(dev);
1649 return -ENODEV;
1650 }
0fb59cbc
KB		 1651 } else
1652 blk_mq_unfreeze_queue(dev->admin_q);
a4aea562
MB		 1653
1654 return 0;
1655}
1656
8d85fce7 1657static int nvme_configure_admin_queue(struct nvme_dev *dev)
b60503ba 1658{
ba47e386 1659 int result;
b60503ba 1660 u32 aqa;
ba47e386 1661 u64 cap = readq(&dev->bar->cap);
b60503ba 1662 struct nvme_queue *nvmeq;
1d090624
KB		 1663 unsigned page_shift = PAGE_SHIFT;
1664 unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12;
1665 unsigned dev_page_max = NVME_CAP_MPSMAX(cap) + 12;
1666
1667 if (page_shift < dev_page_min) {
e75ec752 1668 dev_err(dev->dev,
1d090624
KB		 1669 "Minimum device page size (%u) too large for "
1670 "host (%u)\n", 1 << dev_page_min,
1671 1 << page_shift);
1672 return -ENODEV;
1673 }
1674 if (page_shift > dev_page_max) {
e75ec752 1675 dev_info(dev->dev,
1d090624
KB		 1676 "Device maximum page size (%u) smaller than "
1677 "host (%u); enabling work-around\n",
1678 1 << dev_page_max, 1 << page_shift);
1679 page_shift = dev_page_max;
1680 }
b60503ba 1681
ba47e386
MW		 1682 result = nvme_disable_ctrl(dev, cap);
1683 if (result < 0)
1684 return result;
b60503ba 1685
a4aea562 1686 nvmeq = dev->queues[0];
cd638946 1687 if (!nvmeq) {
2b25d981 1688 nvmeq = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
cd638946
KB		 1689 if (!nvmeq)
1690 return -ENOMEM;
cd638946 1691 }
b60503ba
MW		 1692
1693 aqa = nvmeq->q_depth - 1;
1694 aqa |= aqa << 16;
1695
1d090624
KB		 1696 dev->page_size = 1 << page_shift;
1697
01079522 1698 dev->ctrl_config = NVME_CC_CSS_NVM;
1d090624 1699 dev->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
b60503ba 1700 dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
7f53f9d2 1701 dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
b60503ba
MW		 1702
1703 writel(aqa, &dev->bar->aqa);
1704 writeq(nvmeq->sq_dma_addr, &dev->bar->asq);
1705 writeq(nvmeq->cq_dma_addr, &dev->bar->acq);
b60503ba 1706
ba47e386 1707 result = nvme_enable_ctrl(dev, cap);
025c557a 1708 if (result)
a4aea562
MB		 1709 goto free_nvmeq;
1710
2b25d981 1711 nvmeq->cq_vector = 0;
3193f07b 1712 result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
025c557a 1713 if (result)
0fb59cbc 1714 goto free_nvmeq;
025c557a 1715
b60503ba 1716 return result;
a4aea562 1717
a4aea562
MB		 1718 free_nvmeq:
1719 nvme_free_queues(dev, 0);
1720 return result;
b60503ba
MW		 1721}
1722
a53295b6
MW		 1723static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
1724{
1725 struct nvme_dev *dev = ns->dev;
a53295b6
MW		 1726 struct nvme_user_io io;
1727 struct nvme_command c;
d29ec824 1728 unsigned length, meta_len;
a67a9513 1729 int status, write;
a67a9513
KB		 1730 dma_addr_t meta_dma = 0;
1731 void *meta = NULL;
a53295b6
MW		 1732
1733 if (copy_from_user(&io, uio, sizeof(io)))
1734 return -EFAULT;
6c7d4945
MW		 1735
1736 switch (io.opcode) {
1737 case nvme_cmd_write:
1738 case nvme_cmd_read:
6bbf1acd 1739 case nvme_cmd_compare:
6413214c 1740 break;
6c7d4945 1741 default:
6bbf1acd 1742 return -EINVAL;
6c7d4945
MW		 1743 }
1744
d29ec824
CH		 1745 length = (io.nblocks + 1) << ns->lba_shift;
1746 meta_len = (io.nblocks + 1) * ns->ms;
1747 write = io.opcode & 1;
a53295b6 1748
a67a9513 1749 if (meta_len) {
d29ec824
CH		 1750 if (((io.metadata & 3) || !io.metadata) && !ns->ext)
1751 return -EINVAL;
1752
1753 if (ns->ext) {
1754 length += meta_len;
1755 meta_len = 0;
1756 }
1757
e75ec752 1758 meta = dma_alloc_coherent(dev->dev, meta_len,
a67a9513
KB		 1759 &meta_dma, GFP_KERNEL);
1760 if (!meta) {
1761 status = -ENOMEM;
1762 goto unmap;
1763 }
1764 if (write) {
1765 if (copy_from_user(meta, (void __user *)io.metadata,
1766 meta_len)) {
1767 status = -EFAULT;
1768 goto unmap;
1769 }
1770 }
1771 }
1772
a53295b6
MW		 1773 memset(&c, 0, sizeof(c));
1774 c.rw.opcode = io.opcode;
1775 c.rw.flags = io.flags;
6c7d4945 1776 c.rw.nsid = cpu_to_le32(ns->ns_id);
a53295b6 1777 c.rw.slba = cpu_to_le64(io.slba);
6c7d4945 1778 c.rw.length = cpu_to_le16(io.nblocks);
a53295b6 1779 c.rw.control = cpu_to_le16(io.control);
1c9b5265
MW		 1780 c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
1781 c.rw.reftag = cpu_to_le32(io.reftag);
1782 c.rw.apptag = cpu_to_le16(io.apptag);
1783 c.rw.appmask = cpu_to_le16(io.appmask);
a67a9513 1784 c.rw.metadata = cpu_to_le64(meta_dma);
d29ec824
CH		 1785
1786 status = __nvme_submit_sync_cmd(ns->queue, &c, NULL,
1787 (void __user *)io.addr, length, NULL, 0);
f410c680 1788 unmap:
a67a9513
KB		 1789 if (meta) {
1790 if (status == NVME_SC_SUCCESS && !write) {
1791 if (copy_to_user((void __user *)io.metadata, meta,
1792 meta_len))
1793 status = -EFAULT;
1794 }
e75ec752 1795 dma_free_coherent(dev->dev, meta_len, meta, meta_dma);
f410c680 1796 }
a53295b6
MW		 1797 return status;
1798}
1799
a4aea562
MB		 1800static int nvme_user_cmd(struct nvme_dev *dev, struct nvme_ns *ns,
1801 struct nvme_passthru_cmd __user *ucmd)
6ee44cdc 1802{
7963e521 1803 struct nvme_passthru_cmd cmd;
6ee44cdc 1804 struct nvme_command c;
d29ec824
CH		 1805 unsigned timeout = 0;
1806 int status;
6ee44cdc 1807
6bbf1acd
MW		 1808 if (!capable(CAP_SYS_ADMIN))
1809 return -EACCES;
1810 if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
6ee44cdc 1811 return -EFAULT;
6ee44cdc
MW		 1812
1813 memset(&c, 0, sizeof(c));
6bbf1acd
MW		 1814 c.common.opcode = cmd.opcode;
1815 c.common.flags = cmd.flags;
1816 c.common.nsid = cpu_to_le32(cmd.nsid);
1817 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1818 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1819 c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
1820 c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
1821 c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
1822 c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
1823 c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
1824 c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);
1825
d29ec824
CH		 1826 if (cmd.timeout_ms)
1827 timeout = msecs_to_jiffies(cmd.timeout_ms);
f705f837
CH		 1828
1829 status = __nvme_submit_sync_cmd(ns ? ns->queue : dev->admin_q, &c,
d29ec824
CH		 1830 NULL, (void __user *)cmd.addr, cmd.data_len,
1831 &cmd.result, timeout);
1832 if (status >= 0) {
1833 if (put_user(cmd.result, &ucmd->result))
1834 return -EFAULT;
6bbf1acd 1835 }
f4f117f6 1836
6ee44cdc
MW		 1837 return status;
1838}
1839
b60503ba
MW		 1840static int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd,
1841 unsigned long arg)
1842{
1843 struct nvme_ns *ns = bdev->bd_disk->private_data;
1844
1845 switch (cmd) {
6bbf1acd 1846 case NVME_IOCTL_ID:
c3bfe717 1847 force_successful_syscall_return();
6bbf1acd
MW		 1848 return ns->ns_id;
1849 case NVME_IOCTL_ADMIN_CMD:
a4aea562 1850 return nvme_user_cmd(ns->dev, NULL, (void __user *)arg);
7963e521 1851 case NVME_IOCTL_IO_CMD:
a4aea562 1852 return nvme_user_cmd(ns->dev, ns, (void __user *)arg);
a53295b6
MW		 1853 case NVME_IOCTL_SUBMIT_IO:
1854 return nvme_submit_io(ns, (void __user *)arg);
5d0f6131
VV		 1855 case SG_GET_VERSION_NUM:
1856 return nvme_sg_get_version_num((void __user *)arg);
1857 case SG_IO:
1858 return nvme_sg_io(ns, (void __user *)arg);
b60503ba
MW		 1859 default:
1860 return -ENOTTY;
1861 }
1862}
1863
320a3827
KB		 1864#ifdef CONFIG_COMPAT
1865static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
1866 unsigned int cmd, unsigned long arg)
1867{
320a3827
KB		 1868 switch (cmd) {
1869 case SG_IO:
e179729a 1870 return -ENOIOCTLCMD;
320a3827
KB		 1871 }
1872 return nvme_ioctl(bdev, mode, cmd, arg);
1873}
1874#else
1875#define nvme_compat_ioctl NULL
1876#endif
1877
9ac27090
KB		 1878static int nvme_open(struct block_device *bdev, fmode_t mode)
1879{
9e60352c
KB		 1880 int ret = 0;
1881 struct nvme_ns *ns;
9ac27090 1882
9e60352c
KB		 1883 spin_lock(&dev_list_lock);
1884 ns = bdev->bd_disk->private_data;
1885 if (!ns)
1886 ret = -ENXIO;
1887 else if (!kref_get_unless_zero(&ns->dev->kref))
1888 ret = -ENXIO;
1889 spin_unlock(&dev_list_lock);
1890
1891 return ret;
9ac27090
KB		 1892}
1893
1894static void nvme_free_dev(struct kref *kref);
1895
1896static void nvme_release(struct gendisk *disk, fmode_t mode)
1897{
1898 struct nvme_ns *ns = disk->private_data;
1899 struct nvme_dev *dev = ns->dev;
1900
1901 kref_put(&dev->kref, nvme_free_dev);
1902}
1903
4cc09e2d
KB		 1904static int nvme_getgeo(struct block_device *bd, struct hd_geometry *geo)
1905{
1906 /* some standard values */
1907 geo->heads = 1 << 6;
1908 geo->sectors = 1 << 5;
1909 geo->cylinders = get_capacity(bd->bd_disk) >> 11;
1910 return 0;
1911}
1912
e1e5e564
KB		 1913static void nvme_config_discard(struct nvme_ns *ns)
1914{
1915 u32 logical_block_size = queue_logical_block_size(ns->queue);
1916 ns->queue->limits.discard_zeroes_data = 0;
1917 ns->queue->limits.discard_alignment = logical_block_size;
1918 ns->queue->limits.discard_granularity = logical_block_size;
1919 ns->queue->limits.max_discard_sectors = 0xffffffff;
1920 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
1921}
1922
1b9dbf7f
KB		 1923static int nvme_revalidate_disk(struct gendisk *disk)
1924{
1925 struct nvme_ns *ns = disk->private_data;
1926 struct nvme_dev *dev = ns->dev;
1927 struct nvme_id_ns *id;
a67a9513
KB		 1928 u8 lbaf, pi_type;
1929 u16 old_ms;
e1e5e564 1930 unsigned short bs;
1b9dbf7f 1931
d29ec824
CH		 1932 if (nvme_identify_ns(dev, ns->ns_id, &id)) {
1933 dev_warn(dev->dev, "%s: Identify failure\n", __func__);
1b9dbf7f
KB		 1934 return 0;
1935 }
1936
e1e5e564
KB		 1937 old_ms = ns->ms;
1938 lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
1b9dbf7f 1939 ns->lba_shift = id->lbaf[lbaf].ds;
e1e5e564 1940 ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
a67a9513 1941 ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);
e1e5e564
KB		 1942
1943 /*
1944 * If identify namespace failed, use default 512 byte block size so
1945 * block layer can use before failing read/write for 0 capacity.
1946 */
1947 if (ns->lba_shift == 0)
1948 ns->lba_shift = 9;
1949 bs = 1 << ns->lba_shift;
1950
1951 /* XXX: PI implementation requires metadata equal t10 pi tuple size */
1952 pi_type = ns->ms == sizeof(struct t10_pi_tuple) ?
1953 id->dps & NVME_NS_DPS_PI_MASK : 0;
1954
52b68d7e
KB		 1955 if (blk_get_integrity(disk) && (ns->pi_type != pi_type ||
1956 ns->ms != old_ms ||
e1e5e564 1957 bs != queue_logical_block_size(disk->queue) ||
a67a9513 1958 (ns->ms && ns->ext)))
e1e5e564
KB		 1959 blk_integrity_unregister(disk);
1960
1961 ns->pi_type = pi_type;
1962 blk_queue_logical_block_size(ns->queue, bs);
1963
52b68d7e 1964 if (ns->ms && !blk_get_integrity(disk) && (disk->flags & GENHD_FL_UP) &&
a67a9513 1965 !ns->ext)
e1e5e564
KB		 1966 nvme_init_integrity(ns);
1967
52b68d7e 1968 if (id->ncap == 0 || (ns->ms && !blk_get_integrity(disk)))
e1e5e564
KB		 1969 set_capacity(disk, 0);
1970 else
1971 set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));
1972
1973 if (dev->oncs & NVME_CTRL_ONCS_DSM)
1974 nvme_config_discard(ns);
1b9dbf7f 1975
d29ec824 1976 kfree(id);
1b9dbf7f
KB		 1977 return 0;
1978}
1979
b60503ba
MW		 1980static const struct block_device_operations nvme_fops = {
1981 .owner = THIS_MODULE,
1982 .ioctl = nvme_ioctl,
320a3827 1983 .compat_ioctl = nvme_compat_ioctl,
9ac27090
KB		 1984 .open = nvme_open,
1985 .release = nvme_release,
4cc09e2d 1986 .getgeo = nvme_getgeo,
1b9dbf7f 1987 .revalidate_disk= nvme_revalidate_disk,
b60503ba
MW		 1988};
1989
1fa6aead
MW		 1990static int nvme_kthread(void *data)
1991{
d4b4ff8e 1992 struct nvme_dev *dev, *next;
1fa6aead
MW		 1993
1994 while (!kthread_should_stop()) {
564a232c 1995 set_current_state(TASK_INTERRUPTIBLE);
1fa6aead 1996 spin_lock(&dev_list_lock);
d4b4ff8e 1997 list_for_each_entry_safe(dev, next, &dev_list, node) {
1fa6aead 1998 int i;
07836e65 1999 if (readl(&dev->bar->csts) & NVME_CSTS_CFS) {
d4b4ff8e
KB		 2000 if (work_busy(&dev->reset_work))
2001 continue;
2002 list_del_init(&dev->node);
e75ec752 2003 dev_warn(dev->dev,
a4aea562
MB		 2004 "Failed status: %x, reset controller\n",
2005 readl(&dev->bar->csts));
9ca97374 2006 dev->reset_workfn = nvme_reset_failed_dev;
d4b4ff8e
KB		 2007 queue_work(nvme_workq, &dev->reset_work);
2008 continue;
2009 }
1fa6aead 2010 for (i = 0; i < dev->queue_count; i++) {
a4aea562 2011 struct nvme_queue *nvmeq = dev->queues[i];
740216fc
MW		 2012 if (!nvmeq)
2013 continue;
1fa6aead 2014 spin_lock_irq(&nvmeq->q_lock);
bc57a0f7 2015 nvme_process_cq(nvmeq);
6fccf938
KB		 2016
2017 while ((i == 0) && (dev->event_limit > 0)) {
a4aea562 2018 if (nvme_submit_async_admin_req(dev))
6fccf938
KB		 2019 break;
2020 dev->event_limit--;
2021 }
1fa6aead
MW		 2022 spin_unlock_irq(&nvmeq->q_lock);
2023 }
2024 }
2025 spin_unlock(&dev_list_lock);
acb7aa0d 2026 schedule_timeout(round_jiffies_relative(HZ));
1fa6aead
MW		 2027 }
2028 return 0;
2029}
2030
e1e5e564 2031static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid)
b60503ba
MW		 2032{
2033 struct nvme_ns *ns;
2034 struct gendisk *disk;
e75ec752 2035 int node = dev_to_node(dev->dev);
b60503ba 2036
a4aea562 2037 ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
b60503ba 2038 if (!ns)
e1e5e564
KB		 2039 return;
2040
a4aea562 2041 ns->queue = blk_mq_init_queue(&dev->tagset);
9f173b33 2042 if (IS_ERR(ns->queue))
b60503ba 2043 goto out_free_ns;
4eeb9215
MW		 2044 queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue);
2045 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
a4aea562 2046 queue_flag_set_unlocked(QUEUE_FLAG_SG_GAPS, ns->queue);
b60503ba
MW		 2047 ns->dev = dev;
2048 ns->queue->queuedata = ns;
2049
a4aea562 2050 disk = alloc_disk_node(0, node);
b60503ba
MW		 2051 if (!disk)
2052 goto out_free_queue;
a4aea562 2053
5aff9382 2054 ns->ns_id = nsid;
b60503ba 2055 ns->disk = disk;
e1e5e564
KB		 2056 ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
2057 list_add_tail(&ns->list, &dev->namespaces);
2058
e9ef4636 2059 blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
8fc23e03
KB		 2060 if (dev->max_hw_sectors)
2061 blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors);
a4aea562
MB		 2062 if (dev->stripe_size)
2063 blk_queue_chunk_sectors(ns->queue, dev->stripe_size >> 9);
a7d2ce28
KB		 2064 if (dev->vwc & NVME_CTRL_VWC_PRESENT)
2065 blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA);
b60503ba
MW		 2066
2067 disk->major = nvme_major;
469071a3 2068 disk->first_minor = 0;
b60503ba
MW		 2069 disk->fops = &nvme_fops;
2070 disk->private_data = ns;
2071 disk->queue = ns->queue;
b3fffdef 2072 disk->driverfs_dev = dev->device;
469071a3 2073 disk->flags = GENHD_FL_EXT_DEVT;
5aff9382 2074 sprintf(disk->disk_name, "nvme%dn%d", dev->instance, nsid);
b60503ba 2075
e1e5e564
KB		 2076 /*
2077 * Initialize capacity to 0 until we establish the namespace format and
2078 * setup integrity extentions if necessary. The revalidate_disk after
2079 * add_disk allows the driver to register with integrity if the format
2080 * requires it.
2081 */
2082 set_capacity(disk, 0);
2083 nvme_revalidate_disk(ns->disk);
2084 add_disk(ns->disk);
2085 if (ns->ms)
2086 revalidate_disk(ns->disk);
2087 return;
b60503ba
MW		 2088 out_free_queue:
2089 blk_cleanup_queue(ns->queue);
2090 out_free_ns:
2091 kfree(ns);
b60503ba
MW		 2092}
2093
42f61420
KB		 2094static void nvme_create_io_queues(struct nvme_dev *dev)
2095{
a4aea562 2096 unsigned i;
42f61420 2097
a4aea562 2098 for (i = dev->queue_count; i <= dev->max_qid; i++)
2b25d981 2099 if (!nvme_alloc_queue(dev, i, dev->q_depth))
42f61420
KB		 2100 break;
2101
a4aea562
MB		 2102 for (i = dev->online_queues; i <= dev->queue_count - 1; i++)
2103 if (nvme_create_queue(dev->queues[i], i))
42f61420
KB		 2104 break;
2105}
2106
b3b06812 2107static int set_queue_count(struct nvme_dev *dev, int count)
b60503ba
MW		 2108{
2109 int status;
2110 u32 result;
b3b06812 2111 u32 q_count = (count - 1) | ((count - 1) << 16);
b60503ba 2112
df348139 2113 status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES, q_count, 0,
bc5fc7e4 2114 &result);
27e8166c
MW		 2115 if (status < 0)
2116 return status;
2117 if (status > 0) {
e75ec752 2118 dev_err(dev->dev, "Could not set queue count (%d)\n", status);
badc34d4 2119 return 0;
27e8166c 2120 }
b60503ba
MW		 2121 return min(result & 0xffff, result >> 16) + 1;
2122}
2123
9d713c2b
KB		 2124static size_t db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
2125{
b80d5ccc 2126 return 4096 + ((nr_io_queues + 1) * 8 * dev->db_stride);
9d713c2b
KB		 2127}
2128
8d85fce7 2129static int nvme_setup_io_queues(struct nvme_dev *dev)
b60503ba 2130{
a4aea562 2131 struct nvme_queue *adminq = dev->queues[0];
e75ec752 2132 struct pci_dev *pdev = to_pci_dev(dev->dev);
42f61420 2133 int result, i, vecs, nr_io_queues, size;
b60503ba 2134
42f61420 2135 nr_io_queues = num_possible_cpus();
b348b7d5 2136 result = set_queue_count(dev, nr_io_queues);
badc34d4 2137 if (result <= 0)
1b23484b 2138 return result;
b348b7d5
MW		 2139 if (result < nr_io_queues)
2140 nr_io_queues = result;
b60503ba 2141
9d713c2b
KB		 2142 size = db_bar_size(dev, nr_io_queues);
2143 if (size > 8192) {
f1938f6e 2144 iounmap(dev->bar);
9d713c2b
KB		 2145 do {
2146 dev->bar = ioremap(pci_resource_start(pdev, 0), size);
2147 if (dev->bar)
2148 break;
2149 if (!--nr_io_queues)
2150 return -ENOMEM;
2151 size = db_bar_size(dev, nr_io_queues);
2152 } while (1);
f1938f6e 2153 dev->dbs = ((void __iomem *)dev->bar) + 4096;
5a92e700 2154 adminq->q_db = dev->dbs;
f1938f6e
MW		 2155 }
2156
9d713c2b 2157 /* Deregister the admin queue's interrupt */
3193f07b 2158 free_irq(dev->entry[0].vector, adminq);
9d713c2b 2159
e32efbfc
JA		 2160 /*
2161 * If we enable msix early due to not intx, disable it again before
2162 * setting up the full range we need.
2163 */
2164 if (!pdev->irq)
2165 pci_disable_msix(pdev);
2166
be577fab 2167 for (i = 0; i < nr_io_queues; i++)
1b23484b 2168 dev->entry[i].entry = i;
be577fab
AG		 2169 vecs = pci_enable_msix_range(pdev, dev->entry, 1, nr_io_queues);
2170 if (vecs < 0) {
2171 vecs = pci_enable_msi_range(pdev, 1, min(nr_io_queues, 32));
2172 if (vecs < 0) {
2173 vecs = 1;
2174 } else {
2175 for (i = 0; i < vecs; i++)
2176 dev->entry[i].vector = i + pdev->irq;
fa08a396
RRG		 2177 }
2178 }
2179
063a8096
MW		 2180 /*
2181 * Should investigate if there's a performance win from allocating
2182 * more queues than interrupt vectors; it might allow the submission
2183 * path to scale better, even if the receive path is limited by the
2184 * number of interrupts.
2185 */
2186 nr_io_queues = vecs;
42f61420 2187 dev->max_qid = nr_io_queues;
063a8096 2188
3193f07b 2189 result = queue_request_irq(dev, adminq, adminq->irqname);
a4aea562 2190 if (result)
22404274 2191 goto free_queues;
1b23484b 2192
cd638946 2193 /* Free previously allocated queues that are no longer usable */
42f61420 2194 nvme_free_queues(dev, nr_io_queues + 1);
a4aea562 2195 nvme_create_io_queues(dev);
9ecdc946 2196
22404274 2197 return 0;
b60503ba 2198
22404274 2199 free_queues:
a1a5ef99 2200 nvme_free_queues(dev, 1);
22404274 2201 return result;
b60503ba
MW		 2202}
2203
422ef0c7
MW		 2204/*
2205 * Return: error value if an error occurred setting up the queues or calling
2206 * Identify Device. 0 if these succeeded, even if adding some of the
2207 * namespaces failed. At the moment, these failures are silent. TBD which
2208 * failures should be reported.
2209 */
8d85fce7 2210static int nvme_dev_add(struct nvme_dev *dev)
b60503ba 2211{
e75ec752 2212 struct pci_dev *pdev = to_pci_dev(dev->dev);
c3bfe717
MW		 2213 int res;
2214 unsigned nn, i;
51814232 2215 struct nvme_id_ctrl *ctrl;
159b67d7 2216 int shift = NVME_CAP_MPSMIN(readq(&dev->bar->cap)) + 12;
b60503ba 2217
d29ec824 2218 res = nvme_identify_ctrl(dev, &ctrl);
b60503ba 2219 if (res) {
e75ec752 2220 dev_err(dev->dev, "Identify Controller failed (%d)\n", res);
e1e5e564 2221 return -EIO;
b60503ba
MW		 2222 }
2223
51814232 2224 nn = le32_to_cpup(&ctrl->nn);
0e5e4f0e 2225 dev->oncs = le16_to_cpup(&ctrl->oncs);
c30341dc 2226 dev->abort_limit = ctrl->acl + 1;
a7d2ce28 2227 dev->vwc = ctrl->vwc;
51814232
MW		 2228 memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn));
2229 memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn));
2230 memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr));
159b67d7 2231 if (ctrl->mdts)
8fc23e03 2232 dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9);
68608c26 2233 if ((pdev->vendor == PCI_VENDOR_ID_INTEL) &&
a4aea562
MB		 2234 (pdev->device == 0x0953) && ctrl->vs[3]) {
2235 unsigned int max_hw_sectors;
2236
159b67d7 2237 dev->stripe_size = 1 << (ctrl->vs[3] + shift);
a4aea562
MB		 2238 max_hw_sectors = dev->stripe_size >> (shift - 9);
2239 if (dev->max_hw_sectors) {
2240 dev->max_hw_sectors = min(max_hw_sectors,
2241 dev->max_hw_sectors);
2242 } else
2243 dev->max_hw_sectors = max_hw_sectors;
2244 }
d29ec824 2245 kfree(ctrl);
a4aea562
MB		 2246
2247 dev->tagset.ops = &nvme_mq_ops;
2248 dev->tagset.nr_hw_queues = dev->online_queues - 1;
2249 dev->tagset.timeout = NVME_IO_TIMEOUT;
e75ec752 2250 dev->tagset.numa_node = dev_to_node(dev->dev);
a4aea562
MB		 2251 dev->tagset.queue_depth =
2252 min_t(int, dev->q_depth, BLK_MQ_MAX_DEPTH) - 1;
ac3dd5bd 2253 dev->tagset.cmd_size = nvme_cmd_size(dev);
a4aea562
MB		 2254 dev->tagset.flags = BLK_MQ_F_SHOULD_MERGE;
2255 dev->tagset.driver_data = dev;
2256
2257 if (blk_mq_alloc_tag_set(&dev->tagset))
e1e5e564 2258 return 0;
b60503ba 2259
e1e5e564
KB		 2260 for (i = 1; i <= nn; i++)
2261 nvme_alloc_ns(dev, i);
b60503ba 2262
e1e5e564 2263 return 0;
b60503ba
MW		 2264}
2265
0877cb0d
KB		 2266static int nvme_dev_map(struct nvme_dev *dev)
2267{
42f61420 2268 u64 cap;
0877cb0d 2269 int bars, result = -ENOMEM;
e75ec752 2270 struct pci_dev *pdev = to_pci_dev(dev->dev);
0877cb0d
KB		 2271
2272 if (pci_enable_device_mem(pdev))
2273 return result;
2274
2275 dev->entry[0].vector = pdev->irq;
2276 pci_set_master(pdev);
2277 bars = pci_select_bars(pdev, IORESOURCE_MEM);
be7837e8
JA		 2278 if (!bars)
2279 goto disable_pci;
2280
0877cb0d
KB		 2281 if (pci_request_selected_regions(pdev, bars, "nvme"))
2282 goto disable_pci;
2283
e75ec752
CH		 2284 if (dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(64)) &&
2285 dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(32)))
052d0efa 2286 goto disable;
0877cb0d 2287
0877cb0d
KB		 2288 dev->bar = ioremap(pci_resource_start(pdev, 0), 8192);
2289 if (!dev->bar)
2290 goto disable;
e32efbfc 2291
0e53d180
KB		 2292 if (readl(&dev->bar->csts) == -1) {
2293 result = -ENODEV;
2294 goto unmap;
2295 }
e32efbfc
JA		 2296
2297 /*
2298 * Some devices don't advertse INTx interrupts, pre-enable a single
2299 * MSIX vec for setup. We'll adjust this later.
2300 */
2301 if (!pdev->irq) {
2302 result = pci_enable_msix(pdev, dev->entry, 1);
2303 if (result < 0)
2304 goto unmap;
2305 }
2306
42f61420
KB		 2307 cap = readq(&dev->bar->cap);
2308 dev->q_depth = min_t(int, NVME_CAP_MQES(cap) + 1, NVME_Q_DEPTH);
2309 dev->db_stride = 1 << NVME_CAP_STRIDE(cap);
0877cb0d
KB		 2310 dev->dbs = ((void __iomem *)dev->bar) + 4096;
2311
2312 return 0;
2313
0e53d180
KB		 2314 unmap:
2315 iounmap(dev->bar);
2316 dev->bar = NULL;
0877cb0d
KB		 2317 disable:
2318 pci_release_regions(pdev);
2319 disable_pci:
2320 pci_disable_device(pdev);
2321 return result;
2322}
2323
2324static void nvme_dev_unmap(struct nvme_dev *dev)
2325{
e75ec752
CH		 2326 struct pci_dev *pdev = to_pci_dev(dev->dev);
2327
2328 if (pdev->msi_enabled)
2329 pci_disable_msi(pdev);
2330 else if (pdev->msix_enabled)
2331 pci_disable_msix(pdev);
0877cb0d
KB		 2332
2333 if (dev->bar) {
2334 iounmap(dev->bar);
2335 dev->bar = NULL;
e75ec752 2336 pci_release_regions(pdev);
0877cb0d
KB		 2337 }
2338
e75ec752
CH		 2339 if (pci_is_enabled(pdev))
2340 pci_disable_device(pdev);
0877cb0d
KB		 2341}
2342
4d115420
KB		 2343struct nvme_delq_ctx {
2344 struct task_struct *waiter;
2345 struct kthread_worker *worker;
2346 atomic_t refcount;
2347};
2348
2349static void nvme_wait_dq(struct nvme_delq_ctx *dq, struct nvme_dev *dev)
2350{
2351 dq->waiter = current;
2352 mb();
2353
2354 for (;;) {
2355 set_current_state(TASK_KILLABLE);
2356 if (!atomic_read(&dq->refcount))
2357 break;
2358 if (!schedule_timeout(ADMIN_TIMEOUT) ||
2359 fatal_signal_pending(current)) {
0fb59cbc
KB		 2360 /*
2361 * Disable the controller first since we can't trust it
2362 * at this point, but leave the admin queue enabled
2363 * until all queue deletion requests are flushed.
2364 * FIXME: This may take a while if there are more h/w
2365 * queues than admin tags.
2366 */
4d115420 2367 set_current_state(TASK_RUNNING);
4d115420 2368 nvme_disable_ctrl(dev, readq(&dev->bar->cap));
0fb59cbc 2369 nvme_clear_queue(dev->queues[0]);
4d115420 2370 flush_kthread_worker(dq->worker);
0fb59cbc 2371 nvme_disable_queue(dev, 0);
4d115420
KB		 2372 return;
2373 }
2374 }
2375 set_current_state(TASK_RUNNING);
2376}
2377
2378static void nvme_put_dq(struct nvme_delq_ctx *dq)
2379{
2380 atomic_dec(&dq->refcount);
2381 if (dq->waiter)
2382 wake_up_process(dq->waiter);
2383}
2384
2385static struct nvme_delq_ctx *nvme_get_dq(struct nvme_delq_ctx *dq)
2386{
2387 atomic_inc(&dq->refcount);
2388 return dq;
2389}
2390
2391static void nvme_del_queue_end(struct nvme_queue *nvmeq)
2392{
2393 struct nvme_delq_ctx *dq = nvmeq->cmdinfo.ctx;
4d115420
KB		 2394 nvme_put_dq(dq);
2395}
2396
2397static int adapter_async_del_queue(struct nvme_queue *nvmeq, u8 opcode,
2398 kthread_work_func_t fn)
2399{
2400 struct nvme_command c;
2401
2402 memset(&c, 0, sizeof(c));
2403 c.delete_queue.opcode = opcode;
2404 c.delete_queue.qid = cpu_to_le16(nvmeq->qid);
2405
2406 init_kthread_work(&nvmeq->cmdinfo.work, fn);
a4aea562
MB		 2407 return nvme_submit_admin_async_cmd(nvmeq->dev, &c, &nvmeq->cmdinfo,
2408 ADMIN_TIMEOUT);
4d115420
KB		 2409}
2410
2411static void nvme_del_cq_work_handler(struct kthread_work *work)
2412{
2413 struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
2414 cmdinfo.work);
2415 nvme_del_queue_end(nvmeq);
2416}
2417
2418static int nvme_delete_cq(struct nvme_queue *nvmeq)
2419{
2420 return adapter_async_del_queue(nvmeq, nvme_admin_delete_cq,
2421 nvme_del_cq_work_handler);
2422}
2423
2424static void nvme_del_sq_work_handler(struct kthread_work *work)
2425{
2426 struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
2427 cmdinfo.work);
2428 int status = nvmeq->cmdinfo.status;
2429
2430 if (!status)
2431 status = nvme_delete_cq(nvmeq);
2432 if (status)
2433 nvme_del_queue_end(nvmeq);
2434}
2435
2436static int nvme_delete_sq(struct nvme_queue *nvmeq)
2437{
2438 return adapter_async_del_queue(nvmeq, nvme_admin_delete_sq,
2439 nvme_del_sq_work_handler);
2440}
2441
2442static void nvme_del_queue_start(struct kthread_work *work)
2443{
2444 struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
2445 cmdinfo.work);
4d115420
KB		 2446 if (nvme_delete_sq(nvmeq))
2447 nvme_del_queue_end(nvmeq);
2448}
2449
2450static void nvme_disable_io_queues(struct nvme_dev *dev)
2451{
2452 int i;
2453 DEFINE_KTHREAD_WORKER_ONSTACK(worker);
2454 struct nvme_delq_ctx dq;
2455 struct task_struct *kworker_task = kthread_run(kthread_worker_fn,
2456 &worker, "nvme%d", dev->instance);
2457
2458 if (IS_ERR(kworker_task)) {
e75ec752 2459 dev_err(dev->dev,
4d115420
KB		 2460 "Failed to create queue del task\n");
2461 for (i = dev->queue_count - 1; i > 0; i--)
2462 nvme_disable_queue(dev, i);
2463 return;
2464 }
2465
2466 dq.waiter = NULL;
2467 atomic_set(&dq.refcount, 0);
2468 dq.worker = &worker;
2469 for (i = dev->queue_count - 1; i > 0; i--) {
a4aea562 2470 struct nvme_queue *nvmeq = dev->queues[i];
4d115420
KB		 2471
2472 if (nvme_suspend_queue(nvmeq))
2473 continue;
2474 nvmeq->cmdinfo.ctx = nvme_get_dq(&dq);
2475 nvmeq->cmdinfo.worker = dq.worker;
2476 init_kthread_work(&nvmeq->cmdinfo.work, nvme_del_queue_start);
2477 queue_kthread_work(dq.worker, &nvmeq->cmdinfo.work);
2478 }
2479 nvme_wait_dq(&dq, dev);
2480 kthread_stop(kworker_task);
2481}
2482
b9afca3e
DM		 2483/*
2484* Remove the node from the device list and check
2485* for whether or not we need to stop the nvme_thread.
2486*/
2487static void nvme_dev_list_remove(struct nvme_dev *dev)
2488{
2489 struct task_struct *tmp = NULL;
2490
2491 spin_lock(&dev_list_lock);
2492 list_del_init(&dev->node);
2493 if (list_empty(&dev_list) && !IS_ERR_OR_NULL(nvme_thread)) {
2494 tmp = nvme_thread;
2495 nvme_thread = NULL;
2496 }
2497 spin_unlock(&dev_list_lock);
2498
2499 if (tmp)
2500 kthread_stop(tmp);
2501}
2502
c9d3bf88
KB		 2503static void nvme_freeze_queues(struct nvme_dev *dev)
2504{
2505 struct nvme_ns *ns;
2506
2507 list_for_each_entry(ns, &dev->namespaces, list) {
2508 blk_mq_freeze_queue_start(ns->queue);
2509
cddcd72b 2510 spin_lock_irq(ns->queue->queue_lock);
c9d3bf88 2511 queue_flag_set(QUEUE_FLAG_STOPPED, ns->queue);
cddcd72b 2512 spin_unlock_irq(ns->queue->queue_lock);
c9d3bf88
KB		 2513
2514 blk_mq_cancel_requeue_work(ns->queue);
2515 blk_mq_stop_hw_queues(ns->queue);
2516 }
2517}
2518
2519static void nvme_unfreeze_queues(struct nvme_dev *dev)
2520{
2521 struct nvme_ns *ns;
2522
2523 list_for_each_entry(ns, &dev->namespaces, list) {
2524 queue_flag_clear_unlocked(QUEUE_FLAG_STOPPED, ns->queue);
2525 blk_mq_unfreeze_queue(ns->queue);
2526 blk_mq_start_stopped_hw_queues(ns->queue, true);
2527 blk_mq_kick_requeue_list(ns->queue);
2528 }
2529}
2530
f0b50732 2531static void nvme_dev_shutdown(struct nvme_dev *dev)
b60503ba 2532{
22404274 2533 int i;
7c1b2450 2534 u32 csts = -1;
22404274 2535
b9afca3e 2536 nvme_dev_list_remove(dev);
1fa6aead 2537
c9d3bf88
KB		 2538 if (dev->bar) {
2539 nvme_freeze_queues(dev);
7c1b2450 2540 csts = readl(&dev->bar->csts);
c9d3bf88 2541 }
7c1b2450 2542 if (csts & NVME_CSTS_CFS || !(csts & NVME_CSTS_RDY)) {
4d115420 2543 for (i = dev->queue_count - 1; i >= 0; i--) {
a4aea562 2544 struct nvme_queue *nvmeq = dev->queues[i];
4d115420 2545 nvme_suspend_queue(nvmeq);
4d115420
KB		 2546 }
2547 } else {
2548 nvme_disable_io_queues(dev);
1894d8f1 2549 nvme_shutdown_ctrl(dev);
4d115420
KB		 2550 nvme_disable_queue(dev, 0);
2551 }
f0b50732 2552 nvme_dev_unmap(dev);
07836e65
KB		 2553
2554 for (i = dev->queue_count - 1; i >= 0; i--)
2555 nvme_clear_queue(dev->queues[i]);
f0b50732
KB		 2556}
2557
2558static void nvme_dev_remove(struct nvme_dev *dev)
2559{
9ac27090 2560 struct nvme_ns *ns;
f0b50732 2561
9ac27090 2562 list_for_each_entry(ns, &dev->namespaces, list) {
e1e5e564 2563 if (ns->disk->flags & GENHD_FL_UP) {
52b68d7e 2564 if (blk_get_integrity(ns->disk))
e1e5e564 2565 blk_integrity_unregister(ns->disk);
9ac27090 2566 del_gendisk(ns->disk);
e1e5e564 2567 }
cef6a948
KB		 2568 if (!blk_queue_dying(ns->queue)) {
2569 blk_mq_abort_requeue_list(ns->queue);
9ac27090 2570 blk_cleanup_queue(ns->queue);
cef6a948 2571 }
b60503ba 2572 }
b60503ba
MW		 2573}
2574
091b6092
MW		 2575static int nvme_setup_prp_pools(struct nvme_dev *dev)
2576{
e75ec752 2577 dev->prp_page_pool = dma_pool_create("prp list page", dev->dev,
091b6092
MW		 2578 PAGE_SIZE, PAGE_SIZE, 0);
2579 if (!dev->prp_page_pool)
2580 return -ENOMEM;
2581
99802a7a 2582 /* Optimisation for I/Os between 4k and 128k */
e75ec752 2583 dev->prp_small_pool = dma_pool_create("prp list 256", dev->dev,
99802a7a
MW		 2584 256, 256, 0);
2585 if (!dev->prp_small_pool) {
2586 dma_pool_destroy(dev->prp_page_pool);
2587 return -ENOMEM;
2588 }
091b6092
MW		 2589 return 0;
2590}
2591
2592static void nvme_release_prp_pools(struct nvme_dev *dev)
2593{
2594 dma_pool_destroy(dev->prp_page_pool);
99802a7a 2595 dma_pool_destroy(dev->prp_small_pool);
091b6092
MW		 2596}
2597
cd58ad7d
QSA		 2598static DEFINE_IDA(nvme_instance_ida);
2599
2600static int nvme_set_instance(struct nvme_dev *dev)
b60503ba 2601{
cd58ad7d
QSA		 2602 int instance, error;
2603
2604 do {
2605 if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
2606 return -ENODEV;
2607
2608 spin_lock(&dev_list_lock);
2609 error = ida_get_new(&nvme_instance_ida, &instance);
2610 spin_unlock(&dev_list_lock);
2611 } while (error == -EAGAIN);
2612
2613 if (error)
2614 return -ENODEV;
2615
2616 dev->instance = instance;
2617 return 0;
b60503ba
MW		 2618}
2619
2620static void nvme_release_instance(struct nvme_dev *dev)
2621{
cd58ad7d
QSA		 2622 spin_lock(&dev_list_lock);
2623 ida_remove(&nvme_instance_ida, dev->instance);
2624 spin_unlock(&dev_list_lock);
b60503ba
MW		 2625}
2626
9ac27090
KB		 2627static void nvme_free_namespaces(struct nvme_dev *dev)
2628{
2629 struct nvme_ns *ns, *next;
2630
2631 list_for_each_entry_safe(ns, next, &dev->namespaces, list) {
2632 list_del(&ns->list);
9e60352c
KB		 2633
2634 spin_lock(&dev_list_lock);
2635 ns->disk->private_data = NULL;
2636 spin_unlock(&dev_list_lock);
2637
9ac27090
KB		 2638 put_disk(ns->disk);
2639 kfree(ns);
2640 }
2641}
2642
5e82e952
KB		 2643static void nvme_free_dev(struct kref *kref)
2644{
2645 struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref);
9ac27090 2646
e75ec752 2647 put_device(dev->dev);
b3fffdef 2648 put_device(dev->device);
9ac27090 2649 nvme_free_namespaces(dev);
285dffc9 2650 nvme_release_instance(dev);
a4aea562 2651 blk_mq_free_tag_set(&dev->tagset);
ea191d2f 2652 blk_put_queue(dev->admin_q);
5e82e952
KB		 2653 kfree(dev->queues);
2654 kfree(dev->entry);
2655 kfree(dev);
2656}
2657
2658static int nvme_dev_open(struct inode *inode, struct file *f)
2659{
b3fffdef
KB		 2660 struct nvme_dev *dev;
2661 int instance = iminor(inode);
2662 int ret = -ENODEV;
2663
2664 spin_lock(&dev_list_lock);
2665 list_for_each_entry(dev, &dev_list, node) {
2666 if (dev->instance == instance) {
2e1d8448
KB		 2667 if (!dev->admin_q) {
2668 ret = -EWOULDBLOCK;
2669 break;
2670 }
b3fffdef
KB		 2671 if (!kref_get_unless_zero(&dev->kref))
2672 break;
2673 f->private_data = dev;
2674 ret = 0;
2675 break;
2676 }
2677 }
2678 spin_unlock(&dev_list_lock);
2679
2680 return ret;
5e82e952
KB		 2681}
2682
2683static int nvme_dev_release(struct inode *inode, struct file *f)
2684{
2685 struct nvme_dev *dev = f->private_data;
2686 kref_put(&dev->kref, nvme_free_dev);
2687 return 0;
2688}
2689
2690static long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
2691{
2692 struct nvme_dev *dev = f->private_data;
a4aea562
MB		 2693 struct nvme_ns *ns;
2694
5e82e952
KB		 2695 switch (cmd) {
2696 case NVME_IOCTL_ADMIN_CMD:
a4aea562 2697 return nvme_user_cmd(dev, NULL, (void __user *)arg);
7963e521 2698 case NVME_IOCTL_IO_CMD:
a4aea562
MB		 2699 if (list_empty(&dev->namespaces))
2700 return -ENOTTY;
2701 ns = list_first_entry(&dev->namespaces, struct nvme_ns, list);
2702 return nvme_user_cmd(dev, ns, (void __user *)arg);
5e82e952
KB		 2703 default:
2704 return -ENOTTY;
2705 }
2706}
2707
2708static const struct file_operations nvme_dev_fops = {
2709 .owner = THIS_MODULE,
2710 .open = nvme_dev_open,
2711 .release = nvme_dev_release,
2712 .unlocked_ioctl = nvme_dev_ioctl,
2713 .compat_ioctl = nvme_dev_ioctl,
2714};
2715
a4aea562
MB		 2716static void nvme_set_irq_hints(struct nvme_dev *dev)
2717{
2718 struct nvme_queue *nvmeq;
2719 int i;
2720
2721 for (i = 0; i < dev->online_queues; i++) {
2722 nvmeq = dev->queues[i];
2723
2724 if (!nvmeq->hctx)
2725 continue;
2726
2727 irq_set_affinity_hint(dev->entry[nvmeq->cq_vector].vector,
2728 nvmeq->hctx->cpumask);
2729 }
2730}
2731
f0b50732
KB		 2732static int nvme_dev_start(struct nvme_dev *dev)
2733{
2734 int result;
b9afca3e 2735 bool start_thread = false;
f0b50732
KB		 2736
2737 result = nvme_dev_map(dev);
2738 if (result)
2739 return result;
2740
2741 result = nvme_configure_admin_queue(dev);
2742 if (result)
2743 goto unmap;
2744
2745 spin_lock(&dev_list_lock);
b9afca3e
DM		 2746 if (list_empty(&dev_list) && IS_ERR_OR_NULL(nvme_thread)) {
2747 start_thread = true;
2748 nvme_thread = NULL;
2749 }
f0b50732
KB		 2750 list_add(&dev->node, &dev_list);
2751 spin_unlock(&dev_list_lock);
2752
b9afca3e
DM		 2753 if (start_thread) {
2754 nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");
387caa5a 2755 wake_up_all(&nvme_kthread_wait);
b9afca3e
DM		 2756 } else
2757 wait_event_killable(nvme_kthread_wait, nvme_thread);
2758
2759 if (IS_ERR_OR_NULL(nvme_thread)) {
2760 result = nvme_thread ? PTR_ERR(nvme_thread) : -EINTR;
2761 goto disable;
2762 }
a4aea562
MB		 2763
2764 nvme_init_queue(dev->queues[0], 0);
0fb59cbc
KB		 2765 result = nvme_alloc_admin_tags(dev);
2766 if (result)
2767 goto disable;
b9afca3e 2768
f0b50732 2769 result = nvme_setup_io_queues(dev);
badc34d4 2770 if (result)
0fb59cbc 2771 goto free_tags;
f0b50732 2772
a4aea562
MB		 2773 nvme_set_irq_hints(dev);
2774
1efccc9d 2775 dev->event_limit = 1;
d82e8bfd 2776 return result;
f0b50732 2777
0fb59cbc
KB		 2778 free_tags:
2779 nvme_dev_remove_admin(dev);
f0b50732 2780 disable:
a1a5ef99 2781 nvme_disable_queue(dev, 0);
b9afca3e 2782 nvme_dev_list_remove(dev);
f0b50732
KB		 2783 unmap:
2784 nvme_dev_unmap(dev);
2785 return result;
2786}
2787
9a6b9458
KB		 2788static int nvme_remove_dead_ctrl(void *arg)
2789{
2790 struct nvme_dev *dev = (struct nvme_dev *)arg;
e75ec752 2791 struct pci_dev *pdev = to_pci_dev(dev->dev);
9a6b9458
KB		 2792
2793 if (pci_get_drvdata(pdev))
c81f4975 2794 pci_stop_and_remove_bus_device_locked(pdev);
9a6b9458
KB		 2795 kref_put(&dev->kref, nvme_free_dev);
2796 return 0;
2797}
2798
2799static void nvme_remove_disks(struct work_struct *ws)
2800{
9a6b9458
KB		 2801 struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work);
2802
5a92e700 2803 nvme_free_queues(dev, 1);
302c6727 2804 nvme_dev_remove(dev);
9a6b9458
KB		 2805}
2806
2807static int nvme_dev_resume(struct nvme_dev *dev)
2808{
2809 int ret;
2810
2811 ret = nvme_dev_start(dev);
badc34d4 2812 if (ret)
9a6b9458 2813 return ret;
badc34d4 2814 if (dev->online_queues < 2) {
9a6b9458 2815 spin_lock(&dev_list_lock);
9ca97374 2816 dev->reset_workfn = nvme_remove_disks;
9a6b9458
KB		 2817 queue_work(nvme_workq, &dev->reset_work);
2818 spin_unlock(&dev_list_lock);
c9d3bf88
KB		 2819 } else {
2820 nvme_unfreeze_queues(dev);
2821 nvme_set_irq_hints(dev);
9a6b9458
KB		 2822 }
2823 return 0;
2824}
2825
2826static void nvme_dev_reset(struct nvme_dev *dev)
2827{
2828 nvme_dev_shutdown(dev);
2829 if (nvme_dev_resume(dev)) {
e75ec752 2830 dev_warn(dev->dev, "Device failed to resume\n");
9a6b9458
KB		 2831 kref_get(&dev->kref);
2832 if (IS_ERR(kthread_run(nvme_remove_dead_ctrl, dev, "nvme%d",
2833 dev->instance))) {
e75ec752 2834 dev_err(dev->dev,
9a6b9458
KB		 2835 "Failed to start controller remove task\n");
2836 kref_put(&dev->kref, nvme_free_dev);
2837 }
2838 }
2839}
2840
2841static void nvme_reset_failed_dev(struct work_struct *ws)
2842{
2843 struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work);
2844 nvme_dev_reset(dev);
2845}
2846
9ca97374
TH		 2847static void nvme_reset_workfn(struct work_struct *work)
2848{
2849 struct nvme_dev *dev = container_of(work, struct nvme_dev, reset_work);
2850 dev->reset_workfn(work);
2851}
2852
2e1d8448 2853static void nvme_async_probe(struct work_struct *work);
8d85fce7 2854static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
b60503ba 2855{
a4aea562 2856 int node, result = -ENOMEM;
b60503ba
MW		 2857 struct nvme_dev *dev;
2858
a4aea562
MB		 2859 node = dev_to_node(&pdev->dev);
2860 if (node == NUMA_NO_NODE)
2861 set_dev_node(&pdev->dev, 0);
2862
2863 dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
b60503ba
MW		 2864 if (!dev)
2865 return -ENOMEM;
a4aea562
MB		 2866 dev->entry = kzalloc_node(num_possible_cpus() * sizeof(*dev->entry),
2867 GFP_KERNEL, node);
b60503ba
MW		 2868 if (!dev->entry)
2869 goto free;
a4aea562
MB		 2870 dev->queues = kzalloc_node((num_possible_cpus() + 1) * sizeof(void *),
2871 GFP_KERNEL, node);
b60503ba
MW		 2872 if (!dev->queues)
2873 goto free;
2874
2875 INIT_LIST_HEAD(&dev->namespaces);
9ca97374
TH		 2876 dev->reset_workfn = nvme_reset_failed_dev;
2877 INIT_WORK(&dev->reset_work, nvme_reset_workfn);
e75ec752 2878 dev->dev = get_device(&pdev->dev);
9a6b9458 2879 pci_set_drvdata(pdev, dev);
cd58ad7d
QSA		 2880 result = nvme_set_instance(dev);
2881 if (result)
a96d4f5c 2882 goto put_pci;
b60503ba 2883
091b6092
MW		 2884 result = nvme_setup_prp_pools(dev);
2885 if (result)
0877cb0d 2886 goto release;
091b6092 2887
fb35e914 2888 kref_init(&dev->kref);
b3fffdef
KB		 2889 dev->device = device_create(nvme_class, &pdev->dev,
2890 MKDEV(nvme_char_major, dev->instance),
2891 dev, "nvme%d", dev->instance);
2892 if (IS_ERR(dev->device)) {
2893 result = PTR_ERR(dev->device);
2e1d8448 2894 goto release_pools;
b3fffdef
KB		 2895 }
2896 get_device(dev->device);
740216fc 2897
e6e96d73 2898 INIT_LIST_HEAD(&dev->node);
2e1d8448
KB		 2899 INIT_WORK(&dev->probe_work, nvme_async_probe);
2900 schedule_work(&dev->probe_work);
b60503ba
MW		 2901 return 0;
2902
0877cb0d 2903 release_pools:
091b6092 2904 nvme_release_prp_pools(dev);
0877cb0d
KB		 2905 release:
2906 nvme_release_instance(dev);
a96d4f5c 2907 put_pci:
e75ec752 2908 put_device(dev->dev);
b60503ba
MW		 2909 free:
2910 kfree(dev->queues);
2911 kfree(dev->entry);
2912 kfree(dev);
2913 return result;
2914}
2915
2e1d8448
KB		 2916static void nvme_async_probe(struct work_struct *work)
2917{
2918 struct nvme_dev *dev = container_of(work, struct nvme_dev, probe_work);
2919 int result;
2920
2921 result = nvme_dev_start(dev);
2922 if (result)
2923 goto reset;
2924
2925 if (dev->online_queues > 1)
2926 result = nvme_dev_add(dev);
2927 if (result)
2928 goto reset;
2929
2930 nvme_set_irq_hints(dev);
2e1d8448
KB		 2931 return;
2932 reset:
07836e65
KB		 2933 if (!work_busy(&dev->reset_work)) {
2934 dev->reset_workfn = nvme_reset_failed_dev;
2935 queue_work(nvme_workq, &dev->reset_work);
2936 }
2e1d8448
KB		 2937}
2938
f0d54a54
KB		 2939static void nvme_reset_notify(struct pci_dev *pdev, bool prepare)
2940{
a6739479 2941 struct nvme_dev *dev = pci_get_drvdata(pdev);
f0d54a54 2942
a6739479
KB		 2943 if (prepare)
2944 nvme_dev_shutdown(dev);
2945 else
2946 nvme_dev_resume(dev);
f0d54a54
KB		 2947}
2948
09ece142
KB		 2949static void nvme_shutdown(struct pci_dev *pdev)
2950{
2951 struct nvme_dev *dev = pci_get_drvdata(pdev);
2952 nvme_dev_shutdown(dev);
2953}
2954
8d85fce7 2955static void nvme_remove(struct pci_dev *pdev)
b60503ba
MW		 2956{
2957 struct nvme_dev *dev = pci_get_drvdata(pdev);
9a6b9458
KB		 2958
2959 spin_lock(&dev_list_lock);
2960 list_del_init(&dev->node);
2961 spin_unlock(&dev_list_lock);
2962
2963 pci_set_drvdata(pdev, NULL);
2e1d8448 2964 flush_work(&dev->probe_work);
9a6b9458 2965 flush_work(&dev->reset_work);
9a6b9458 2966 nvme_dev_shutdown(dev);
c9d3bf88 2967 nvme_dev_remove(dev);
a4aea562 2968 nvme_dev_remove_admin(dev);
b3fffdef 2969 device_destroy(nvme_class, MKDEV(nvme_char_major, dev->instance));
a1a5ef99 2970 nvme_free_queues(dev, 0);
9a6b9458 2971 nvme_release_prp_pools(dev);
5e82e952 2972 kref_put(&dev->kref, nvme_free_dev);
b60503ba
MW		 2973}
2974
2975/* These functions are yet to be implemented */
2976#define nvme_error_detected NULL
2977#define nvme_dump_registers NULL
2978#define nvme_link_reset NULL
2979#define nvme_slot_reset NULL
2980#define nvme_error_resume NULL
cd638946 2981
671a6018 2982#ifdef CONFIG_PM_SLEEP
cd638946
KB		 2983static int nvme_suspend(struct device *dev)
2984{
2985 struct pci_dev *pdev = to_pci_dev(dev);
2986 struct nvme_dev *ndev = pci_get_drvdata(pdev);
2987
2988 nvme_dev_shutdown(ndev);
2989 return 0;
2990}
2991
2992static int nvme_resume(struct device *dev)
2993{
2994 struct pci_dev *pdev = to_pci_dev(dev);
2995 struct nvme_dev *ndev = pci_get_drvdata(pdev);
cd638946 2996
9a6b9458 2997 if (nvme_dev_resume(ndev) && !work_busy(&ndev->reset_work)) {
9ca97374 2998 ndev->reset_workfn = nvme_reset_failed_dev;
9a6b9458
KB		 2999 queue_work(nvme_workq, &ndev->reset_work);
3000 }
3001 return 0;
cd638946 3002}
671a6018 3003#endif
cd638946
KB		 3004
3005static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);
b60503ba 3006
1d352035 3007static const struct pci_error_handlers nvme_err_handler = {
b60503ba
MW		 3008 .error_detected = nvme_error_detected,
3009 .mmio_enabled = nvme_dump_registers,
3010 .link_reset = nvme_link_reset,
3011 .slot_reset = nvme_slot_reset,
3012 .resume = nvme_error_resume,
f0d54a54 3013 .reset_notify = nvme_reset_notify,
b60503ba
MW		 3014};
3015
3016/* Move to pci_ids.h later */
3017#define PCI_CLASS_STORAGE_EXPRESS 0x010802
3018
6eb0d698 3019static const struct pci_device_id nvme_id_table[] = {
b60503ba
MW		 3020 { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
3021 { 0, }
3022};
3023MODULE_DEVICE_TABLE(pci, nvme_id_table);
3024
3025static struct pci_driver nvme_driver = {
3026 .name = "nvme",
3027 .id_table = nvme_id_table,
3028 .probe = nvme_probe,
8d85fce7 3029 .remove = nvme_remove,
09ece142 3030 .shutdown = nvme_shutdown,
cd638946
KB		 3031 .driver = {
3032 .pm = &nvme_dev_pm_ops,
3033 },
b60503ba
MW		 3034 .err_handler = &nvme_err_handler,
3035};
3036
3037static int __init nvme_init(void)
3038{
0ac13140 3039 int result;
1fa6aead 3040
b9afca3e 3041 init_waitqueue_head(&nvme_kthread_wait);
b60503ba 3042
9a6b9458
KB		 3043 nvme_workq = create_singlethread_workqueue("nvme");
3044 if (!nvme_workq)
b9afca3e 3045 return -ENOMEM;
9a6b9458 3046
5c42ea16
KB		 3047 result = register_blkdev(nvme_major, "nvme");
3048 if (result < 0)
9a6b9458 3049 goto kill_workq;
5c42ea16 3050 else if (result > 0)
0ac13140 3051 nvme_major = result;
b60503ba 3052
b3fffdef
KB		 3053 result = __register_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme",
3054 &nvme_dev_fops);
3055 if (result < 0)
3056 goto unregister_blkdev;
3057 else if (result > 0)
3058 nvme_char_major = result;
3059
3060 nvme_class = class_create(THIS_MODULE, "nvme");
c727040b
AK		 3061 if (IS_ERR(nvme_class)) {
3062 result = PTR_ERR(nvme_class);
b3fffdef 3063 goto unregister_chrdev;
c727040b 3064 }
b3fffdef 3065
f3db22fe
KB		 3066 result = pci_register_driver(&nvme_driver);
3067 if (result)
b3fffdef 3068 goto destroy_class;
1fa6aead 3069 return 0;
b60503ba 3070
b3fffdef
KB		 3071 destroy_class:
3072 class_destroy(nvme_class);
3073 unregister_chrdev:
3074 __unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
1fa6aead 3075 unregister_blkdev:
b60503ba 3076 unregister_blkdev(nvme_major, "nvme");
9a6b9458
KB		 3077 kill_workq:
3078 destroy_workqueue(nvme_workq);
b60503ba
MW		 3079 return result;
3080}
3081
3082static void __exit nvme_exit(void)
3083{
3084 pci_unregister_driver(&nvme_driver);
3085 unregister_blkdev(nvme_major, "nvme");
9a6b9458 3086 destroy_workqueue(nvme_workq);
b3fffdef
KB		 3087 class_destroy(nvme_class);
3088 __unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
b9afca3e 3089 BUG_ON(nvme_thread && !IS_ERR(nvme_thread));
21bd78bc 3090 _nvme_check_size();
b60503ba
MW		 3091}
3092
3093MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
3094MODULE_LICENSE("GPL");
c78b4713 3095MODULE_VERSION("1.0");
b60503ba
MW		 3096module_init(nvme_init);
3097module_exit(nvme_exit);
Linux kernel - Deliverables
This page took 0.356123 seconds and 5 git commands to generate.