projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
NVMe: Update SCSI Inquiry VPD 83h translation
[deliverable/linux.git] / drivers / block / nvme-core.c
CommitLineData
b60503ba
MW		 1/*
2 * NVM Express device driver
6eb0d698 3 * Copyright (c) 2011-2014, Intel Corporation.
b60503ba
MW		 4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
b60503ba
MW		 13 */
14
15#include <linux/nvme.h>
8de05535 16#include <linux/bitops.h>
b60503ba 17#include <linux/blkdev.h>
a4aea562 18#include <linux/blk-mq.h>
42f61420 19#include <linux/cpu.h>
fd63e9ce 20#include <linux/delay.h>
b60503ba
MW		 21#include <linux/errno.h>
22#include <linux/fs.h>
23#include <linux/genhd.h>
4cc09e2d 24#include <linux/hdreg.h>
5aff9382 25#include <linux/idr.h>
b60503ba
MW		 26#include <linux/init.h>
27#include <linux/interrupt.h>
28#include <linux/io.h>
29#include <linux/kdev_t.h>
1fa6aead 30#include <linux/kthread.h>
b60503ba
MW		 31#include <linux/kernel.h>
32#include <linux/mm.h>
33#include <linux/module.h>
34#include <linux/moduleparam.h>
35#include <linux/pci.h>
be7b6275 36#include <linux/poison.h>
c3bfe717 37#include <linux/ptrace.h>
b60503ba
MW		 38#include <linux/sched.h>
39#include <linux/slab.h>
e1e5e564 40#include <linux/t10-pi.h>
b60503ba 41#include <linux/types.h>
5d0f6131 42#include <scsi/sg.h>
797a796a
HM		 43#include <asm-generic/io-64-nonatomic-lo-hi.h>
44
9d43cf64 45#define NVME_Q_DEPTH 1024
a4aea562 46#define NVME_AQ_DEPTH 64
b60503ba
MW		 47#define SQ_SIZE(depth) (depth * sizeof(struct nvme_command))
48#define CQ_SIZE(depth) (depth * sizeof(struct nvme_completion))
9d43cf64 49#define ADMIN_TIMEOUT (admin_timeout * HZ)
2484f407 50#define SHUTDOWN_TIMEOUT (shutdown_timeout * HZ)
9d43cf64
KB		 51#define IOD_TIMEOUT (retry_time * HZ)
52
53static unsigned char admin_timeout = 60;
54module_param(admin_timeout, byte, 0644);
55MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
b60503ba 56
bd67608a
MW		 57unsigned char nvme_io_timeout = 30;
58module_param_named(io_timeout, nvme_io_timeout, byte, 0644);
b355084a 59MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
b60503ba 60
61e4ce08
KB		 61static unsigned char retry_time = 30;
62module_param(retry_time, byte, 0644);
63MODULE_PARM_DESC(retry_time, "time in seconds to retry failed I/O");
64
2484f407
DM		 65static unsigned char shutdown_timeout = 5;
66module_param(shutdown_timeout, byte, 0644);
67MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
68
b60503ba
MW		 69static int nvme_major;
70module_param(nvme_major, int, 0);
71
58ffacb5
MW		 72static int use_threaded_interrupts;
73module_param(use_threaded_interrupts, int, 0);
74
1fa6aead
MW		 75static DEFINE_SPINLOCK(dev_list_lock);
76static LIST_HEAD(dev_list);
77static struct task_struct *nvme_thread;
9a6b9458 78static struct workqueue_struct *nvme_workq;
b9afca3e 79static wait_queue_head_t nvme_kthread_wait;
f3db22fe 80static struct notifier_block nvme_nb;
1fa6aead 81
d4b4ff8e 82static void nvme_reset_failed_dev(struct work_struct *ws);
a4aea562 83static int nvme_process_cq(struct nvme_queue *nvmeq);
d4b4ff8e 84
4d115420
KB		 85struct async_cmd_info {
86 struct kthread_work work;
87 struct kthread_worker *worker;
a4aea562 88 struct request *req;
4d115420
KB		 89 u32 result;
90 int status;
91 void *ctx;
92};
1fa6aead 93
b60503ba
MW		 94/*
95 * An NVM Express queue. Each device has at least two (one for admin
96 * commands and one for I/O commands).
97 */
98struct nvme_queue {
f435c282 99 struct llist_node node;
b60503ba 100 struct device *q_dmadev;
091b6092 101 struct nvme_dev *dev;
3193f07b 102 char irqname[24]; /* nvme4294967295-65535\0 */
b60503ba
MW		 103 spinlock_t q_lock;
104 struct nvme_command *sq_cmds;
105 volatile struct nvme_completion *cqes;
106 dma_addr_t sq_dma_addr;
107 dma_addr_t cq_dma_addr;
b60503ba
MW		 108 u32 __iomem *q_db;
109 u16 q_depth;
6222d172 110 s16 cq_vector;
b60503ba
MW		 111 u16 sq_head;
112 u16 sq_tail;
113 u16 cq_head;
c30341dc 114 u16 qid;
e9539f47
MW		 115 u8 cq_phase;
116 u8 cqe_seen;
4d115420 117 struct async_cmd_info cmdinfo;
a4aea562 118 struct blk_mq_hw_ctx *hctx;
b60503ba
MW		 119};
120
121/*
122 * Check we didin't inadvertently grow the command struct
123 */
124static inline void _nvme_check_size(void)
125{
126 BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
127 BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
128 BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
129 BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
130 BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
f8ebf840 131 BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
c30341dc 132 BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
b60503ba
MW		 133 BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
134 BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096);
135 BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096);
136 BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
6ecec745 137 BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
b60503ba
MW		 138}
139
edd10d33 140typedef void (*nvme_completion_fn)(struct nvme_queue *, void *,
c2f5b650
MW		 141 struct nvme_completion *);
142
e85248e5 143struct nvme_cmd_info {
c2f5b650
MW		 144 nvme_completion_fn fn;
145 void *ctx;
c30341dc 146 int aborted;
a4aea562 147 struct nvme_queue *nvmeq;
ac3dd5bd 148 struct nvme_iod iod[0];
e85248e5
MW		 149};
150
ac3dd5bd
JA		 151/*
152 * Max size of iod being embedded in the request payload
153 */
154#define NVME_INT_PAGES 2
155#define NVME_INT_BYTES(dev) (NVME_INT_PAGES * (dev)->page_size)
156
157/*
158 * Will slightly overestimate the number of pages needed. This is OK
159 * as it only leads to a small amount of wasted memory for the lifetime of
160 * the I/O.
161 */
162static int nvme_npages(unsigned size, struct nvme_dev *dev)
163{
164 unsigned nprps = DIV_ROUND_UP(size + dev->page_size, dev->page_size);
165 return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
166}
167
168static unsigned int nvme_cmd_size(struct nvme_dev *dev)
169{
170 unsigned int ret = sizeof(struct nvme_cmd_info);
171
172 ret += sizeof(struct nvme_iod);
173 ret += sizeof(__le64 *) * nvme_npages(NVME_INT_BYTES(dev), dev);
174 ret += sizeof(struct scatterlist) * NVME_INT_PAGES;
175
176 return ret;
177}
178
a4aea562
MB		 179static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
180 unsigned int hctx_idx)
e85248e5 181{
a4aea562
MB		 182 struct nvme_dev *dev = data;
183 struct nvme_queue *nvmeq = dev->queues[0];
184
185 WARN_ON(nvmeq->hctx);
186 nvmeq->hctx = hctx;
187 hctx->driver_data = nvmeq;
188 return 0;
e85248e5
MW		 189}
190
a4aea562
MB		 191static int nvme_admin_init_request(void *data, struct request *req,
192 unsigned int hctx_idx, unsigned int rq_idx,
193 unsigned int numa_node)
22404274 194{
a4aea562
MB		 195 struct nvme_dev *dev = data;
196 struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
197 struct nvme_queue *nvmeq = dev->queues[0];
198
199 BUG_ON(!nvmeq);
200 cmd->nvmeq = nvmeq;
201 return 0;
22404274
KB		 202}
203
2c30540b
JA		 204static void nvme_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
205{
206 struct nvme_queue *nvmeq = hctx->driver_data;
207
208 nvmeq->hctx = NULL;
209}
210
a4aea562
MB		 211static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
212 unsigned int hctx_idx)
b60503ba 213{
a4aea562
MB		 214 struct nvme_dev *dev = data;
215 struct nvme_queue *nvmeq = dev->queues[
216 (hctx_idx % dev->queue_count) + 1];
b60503ba 217
a4aea562
MB		 218 if (!nvmeq->hctx)
219 nvmeq->hctx = hctx;
220
221 /* nvmeq queues are shared between namespaces. We assume here that
222 * blk-mq map the tags so they match up with the nvme queue tags. */
223 WARN_ON(nvmeq->hctx->tags != hctx->tags);
b60503ba 224
a4aea562
MB		 225 hctx->driver_data = nvmeq;
226 return 0;
b60503ba
MW		 227}
228
a4aea562
MB		 229static int nvme_init_request(void *data, struct request *req,
230 unsigned int hctx_idx, unsigned int rq_idx,
231 unsigned int numa_node)
b60503ba 232{
a4aea562
MB		 233 struct nvme_dev *dev = data;
234 struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
235 struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1];
236
237 BUG_ON(!nvmeq);
238 cmd->nvmeq = nvmeq;
239 return 0;
240}
241
242static void nvme_set_info(struct nvme_cmd_info *cmd, void *ctx,
243 nvme_completion_fn handler)
244{
245 cmd->fn = handler;
246 cmd->ctx = ctx;
247 cmd->aborted = 0;
c917dfe5 248 blk_mq_start_request(blk_mq_rq_from_pdu(cmd));
b60503ba
MW		 249}
250
ac3dd5bd
JA		 251static void *iod_get_private(struct nvme_iod *iod)
252{
253 return (void *) (iod->private & ~0x1UL);
254}
255
256/*
257 * If bit 0 is set, the iod is embedded in the request payload.
258 */
259static bool iod_should_kfree(struct nvme_iod *iod)
260{
261 return (iod->private & 0x01) == 0;
262}
263
c2f5b650
MW		 264/* Special values must be less than 0x1000 */
265#define CMD_CTX_BASE ((void *)POISON_POINTER_DELTA)
d2d87034
MW		 266#define CMD_CTX_CANCELLED (0x30C + CMD_CTX_BASE)
267#define CMD_CTX_COMPLETED (0x310 + CMD_CTX_BASE)
268#define CMD_CTX_INVALID (0x314 + CMD_CTX_BASE)
be7b6275 269
edd10d33 270static void special_completion(struct nvme_queue *nvmeq, void *ctx,
c2f5b650
MW		 271 struct nvme_completion *cqe)
272{
273 if (ctx == CMD_CTX_CANCELLED)
274 return;
c2f5b650 275 if (ctx == CMD_CTX_COMPLETED) {
edd10d33 276 dev_warn(nvmeq->q_dmadev,
c2f5b650
MW		 277 "completed id %d twice on queue %d\n",
278 cqe->command_id, le16_to_cpup(&cqe->sq_id));
279 return;
280 }
281 if (ctx == CMD_CTX_INVALID) {
edd10d33 282 dev_warn(nvmeq->q_dmadev,
c2f5b650
MW		 283 "invalid id %d completed on queue %d\n",
284 cqe->command_id, le16_to_cpup(&cqe->sq_id));
285 return;
286 }
edd10d33 287 dev_warn(nvmeq->q_dmadev, "Unknown special completion %p\n", ctx);
c2f5b650
MW		 288}
289
a4aea562 290static void *cancel_cmd_info(struct nvme_cmd_info *cmd, nvme_completion_fn *fn)
b60503ba 291{
c2f5b650 292 void *ctx;
b60503ba 293
859361a2 294 if (fn)
a4aea562
MB		 295 *fn = cmd->fn;
296 ctx = cmd->ctx;
297 cmd->fn = special_completion;
298 cmd->ctx = CMD_CTX_CANCELLED;
c2f5b650 299 return ctx;
b60503ba
MW		 300}
301
a4aea562
MB		 302static void async_req_completion(struct nvme_queue *nvmeq, void *ctx,
303 struct nvme_completion *cqe)
3c0cf138 304{
a4aea562 305 struct request *req = ctx;
3c0cf138 306
a4aea562
MB		 307 u32 result = le32_to_cpup(&cqe->result);
308 u16 status = le16_to_cpup(&cqe->status) >> 1;
309
310 if (status == NVME_SC_SUCCESS || status == NVME_SC_ABORT_REQ)
311 ++nvmeq->dev->event_limit;
312 if (status == NVME_SC_SUCCESS)
313 dev_warn(nvmeq->q_dmadev,
314 "async event result %08x\n", result);
315
9d135bb8 316 blk_mq_free_hctx_request(nvmeq->hctx, req);
b60503ba
MW		 317}
318
a4aea562
MB		 319static void abort_completion(struct nvme_queue *nvmeq, void *ctx,
320 struct nvme_completion *cqe)
5a92e700 321{
a4aea562
MB		 322 struct request *req = ctx;
323
324 u16 status = le16_to_cpup(&cqe->status) >> 1;
325 u32 result = le32_to_cpup(&cqe->result);
a51afb54 326
9d135bb8 327 blk_mq_free_hctx_request(nvmeq->hctx, req);
a51afb54 328
a4aea562
MB		 329 dev_warn(nvmeq->q_dmadev, "Abort status:%x result:%x", status, result);
330 ++nvmeq->dev->abort_limit;
5a92e700
KB		 331}
332
a4aea562
MB		 333static void async_completion(struct nvme_queue *nvmeq, void *ctx,
334 struct nvme_completion *cqe)
b60503ba 335{
a4aea562
MB		 336 struct async_cmd_info *cmdinfo = ctx;
337 cmdinfo->result = le32_to_cpup(&cqe->result);
338 cmdinfo->status = le16_to_cpup(&cqe->status) >> 1;
339 queue_kthread_work(cmdinfo->worker, &cmdinfo->work);
9d135bb8 340 blk_mq_free_hctx_request(nvmeq->hctx, cmdinfo->req);
b60503ba
MW		 341}
342
a4aea562
MB		 343static inline struct nvme_cmd_info *get_cmd_from_tag(struct nvme_queue *nvmeq,
344 unsigned int tag)
b60503ba 345{
a4aea562
MB		 346 struct blk_mq_hw_ctx *hctx = nvmeq->hctx;
347 struct request *req = blk_mq_tag_to_rq(hctx->tags, tag);
a51afb54 348
a4aea562 349 return blk_mq_rq_to_pdu(req);
4f5099af
KB		 350}
351
a4aea562
MB		 352/*
353 * Called with local interrupts disabled and the q_lock held. May not sleep.
354 */
355static void *nvme_finish_cmd(struct nvme_queue *nvmeq, int tag,
356 nvme_completion_fn *fn)
4f5099af 357{
a4aea562
MB		 358 struct nvme_cmd_info *cmd = get_cmd_from_tag(nvmeq, tag);
359 void *ctx;
360 if (tag >= nvmeq->q_depth) {
361 *fn = special_completion;
362 return CMD_CTX_INVALID;
363 }
364 if (fn)
365 *fn = cmd->fn;
366 ctx = cmd->ctx;
367 cmd->fn = special_completion;
368 cmd->ctx = CMD_CTX_COMPLETED;
369 return ctx;
b60503ba
MW		 370}
371
372/**
714a7a22 373 * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
b60503ba
MW		 374 * @nvmeq: The queue to use
375 * @cmd: The command to send
376 *
377 * Safe to use from interrupt context
378 */
a4aea562 379static int __nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
b60503ba 380{
a4aea562
MB		 381 u16 tail = nvmeq->sq_tail;
382
b60503ba 383 memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));
b60503ba
MW		 384 if (++tail == nvmeq->q_depth)
385 tail = 0;
7547881d 386 writel(tail, nvmeq->q_db);
b60503ba 387 nvmeq->sq_tail = tail;
b60503ba
MW		 388
389 return 0;
390}
391
a4aea562
MB		 392static int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
393{
394 unsigned long flags;
395 int ret;
396 spin_lock_irqsave(&nvmeq->q_lock, flags);
397 ret = __nvme_submit_cmd(nvmeq, cmd);
398 spin_unlock_irqrestore(&nvmeq->q_lock, flags);
399 return ret;
400}
401
eca18b23 402static __le64 **iod_list(struct nvme_iod *iod)
e025344c 403{
eca18b23 404 return ((void *)iod) + iod->offset;
e025344c
SMM		 405}
406
ac3dd5bd
JA		 407static inline void iod_init(struct nvme_iod *iod, unsigned nbytes,
408 unsigned nseg, unsigned long private)
eca18b23 409{
ac3dd5bd
JA		 410 iod->private = private;
411 iod->offset = offsetof(struct nvme_iod, sg[nseg]);
412 iod->npages = -1;
413 iod->length = nbytes;
414 iod->nents = 0;
eca18b23 415}
b60503ba 416
eca18b23 417static struct nvme_iod *
ac3dd5bd
JA		 418__nvme_alloc_iod(unsigned nseg, unsigned bytes, struct nvme_dev *dev,
419 unsigned long priv, gfp_t gfp)
b60503ba 420{
eca18b23 421 struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) +
ac3dd5bd 422 sizeof(__le64 *) * nvme_npages(bytes, dev) +
eca18b23
MW		 423 sizeof(struct scatterlist) * nseg, gfp);
424
ac3dd5bd
JA		 425 if (iod)
426 iod_init(iod, bytes, nseg, priv);
eca18b23
MW		 427
428 return iod;
b60503ba
MW		 429}
430
ac3dd5bd
JA		 431static struct nvme_iod *nvme_alloc_iod(struct request *rq, struct nvme_dev *dev,
432 gfp_t gfp)
433{
434 unsigned size = !(rq->cmd_flags & REQ_DISCARD) ? blk_rq_bytes(rq) :
435 sizeof(struct nvme_dsm_range);
436 unsigned long mask = 0;
437 struct nvme_iod *iod;
438
439 if (rq->nr_phys_segments <= NVME_INT_PAGES &&
440 size <= NVME_INT_BYTES(dev)) {
441 struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(rq);
442
443 iod = cmd->iod;
444 mask = 0x01;
445 iod_init(iod, size, rq->nr_phys_segments,
446 (unsigned long) rq | 0x01);
447 return iod;
448 }
449
450 return __nvme_alloc_iod(rq->nr_phys_segments, size, dev,
451 (unsigned long) rq, gfp);
452}
453
5d0f6131 454void nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod)
b60503ba 455{
1d090624 456 const int last_prp = dev->page_size / 8 - 1;
eca18b23
MW		 457 int i;
458 __le64 **list = iod_list(iod);
459 dma_addr_t prp_dma = iod->first_dma;
460
461 if (iod->npages == 0)
462 dma_pool_free(dev->prp_small_pool, list[0], prp_dma);
463 for (i = 0; i < iod->npages; i++) {
464 __le64 *prp_list = list[i];
465 dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]);
466 dma_pool_free(dev->prp_page_pool, prp_list, prp_dma);
467 prp_dma = next_prp_dma;
468 }
ac3dd5bd
JA		 469
470 if (iod_should_kfree(iod))
471 kfree(iod);
b60503ba
MW		 472}
473
b4ff9c8d
KB		 474static int nvme_error_status(u16 status)
475{
476 switch (status & 0x7ff) {
477 case NVME_SC_SUCCESS:
478 return 0;
479 case NVME_SC_CAP_EXCEEDED:
480 return -ENOSPC;
481 default:
482 return -EIO;
483 }
484}
485
e1e5e564
KB		 486static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
487{
488 if (be32_to_cpu(pi->ref_tag) == v)
489 pi->ref_tag = cpu_to_be32(p);
490}
491
492static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
493{
494 if (be32_to_cpu(pi->ref_tag) == p)
495 pi->ref_tag = cpu_to_be32(v);
496}
497
498/**
499 * nvme_dif_remap - remaps ref tags to bip seed and physical lba
500 *
501 * The virtual start sector is the one that was originally submitted by the
502 * block layer. Due to partitioning, MD/DM cloning, etc. the actual physical
503 * start sector may be different. Remap protection information to match the
504 * physical LBA on writes, and back to the original seed on reads.
505 *
506 * Type 0 and 3 do not have a ref tag, so no remapping required.
507 */
508static void nvme_dif_remap(struct request *req,
509 void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
510{
511 struct nvme_ns *ns = req->rq_disk->private_data;
512 struct bio_integrity_payload *bip;
513 struct t10_pi_tuple *pi;
514 void *p, *pmap;
515 u32 i, nlb, ts, phys, virt;
516
517 if (!ns->pi_type || ns->pi_type == NVME_NS_DPS_PI_TYPE3)
518 return;
519
520 bip = bio_integrity(req->bio);
521 if (!bip)
522 return;
523
524 pmap = kmap_atomic(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset;
525 if (!pmap)
526 return;
527
528 p = pmap;
529 virt = bip_get_seed(bip);
530 phys = nvme_block_nr(ns, blk_rq_pos(req));
531 nlb = (blk_rq_bytes(req) >> ns->lba_shift);
532 ts = ns->disk->integrity->tuple_size;
533
534 for (i = 0; i < nlb; i++, virt++, phys++) {
535 pi = (struct t10_pi_tuple *)p;
536 dif_swap(phys, virt, pi);
537 p += ts;
538 }
539 kunmap_atomic(pmap);
540}
541
a4aea562 542static void req_completion(struct nvme_queue *nvmeq, void *ctx,
b60503ba
MW		 543 struct nvme_completion *cqe)
544{
eca18b23 545 struct nvme_iod *iod = ctx;
ac3dd5bd 546 struct request *req = iod_get_private(iod);
a4aea562
MB		 547 struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
548
b60503ba
MW		 549 u16 status = le16_to_cpup(&cqe->status) >> 1;
550
edd10d33 551 if (unlikely(status)) {
a4aea562
MB		 552 if (!(status & NVME_SC_DNR || blk_noretry_request(req))
553 && (jiffies - req->start_time) < req->timeout) {
c9d3bf88
KB		 554 unsigned long flags;
555
a4aea562 556 blk_mq_requeue_request(req);
c9d3bf88
KB		 557 spin_lock_irqsave(req->q->queue_lock, flags);
558 if (!blk_queue_stopped(req->q))
559 blk_mq_kick_requeue_list(req->q);
560 spin_unlock_irqrestore(req->q->queue_lock, flags);
edd10d33
KB		 561 return;
562 }
a4aea562
MB		 563 req->errors = nvme_error_status(status);
564 } else
565 req->errors = 0;
566
567 if (cmd_rq->aborted)
568 dev_warn(&nvmeq->dev->pci_dev->dev,
569 "completing aborted command with status:%04x\n",
570 status);
571
e1e5e564 572 if (iod->nents) {
a4aea562
MB		 573 dma_unmap_sg(&nvmeq->dev->pci_dev->dev, iod->sg, iod->nents,
574 rq_data_dir(req) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
e1e5e564
KB		 575 if (blk_integrity_rq(req)) {
576 if (!rq_data_dir(req))
577 nvme_dif_remap(req, nvme_dif_complete);
578 dma_unmap_sg(&nvmeq->dev->pci_dev->dev, iod->meta_sg, 1,
579 rq_data_dir(req) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
580 }
581 }
edd10d33 582 nvme_free_iod(nvmeq->dev, iod);
3291fa57 583
a4aea562 584 blk_mq_complete_request(req);
b60503ba
MW		 585}
586
184d2944 587/* length is in bytes. gfp flags indicates whether we may sleep. */
edd10d33
KB		 588int nvme_setup_prps(struct nvme_dev *dev, struct nvme_iod *iod, int total_len,
589 gfp_t gfp)
ff22b54f 590{
99802a7a 591 struct dma_pool *pool;
eca18b23
MW		 592 int length = total_len;
593 struct scatterlist *sg = iod->sg;
ff22b54f
MW		 594 int dma_len = sg_dma_len(sg);
595 u64 dma_addr = sg_dma_address(sg);
596 int offset = offset_in_page(dma_addr);
e025344c 597 __le64 *prp_list;
eca18b23 598 __le64 **list = iod_list(iod);
e025344c 599 dma_addr_t prp_dma;
eca18b23 600 int nprps, i;
1d090624 601 u32 page_size = dev->page_size;
ff22b54f 602
1d090624 603 length -= (page_size - offset);
ff22b54f 604 if (length <= 0)
eca18b23 605 return total_len;
ff22b54f 606
1d090624 607 dma_len -= (page_size - offset);
ff22b54f 608 if (dma_len) {
1d090624 609 dma_addr += (page_size - offset);
ff22b54f
MW		 610 } else {
611 sg = sg_next(sg);
612 dma_addr = sg_dma_address(sg);
613 dma_len = sg_dma_len(sg);
614 }
615
1d090624 616 if (length <= page_size) {
edd10d33 617 iod->first_dma = dma_addr;
eca18b23 618 return total_len;
e025344c
SMM		 619 }
620
1d090624 621 nprps = DIV_ROUND_UP(length, page_size);
99802a7a
MW		 622 if (nprps <= (256 / 8)) {
623 pool = dev->prp_small_pool;
eca18b23 624 iod->npages = 0;
99802a7a
MW		 625 } else {
626 pool = dev->prp_page_pool;
eca18b23 627 iod->npages = 1;
99802a7a
MW		 628 }
629
b77954cb
MW		 630 prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
631 if (!prp_list) {
edd10d33 632 iod->first_dma = dma_addr;
eca18b23 633 iod->npages = -1;
1d090624 634 return (total_len - length) + page_size;
b77954cb 635 }
eca18b23
MW		 636 list[0] = prp_list;
637 iod->first_dma = prp_dma;
e025344c
SMM		 638 i = 0;
639 for (;;) {
1d090624 640 if (i == page_size >> 3) {
e025344c 641 __le64 *old_prp_list = prp_list;
b77954cb 642 prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
eca18b23
MW		 643 if (!prp_list)
644 return total_len - length;
645 list[iod->npages++] = prp_list;
7523d834
MW		 646 prp_list[0] = old_prp_list[i - 1];
647 old_prp_list[i - 1] = cpu_to_le64(prp_dma);
648 i = 1;
e025344c
SMM		 649 }
650 prp_list[i++] = cpu_to_le64(dma_addr);
1d090624
KB		 651 dma_len -= page_size;
652 dma_addr += page_size;
653 length -= page_size;
e025344c
SMM		 654 if (length <= 0)
655 break;
656 if (dma_len > 0)
657 continue;
658 BUG_ON(dma_len < 0);
659 sg = sg_next(sg);
660 dma_addr = sg_dma_address(sg);
661 dma_len = sg_dma_len(sg);
ff22b54f
MW		 662 }
663
eca18b23 664 return total_len;
ff22b54f
MW		 665}
666
a4aea562
MB		 667/*
668 * We reuse the small pool to allocate the 16-byte range here as it is not
669 * worth having a special pool for these or additional cases to handle freeing
670 * the iod.
671 */
672static void nvme_submit_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns,
673 struct request *req, struct nvme_iod *iod)
0e5e4f0e 674{
edd10d33
KB		 675 struct nvme_dsm_range *range =
676 (struct nvme_dsm_range *)iod_list(iod)[0];
0e5e4f0e
KB		 677 struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
678
0e5e4f0e 679 range->cattr = cpu_to_le32(0);
a4aea562
MB		 680 range->nlb = cpu_to_le32(blk_rq_bytes(req) >> ns->lba_shift);
681 range->slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
0e5e4f0e
KB		 682
683 memset(cmnd, 0, sizeof(*cmnd));
684 cmnd->dsm.opcode = nvme_cmd_dsm;
a4aea562 685 cmnd->dsm.command_id = req->tag;
0e5e4f0e
KB		 686 cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
687 cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma);
688 cmnd->dsm.nr = 0;
689 cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
690
691 if (++nvmeq->sq_tail == nvmeq->q_depth)
692 nvmeq->sq_tail = 0;
693 writel(nvmeq->sq_tail, nvmeq->q_db);
0e5e4f0e
KB		 694}
695
a4aea562 696static void nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns,
00df5cb4
MW		 697 int cmdid)
698{
699 struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
700
701 memset(cmnd, 0, sizeof(*cmnd));
702 cmnd->common.opcode = nvme_cmd_flush;
703 cmnd->common.command_id = cmdid;
704 cmnd->common.nsid = cpu_to_le32(ns->ns_id);
705
706 if (++nvmeq->sq_tail == nvmeq->q_depth)
707 nvmeq->sq_tail = 0;
708 writel(nvmeq->sq_tail, nvmeq->q_db);
00df5cb4
MW		 709}
710
a4aea562
MB		 711static int nvme_submit_iod(struct nvme_queue *nvmeq, struct nvme_iod *iod,
712 struct nvme_ns *ns)
b60503ba 713{
ac3dd5bd 714 struct request *req = iod_get_private(iod);
ff22b54f 715 struct nvme_command *cmnd;
a4aea562
MB		 716 u16 control = 0;
717 u32 dsmgmt = 0;
00df5cb4 718
a4aea562 719 if (req->cmd_flags & REQ_FUA)
b60503ba 720 control |= NVME_RW_FUA;
a4aea562 721 if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
b60503ba
MW		 722 control |= NVME_RW_LR;
723
a4aea562 724 if (req->cmd_flags & REQ_RAHEAD)
b60503ba
MW		 725 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
726
ff22b54f 727 cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
b8deb62c 728 memset(cmnd, 0, sizeof(*cmnd));
b60503ba 729
a4aea562
MB		 730 cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);
731 cmnd->rw.command_id = req->tag;
ff22b54f 732 cmnd->rw.nsid = cpu_to_le32(ns->ns_id);
edd10d33
KB		 733 cmnd->rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
734 cmnd->rw.prp2 = cpu_to_le64(iod->first_dma);
a4aea562
MB		 735 cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
736 cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
e1e5e564
KB		 737
738 if (blk_integrity_rq(req)) {
739 cmnd->rw.metadata = cpu_to_le64(sg_dma_address(iod->meta_sg));
740 switch (ns->pi_type) {
741 case NVME_NS_DPS_PI_TYPE3:
742 control |= NVME_RW_PRINFO_PRCHK_GUARD;
743 break;
744 case NVME_NS_DPS_PI_TYPE1:
745 case NVME_NS_DPS_PI_TYPE2:
746 control |= NVME_RW_PRINFO_PRCHK_GUARD |
747 NVME_RW_PRINFO_PRCHK_REF;
748 cmnd->rw.reftag = cpu_to_le32(
749 nvme_block_nr(ns, blk_rq_pos(req)));
750 break;
751 }
752 } else if (ns->ms)
753 control |= NVME_RW_PRINFO_PRACT;
754
ff22b54f
MW		 755 cmnd->rw.control = cpu_to_le16(control);
756 cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
b60503ba 757
b60503ba
MW		 758 if (++nvmeq->sq_tail == nvmeq->q_depth)
759 nvmeq->sq_tail = 0;
7547881d 760 writel(nvmeq->sq_tail, nvmeq->q_db);
b60503ba 761
1974b1ae 762 return 0;
edd10d33
KB		 763}
764
a4aea562
MB		 765static int nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
766 const struct blk_mq_queue_data *bd)
edd10d33 767{
a4aea562
MB		 768 struct nvme_ns *ns = hctx->queue->queuedata;
769 struct nvme_queue *nvmeq = hctx->driver_data;
770 struct request *req = bd->rq;
771 struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
edd10d33 772 struct nvme_iod *iod;
a4aea562 773 enum dma_data_direction dma_dir;
edd10d33 774
e1e5e564
KB		 775 /*
776 * If formated with metadata, require the block layer provide a buffer
777 * unless this namespace is formated such that the metadata can be
778 * stripped/generated by the controller with PRACT=1.
779 */
780 if (ns->ms && !blk_integrity_rq(req)) {
781 if (!(ns->pi_type && ns->ms == 8)) {
782 req->errors = -EFAULT;
783 blk_mq_complete_request(req);
784 return BLK_MQ_RQ_QUEUE_OK;
785 }
786 }
787
ac3dd5bd 788 iod = nvme_alloc_iod(req, ns->dev, GFP_ATOMIC);
edd10d33 789 if (!iod)
fe54303e 790 return BLK_MQ_RQ_QUEUE_BUSY;
a4aea562 791
a4aea562 792 if (req->cmd_flags & REQ_DISCARD) {
edd10d33
KB		 793 void *range;
794 /*
795 * We reuse the small pool to allocate the 16-byte range here
796 * as it is not worth having a special pool for these or
797 * additional cases to handle freeing the iod.
798 */
799 range = dma_pool_alloc(nvmeq->dev->prp_small_pool,
800 GFP_ATOMIC,
801 &iod->first_dma);
a4aea562 802 if (!range)
fe54303e 803 goto retry_cmd;
edd10d33
KB		 804 iod_list(iod)[0] = (__le64 *)range;
805 iod->npages = 0;
ac3dd5bd 806 } else if (req->nr_phys_segments) {
a4aea562
MB		 807 dma_dir = rq_data_dir(req) ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
808
ac3dd5bd 809 sg_init_table(iod->sg, req->nr_phys_segments);
a4aea562 810 iod->nents = blk_rq_map_sg(req->q, req, iod->sg);
fe54303e
JA		 811 if (!iod->nents)
812 goto error_cmd;
a4aea562
MB		 813
814 if (!dma_map_sg(nvmeq->q_dmadev, iod->sg, iod->nents, dma_dir))
fe54303e 815 goto retry_cmd;
a4aea562 816
fe54303e
JA		 817 if (blk_rq_bytes(req) !=
818 nvme_setup_prps(nvmeq->dev, iod, blk_rq_bytes(req), GFP_ATOMIC)) {
819 dma_unmap_sg(&nvmeq->dev->pci_dev->dev, iod->sg,
820 iod->nents, dma_dir);
821 goto retry_cmd;
822 }
e1e5e564
KB		 823 if (blk_integrity_rq(req)) {
824 if (blk_rq_count_integrity_sg(req->q, req->bio) != 1)
825 goto error_cmd;
826
827 sg_init_table(iod->meta_sg, 1);
828 if (blk_rq_map_integrity_sg(
829 req->q, req->bio, iod->meta_sg) != 1)
830 goto error_cmd;
831
832 if (rq_data_dir(req))
833 nvme_dif_remap(req, nvme_dif_prep);
834
835 if (!dma_map_sg(nvmeq->q_dmadev, iod->meta_sg, 1, dma_dir))
836 goto error_cmd;
837 }
edd10d33 838 }
1974b1ae 839
9af8785a 840 nvme_set_info(cmd, iod, req_completion);
a4aea562
MB		 841 spin_lock_irq(&nvmeq->q_lock);
842 if (req->cmd_flags & REQ_DISCARD)
843 nvme_submit_discard(nvmeq, ns, req, iod);
844 else if (req->cmd_flags & REQ_FLUSH)
845 nvme_submit_flush(nvmeq, ns, req->tag);
846 else
847 nvme_submit_iod(nvmeq, iod, ns);
848
849 nvme_process_cq(nvmeq);
850 spin_unlock_irq(&nvmeq->q_lock);
851 return BLK_MQ_RQ_QUEUE_OK;
852
fe54303e
JA		 853 error_cmd:
854 nvme_free_iod(nvmeq->dev, iod);
855 return BLK_MQ_RQ_QUEUE_ERROR;
856 retry_cmd:
eca18b23 857 nvme_free_iod(nvmeq->dev, iod);
fe54303e 858 return BLK_MQ_RQ_QUEUE_BUSY;
b60503ba
MW		 859}
860
e9539f47 861static int nvme_process_cq(struct nvme_queue *nvmeq)
b60503ba 862{
82123460 863 u16 head, phase;
b60503ba 864
b60503ba 865 head = nvmeq->cq_head;
82123460 866 phase = nvmeq->cq_phase;
b60503ba
MW		 867
868 for (;;) {
c2f5b650
MW		 869 void *ctx;
870 nvme_completion_fn fn;
b60503ba 871 struct nvme_completion cqe = nvmeq->cqes[head];
82123460 872 if ((le16_to_cpu(cqe.status) & 1) != phase)
b60503ba
MW		 873 break;
874 nvmeq->sq_head = le16_to_cpu(cqe.sq_head);
875 if (++head == nvmeq->q_depth) {
876 head = 0;
82123460 877 phase = !phase;
b60503ba 878 }
a4aea562 879 ctx = nvme_finish_cmd(nvmeq, cqe.command_id, &fn);
edd10d33 880 fn(nvmeq, ctx, &cqe);
b60503ba
MW		 881 }
882
883 /* If the controller ignores the cq head doorbell and continuously
884 * writes to the queue, it is theoretically possible to wrap around
885 * the queue twice and mistakenly return IRQ_NONE. Linux only
886 * requires that 0.1% of your interrupts are handled, so this isn't
887 * a big problem.
888 */
82123460 889 if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
e9539f47 890 return 0;
b60503ba 891
b80d5ccc 892 writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
b60503ba 893 nvmeq->cq_head = head;
82123460 894 nvmeq->cq_phase = phase;
b60503ba 895
e9539f47
MW		 896 nvmeq->cqe_seen = 1;
897 return 1;
b60503ba
MW		 898}
899
a4aea562
MB		 900/* Admin queue isn't initialized as a request queue. If at some point this
901 * happens anyway, make sure to notify the user */
902static int nvme_admin_queue_rq(struct blk_mq_hw_ctx *hctx,
903 const struct blk_mq_queue_data *bd)
7d822457 904{
a4aea562
MB		 905 WARN_ON_ONCE(1);
906 return BLK_MQ_RQ_QUEUE_ERROR;
7d822457
MW		 907}
908
b60503ba 909static irqreturn_t nvme_irq(int irq, void *data)
58ffacb5
MW		 910{
911 irqreturn_t result;
912 struct nvme_queue *nvmeq = data;
913 spin_lock(&nvmeq->q_lock);
e9539f47
MW		 914 nvme_process_cq(nvmeq);
915 result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE;
916 nvmeq->cqe_seen = 0;
58ffacb5
MW		 917 spin_unlock(&nvmeq->q_lock);
918 return result;
919}
920
921static irqreturn_t nvme_irq_check(int irq, void *data)
922{
923 struct nvme_queue *nvmeq = data;
924 struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head];
925 if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase)
926 return IRQ_NONE;
927 return IRQ_WAKE_THREAD;
928}
929
a4aea562
MB		 930static void nvme_abort_cmd_info(struct nvme_queue *nvmeq, struct nvme_cmd_info *
931 cmd_info)
3c0cf138
MW		 932{
933 spin_lock_irq(&nvmeq->q_lock);
a4aea562 934 cancel_cmd_info(cmd_info, NULL);
3c0cf138
MW		 935 spin_unlock_irq(&nvmeq->q_lock);
936}
937
c2f5b650
MW		 938struct sync_cmd_info {
939 struct task_struct *task;
940 u32 result;
941 int status;
942};
943
edd10d33 944static void sync_completion(struct nvme_queue *nvmeq, void *ctx,
c2f5b650
MW		 945 struct nvme_completion *cqe)
946{
947 struct sync_cmd_info *cmdinfo = ctx;
948 cmdinfo->result = le32_to_cpup(&cqe->result);
949 cmdinfo->status = le16_to_cpup(&cqe->status) >> 1;
950 wake_up_process(cmdinfo->task);
951}
952
b60503ba
MW		 953/*
954 * Returns 0 on success. If the result is negative, it's a Linux error code;
955 * if the result is positive, it's an NVM Express status code
956 */
a4aea562 957static int nvme_submit_sync_cmd(struct request *req, struct nvme_command *cmd,
5d0f6131 958 u32 *result, unsigned timeout)
b60503ba 959{
a4aea562 960 int ret;
b60503ba 961 struct sync_cmd_info cmdinfo;
a4aea562
MB		 962 struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
963 struct nvme_queue *nvmeq = cmd_rq->nvmeq;
b60503ba
MW		 964
965 cmdinfo.task = current;
966 cmdinfo.status = -EINTR;
967
a4aea562
MB		 968 cmd->common.command_id = req->tag;
969
970 nvme_set_info(cmd_rq, &cmdinfo, sync_completion);
b60503ba 971
3c0cf138 972 set_current_state(TASK_KILLABLE);
4f5099af
KB		 973 ret = nvme_submit_cmd(nvmeq, cmd);
974 if (ret) {
a4aea562 975 nvme_finish_cmd(nvmeq, req->tag, NULL);
4f5099af 976 set_current_state(TASK_RUNNING);
4f5099af 977 }
849c6e77 978 ret = schedule_timeout(timeout);
b60503ba 979
849c6e77
JA		 980 /*
981 * Ensure that sync_completion has either run, or that it will
982 * never run.
983 */
984 nvme_abort_cmd_info(nvmeq, blk_mq_rq_to_pdu(req));
985
986 /*
987 * We never got the completion
988 */
989 if (cmdinfo.status == -EINTR)
3c0cf138 990 return -EINTR;
3c0cf138 991
b60503ba
MW		 992 if (result)
993 *result = cmdinfo.result;
994
995 return cmdinfo.status;
996}
997
a4aea562
MB		 998static int nvme_submit_async_admin_req(struct nvme_dev *dev)
999{
1000 struct nvme_queue *nvmeq = dev->queues[0];
1001 struct nvme_command c;
1002 struct nvme_cmd_info *cmd_info;
1003 struct request *req;
1004
6dcc0cf6 1005 req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_ATOMIC, false);
9f173b33
DC		 1006 if (IS_ERR(req))
1007 return PTR_ERR(req);
a4aea562 1008
c917dfe5 1009 req->cmd_flags |= REQ_NO_TIMEOUT;
a4aea562
MB		 1010 cmd_info = blk_mq_rq_to_pdu(req);
1011 nvme_set_info(cmd_info, req, async_req_completion);
1012
1013 memset(&c, 0, sizeof(c));
1014 c.common.opcode = nvme_admin_async_event;
1015 c.common.command_id = req->tag;
1016
1017 return __nvme_submit_cmd(nvmeq, &c);
1018}
1019
1020static int nvme_submit_admin_async_cmd(struct nvme_dev *dev,
4d115420
KB		 1021 struct nvme_command *cmd,
1022 struct async_cmd_info *cmdinfo, unsigned timeout)
1023{
a4aea562
MB		 1024 struct nvme_queue *nvmeq = dev->queues[0];
1025 struct request *req;
1026 struct nvme_cmd_info *cmd_rq;
4d115420 1027
a4aea562 1028 req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_KERNEL, false);
9f173b33
DC		 1029 if (IS_ERR(req))
1030 return PTR_ERR(req);
a4aea562
MB		 1031
1032 req->timeout = timeout;
1033 cmd_rq = blk_mq_rq_to_pdu(req);
1034 cmdinfo->req = req;
1035 nvme_set_info(cmd_rq, cmdinfo, async_completion);
4d115420 1036 cmdinfo->status = -EINTR;
a4aea562
MB		 1037
1038 cmd->common.command_id = req->tag;
1039
4f5099af 1040 return nvme_submit_cmd(nvmeq, cmd);
4d115420
KB		 1041}
1042
a64e6bb4 1043static int __nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd,
a4aea562 1044 u32 *result, unsigned timeout)
b60503ba 1045{
a4aea562
MB		 1046 int res;
1047 struct request *req;
1048
1049 req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_KERNEL, false);
97fe3832
JA		 1050 if (IS_ERR(req))
1051 return PTR_ERR(req);
a4aea562 1052 res = nvme_submit_sync_cmd(req, cmd, result, timeout);
9d135bb8 1053 blk_mq_free_request(req);
a4aea562 1054 return res;
4f5099af
KB		 1055}
1056
a4aea562 1057int nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd,
4f5099af
KB		 1058 u32 *result)
1059{
a4aea562 1060 return __nvme_submit_admin_cmd(dev, cmd, result, ADMIN_TIMEOUT);
b60503ba
MW		 1061}
1062
a4aea562
MB		 1063int nvme_submit_io_cmd(struct nvme_dev *dev, struct nvme_ns *ns,
1064 struct nvme_command *cmd, u32 *result)
4d115420 1065{
a4aea562
MB		 1066 int res;
1067 struct request *req;
1068
1069 req = blk_mq_alloc_request(ns->queue, WRITE, (GFP_KERNEL|__GFP_WAIT),
1070 false);
97fe3832
JA		 1071 if (IS_ERR(req))
1072 return PTR_ERR(req);
a4aea562 1073 res = nvme_submit_sync_cmd(req, cmd, result, NVME_IO_TIMEOUT);
9d135bb8 1074 blk_mq_free_request(req);
a4aea562 1075 return res;
4d115420
KB		 1076}
1077
b60503ba
MW		 1078static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
1079{
b60503ba
MW		 1080 struct nvme_command c;
1081
1082 memset(&c, 0, sizeof(c));
1083 c.delete_queue.opcode = opcode;
1084 c.delete_queue.qid = cpu_to_le16(id);
1085
a4aea562 1086 return nvme_submit_admin_cmd(dev, &c, NULL);
b60503ba
MW		 1087}
1088
1089static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
1090 struct nvme_queue *nvmeq)
1091{
b60503ba
MW		 1092 struct nvme_command c;
1093 int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;
1094
1095 memset(&c, 0, sizeof(c));
1096 c.create_cq.opcode = nvme_admin_create_cq;
1097 c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
1098 c.create_cq.cqid = cpu_to_le16(qid);
1099 c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
1100 c.create_cq.cq_flags = cpu_to_le16(flags);
1101 c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);
1102
a4aea562 1103 return nvme_submit_admin_cmd(dev, &c, NULL);
b60503ba
MW		 1104}
1105
1106static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
1107 struct nvme_queue *nvmeq)
1108{
b60503ba
MW		 1109 struct nvme_command c;
1110 int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;
1111
1112 memset(&c, 0, sizeof(c));
1113 c.create_sq.opcode = nvme_admin_create_sq;
1114 c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
1115 c.create_sq.sqid = cpu_to_le16(qid);
1116 c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
1117 c.create_sq.sq_flags = cpu_to_le16(flags);
1118 c.create_sq.cqid = cpu_to_le16(qid);
1119
a4aea562 1120 return nvme_submit_admin_cmd(dev, &c, NULL);
b60503ba
MW		 1121}
1122
1123static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
1124{
1125 return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
1126}
1127
1128static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
1129{
1130 return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
1131}
1132
5d0f6131 1133int nvme_identify(struct nvme_dev *dev, unsigned nsid, unsigned cns,
bc5fc7e4
MW		 1134 dma_addr_t dma_addr)
1135{
1136 struct nvme_command c;
1137
1138 memset(&c, 0, sizeof(c));
1139 c.identify.opcode = nvme_admin_identify;
1140 c.identify.nsid = cpu_to_le32(nsid);
1141 c.identify.prp1 = cpu_to_le64(dma_addr);
1142 c.identify.cns = cpu_to_le32(cns);
1143
1144 return nvme_submit_admin_cmd(dev, &c, NULL);
1145}
1146
5d0f6131 1147int nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid,
08df1e05 1148 dma_addr_t dma_addr, u32 *result)
bc5fc7e4
MW		 1149{
1150 struct nvme_command c;
1151
1152 memset(&c, 0, sizeof(c));
1153 c.features.opcode = nvme_admin_get_features;
a42cecce 1154 c.features.nsid = cpu_to_le32(nsid);
bc5fc7e4
MW		 1155 c.features.prp1 = cpu_to_le64(dma_addr);
1156 c.features.fid = cpu_to_le32(fid);
bc5fc7e4 1157
08df1e05 1158 return nvme_submit_admin_cmd(dev, &c, result);
df348139
MW		 1159}
1160
5d0f6131
VV		 1161int nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11,
1162 dma_addr_t dma_addr, u32 *result)
df348139
MW		 1163{
1164 struct nvme_command c;
1165
1166 memset(&c, 0, sizeof(c));
1167 c.features.opcode = nvme_admin_set_features;
1168 c.features.prp1 = cpu_to_le64(dma_addr);
1169 c.features.fid = cpu_to_le32(fid);
1170 c.features.dword11 = cpu_to_le32(dword11);
1171
bc5fc7e4
MW		 1172 return nvme_submit_admin_cmd(dev, &c, result);
1173}
1174
c30341dc 1175/**
a4aea562 1176 * nvme_abort_req - Attempt aborting a request
c30341dc
KB		 1177 *
1178 * Schedule controller reset if the command was already aborted once before and
1179 * still hasn't been returned to the driver, or if this is the admin queue.
1180 */
a4aea562 1181static void nvme_abort_req(struct request *req)
c30341dc 1182{
a4aea562
MB		 1183 struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
1184 struct nvme_queue *nvmeq = cmd_rq->nvmeq;
c30341dc 1185 struct nvme_dev *dev = nvmeq->dev;
a4aea562
MB		 1186 struct request *abort_req;
1187 struct nvme_cmd_info *abort_cmd;
1188 struct nvme_command cmd;
c30341dc 1189
a4aea562 1190 if (!nvmeq->qid || cmd_rq->aborted) {
7a509a6b
KB		 1191 unsigned long flags;
1192
1193 spin_lock_irqsave(&dev_list_lock, flags);
c30341dc 1194 if (work_busy(&dev->reset_work))
7a509a6b 1195 goto out;
c30341dc
KB		 1196 list_del_init(&dev->node);
1197 dev_warn(&dev->pci_dev->dev,
a4aea562
MB		 1198 "I/O %d QID %d timeout, reset controller\n",
1199 req->tag, nvmeq->qid);
9ca97374 1200 dev->reset_workfn = nvme_reset_failed_dev;
c30341dc 1201 queue_work(nvme_workq, &dev->reset_work);
7a509a6b
KB		 1202 out:
1203 spin_unlock_irqrestore(&dev_list_lock, flags);
c30341dc
KB		 1204 return;
1205 }
1206
1207 if (!dev->abort_limit)
1208 return;
1209
a4aea562
MB		 1210 abort_req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_ATOMIC,
1211 false);
9f173b33 1212 if (IS_ERR(abort_req))
c30341dc
KB		 1213 return;
1214
a4aea562
MB		 1215 abort_cmd = blk_mq_rq_to_pdu(abort_req);
1216 nvme_set_info(abort_cmd, abort_req, abort_completion);
1217
c30341dc
KB		 1218 memset(&cmd, 0, sizeof(cmd));
1219 cmd.abort.opcode = nvme_admin_abort_cmd;
a4aea562 1220 cmd.abort.cid = req->tag;
c30341dc 1221 cmd.abort.sqid = cpu_to_le16(nvmeq->qid);
a4aea562 1222 cmd.abort.command_id = abort_req->tag;
c30341dc
KB		 1223
1224 --dev->abort_limit;
a4aea562 1225 cmd_rq->aborted = 1;
c30341dc 1226
a4aea562 1227 dev_warn(nvmeq->q_dmadev, "Aborting I/O %d QID %d\n", req->tag,
c30341dc 1228 nvmeq->qid);
a4aea562
MB		 1229 if (nvme_submit_cmd(dev->queues[0], &cmd) < 0) {
1230 dev_warn(nvmeq->q_dmadev,
1231 "Could not abort I/O %d QID %d",
1232 req->tag, nvmeq->qid);
c87fd540 1233 blk_mq_free_request(abort_req);
a4aea562 1234 }
c30341dc
KB		 1235}
1236
a4aea562
MB		 1237static void nvme_cancel_queue_ios(struct blk_mq_hw_ctx *hctx,
1238 struct request *req, void *data, bool reserved)
a09115b2 1239{
a4aea562
MB		 1240 struct nvme_queue *nvmeq = data;
1241 void *ctx;
1242 nvme_completion_fn fn;
1243 struct nvme_cmd_info *cmd;
cef6a948
KB		 1244 struct nvme_completion cqe;
1245
1246 if (!blk_mq_request_started(req))
1247 return;
a09115b2 1248
a4aea562 1249 cmd = blk_mq_rq_to_pdu(req);
a09115b2 1250
a4aea562
MB		 1251 if (cmd->ctx == CMD_CTX_CANCELLED)
1252 return;
1253
cef6a948
KB		 1254 if (blk_queue_dying(req->q))
1255 cqe.status = cpu_to_le16((NVME_SC_ABORT_REQ | NVME_SC_DNR) << 1);
1256 else
1257 cqe.status = cpu_to_le16(NVME_SC_ABORT_REQ << 1);
1258
1259
a4aea562
MB		 1260 dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d QID %d\n",
1261 req->tag, nvmeq->qid);
1262 ctx = cancel_cmd_info(cmd, &fn);
1263 fn(nvmeq, ctx, &cqe);
a09115b2
MW		 1264}
1265
a4aea562 1266static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
9e866774 1267{
a4aea562
MB		 1268 struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
1269 struct nvme_queue *nvmeq = cmd->nvmeq;
1270
7a509a6b
KB		 1271 /*
1272 * The aborted req will be completed on receiving the abort req.
1273 * We enable the timer again. If hit twice, it'll cause a device reset,
1274 * as the device then is in a faulty state.
1275 */
1276 int ret = BLK_EH_RESET_TIMER;
1277
a4aea562
MB		 1278 dev_warn(nvmeq->q_dmadev, "Timeout I/O %d QID %d\n", req->tag,
1279 nvmeq->qid);
c917dfe5 1280
7a509a6b 1281 spin_lock_irq(&nvmeq->q_lock);
c917dfe5
KB		 1282 if (!nvmeq->dev->initialized) {
1283 /*
1284 * Force cancelled command frees the request, which requires we
1285 * return BLK_EH_NOT_HANDLED.
1286 */
1287 nvme_cancel_queue_ios(nvmeq->hctx, req, nvmeq, reserved);
7a509a6b
KB		 1288 ret = BLK_EH_NOT_HANDLED;
1289 } else
1290 nvme_abort_req(req);
1291 spin_unlock_irq(&nvmeq->q_lock);
a4aea562 1292
7a509a6b 1293 return ret;
a4aea562 1294}
22404274 1295
a4aea562
MB		 1296static void nvme_free_queue(struct nvme_queue *nvmeq)
1297{
9e866774
MW		 1298 dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
1299 (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
1300 dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
1301 nvmeq->sq_cmds, nvmeq->sq_dma_addr);
1302 kfree(nvmeq);
1303}
1304
a1a5ef99 1305static void nvme_free_queues(struct nvme_dev *dev, int lowest)
22404274
KB		 1306{
1307 int i;
1308
a1a5ef99 1309 for (i = dev->queue_count - 1; i >= lowest; i--) {
a4aea562 1310 struct nvme_queue *nvmeq = dev->queues[i];
22404274 1311 dev->queue_count--;
a4aea562 1312 dev->queues[i] = NULL;
f435c282 1313 nvme_free_queue(nvmeq);
121c7ad4 1314 }
22404274
KB		 1315}
1316
4d115420
KB		 1317/**
1318 * nvme_suspend_queue - put queue into suspended state
1319 * @nvmeq - queue to suspend
4d115420
KB		 1320 */
1321static int nvme_suspend_queue(struct nvme_queue *nvmeq)
b60503ba 1322{
2b25d981 1323 int vector;
b60503ba 1324
a09115b2 1325 spin_lock_irq(&nvmeq->q_lock);
2b25d981
KB		 1326 if (nvmeq->cq_vector == -1) {
1327 spin_unlock_irq(&nvmeq->q_lock);
1328 return 1;
1329 }
1330 vector = nvmeq->dev->entry[nvmeq->cq_vector].vector;
42f61420 1331 nvmeq->dev->online_queues--;
2b25d981 1332 nvmeq->cq_vector = -1;
a09115b2
MW		 1333 spin_unlock_irq(&nvmeq->q_lock);
1334
aba2080f
MW		 1335 irq_set_affinity_hint(vector, NULL);
1336 free_irq(vector, nvmeq);
b60503ba 1337
4d115420
KB		 1338 return 0;
1339}
b60503ba 1340
4d115420
KB		 1341static void nvme_clear_queue(struct nvme_queue *nvmeq)
1342{
a4aea562
MB		 1343 struct blk_mq_hw_ctx *hctx = nvmeq->hctx;
1344
22404274
KB		 1345 spin_lock_irq(&nvmeq->q_lock);
1346 nvme_process_cq(nvmeq);
a4aea562
MB		 1347 if (hctx && hctx->tags)
1348 blk_mq_tag_busy_iter(hctx, nvme_cancel_queue_ios, nvmeq);
22404274 1349 spin_unlock_irq(&nvmeq->q_lock);
b60503ba
MW		 1350}
1351
4d115420
KB		 1352static void nvme_disable_queue(struct nvme_dev *dev, int qid)
1353{
a4aea562 1354 struct nvme_queue *nvmeq = dev->queues[qid];
4d115420
KB		 1355
1356 if (!nvmeq)
1357 return;
1358 if (nvme_suspend_queue(nvmeq))
1359 return;
1360
0e53d180
KB		 1361 /* Don't tell the adapter to delete the admin queue.
1362 * Don't tell a removed adapter to delete IO queues. */
1363 if (qid && readl(&dev->bar->csts) != -1) {
b60503ba
MW		 1364 adapter_delete_sq(dev, qid);
1365 adapter_delete_cq(dev, qid);
1366 }
0fb59cbc
KB		 1367 if (!qid && dev->admin_q)
1368 blk_mq_freeze_queue_start(dev->admin_q);
4d115420 1369 nvme_clear_queue(nvmeq);
b60503ba
MW		 1370}
1371
1372static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
2b25d981 1373 int depth)
b60503ba
MW		 1374{
1375 struct device *dmadev = &dev->pci_dev->dev;
a4aea562 1376 struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq), GFP_KERNEL);
b60503ba
MW		 1377 if (!nvmeq)
1378 return NULL;
1379
4d51abf9
JP		 1380 nvmeq->cqes = dma_zalloc_coherent(dmadev, CQ_SIZE(depth),
1381 &nvmeq->cq_dma_addr, GFP_KERNEL);
b60503ba
MW		 1382 if (!nvmeq->cqes)
1383 goto free_nvmeq;
b60503ba
MW		 1384
1385 nvmeq->sq_cmds = dma_alloc_coherent(dmadev, SQ_SIZE(depth),
1386 &nvmeq->sq_dma_addr, GFP_KERNEL);
1387 if (!nvmeq->sq_cmds)
1388 goto free_cqdma;
1389
1390 nvmeq->q_dmadev = dmadev;
091b6092 1391 nvmeq->dev = dev;
3193f07b
MW		 1392 snprintf(nvmeq->irqname, sizeof(nvmeq->irqname), "nvme%dq%d",
1393 dev->instance, qid);
b60503ba
MW		 1394 spin_lock_init(&nvmeq->q_lock);
1395 nvmeq->cq_head = 0;
82123460 1396 nvmeq->cq_phase = 1;
b80d5ccc 1397 nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
b60503ba 1398 nvmeq->q_depth = depth;
c30341dc 1399 nvmeq->qid = qid;
22404274 1400 dev->queue_count++;
a4aea562 1401 dev->queues[qid] = nvmeq;
b60503ba
MW		 1402
1403 return nvmeq;
1404
1405 free_cqdma:
68b8eca5 1406 dma_free_coherent(dmadev, CQ_SIZE(depth), (void *)nvmeq->cqes,
b60503ba
MW		 1407 nvmeq->cq_dma_addr);
1408 free_nvmeq:
1409 kfree(nvmeq);
1410 return NULL;
1411}
1412
3001082c
MW		 1413static int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq,
1414 const char *name)
1415{
58ffacb5
MW		 1416 if (use_threaded_interrupts)
1417 return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector,
481e5bad 1418 nvme_irq_check, nvme_irq, IRQF_SHARED,
58ffacb5 1419 name, nvmeq);
3001082c 1420 return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq,
481e5bad 1421 IRQF_SHARED, name, nvmeq);
3001082c
MW		 1422}
1423
22404274 1424static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
b60503ba 1425{
22404274 1426 struct nvme_dev *dev = nvmeq->dev;
b60503ba 1427
7be50e93 1428 spin_lock_irq(&nvmeq->q_lock);
22404274
KB		 1429 nvmeq->sq_tail = 0;
1430 nvmeq->cq_head = 0;
1431 nvmeq->cq_phase = 1;
b80d5ccc 1432 nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
22404274 1433 memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
42f61420 1434 dev->online_queues++;
7be50e93 1435 spin_unlock_irq(&nvmeq->q_lock);
22404274
KB		 1436}
1437
1438static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
1439{
1440 struct nvme_dev *dev = nvmeq->dev;
1441 int result;
3f85d50b 1442
2b25d981 1443 nvmeq->cq_vector = qid - 1;
b60503ba
MW		 1444 result = adapter_alloc_cq(dev, qid, nvmeq);
1445 if (result < 0)
22404274 1446 return result;
b60503ba
MW		 1447
1448 result = adapter_alloc_sq(dev, qid, nvmeq);
1449 if (result < 0)
1450 goto release_cq;
1451
3193f07b 1452 result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
b60503ba
MW		 1453 if (result < 0)
1454 goto release_sq;
1455
22404274 1456 nvme_init_queue(nvmeq, qid);
22404274 1457 return result;
b60503ba
MW		 1458
1459 release_sq:
1460 adapter_delete_sq(dev, qid);
1461 release_cq:
1462 adapter_delete_cq(dev, qid);
22404274 1463 return result;
b60503ba
MW		 1464}
1465
ba47e386
MW		 1466static int nvme_wait_ready(struct nvme_dev *dev, u64 cap, bool enabled)
1467{
1468 unsigned long timeout;
1469 u32 bit = enabled ? NVME_CSTS_RDY : 0;
1470
1471 timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
1472
1473 while ((readl(&dev->bar->csts) & NVME_CSTS_RDY) != bit) {
1474 msleep(100);
1475 if (fatal_signal_pending(current))
1476 return -EINTR;
1477 if (time_after(jiffies, timeout)) {
1478 dev_err(&dev->pci_dev->dev,
27e8166c
MW		 1479 "Device not ready; aborting %s\n", enabled ?
1480 "initialisation" : "reset");
ba47e386
MW		 1481 return -ENODEV;
1482 }
1483 }
1484
1485 return 0;
1486}
1487
1488/*
1489 * If the device has been passed off to us in an enabled state, just clear
1490 * the enabled bit. The spec says we should set the 'shutdown notification
1491 * bits', but doing so may cause the device to complete commands to the
1492 * admin queue ... and we don't know what memory that might be pointing at!
1493 */
1494static int nvme_disable_ctrl(struct nvme_dev *dev, u64 cap)
1495{
01079522
DM		 1496 dev->ctrl_config &= ~NVME_CC_SHN_MASK;
1497 dev->ctrl_config &= ~NVME_CC_ENABLE;
1498 writel(dev->ctrl_config, &dev->bar->cc);
44af146a 1499
ba47e386
MW		 1500 return nvme_wait_ready(dev, cap, false);
1501}
1502
1503static int nvme_enable_ctrl(struct nvme_dev *dev, u64 cap)
1504{
01079522
DM		 1505 dev->ctrl_config &= ~NVME_CC_SHN_MASK;
1506 dev->ctrl_config |= NVME_CC_ENABLE;
1507 writel(dev->ctrl_config, &dev->bar->cc);
1508
ba47e386
MW		 1509 return nvme_wait_ready(dev, cap, true);
1510}
1511
1894d8f1
KB		 1512static int nvme_shutdown_ctrl(struct nvme_dev *dev)
1513{
1514 unsigned long timeout;
1894d8f1 1515
01079522
DM		 1516 dev->ctrl_config &= ~NVME_CC_SHN_MASK;
1517 dev->ctrl_config |= NVME_CC_SHN_NORMAL;
1518
1519 writel(dev->ctrl_config, &dev->bar->cc);
1894d8f1 1520
2484f407 1521 timeout = SHUTDOWN_TIMEOUT + jiffies;
1894d8f1
KB		 1522 while ((readl(&dev->bar->csts) & NVME_CSTS_SHST_MASK) !=
1523 NVME_CSTS_SHST_CMPLT) {
1524 msleep(100);
1525 if (fatal_signal_pending(current))
1526 return -EINTR;
1527 if (time_after(jiffies, timeout)) {
1528 dev_err(&dev->pci_dev->dev,
1529 "Device shutdown incomplete; abort shutdown\n");
1530 return -ENODEV;
1531 }
1532 }
1533
1534 return 0;
1535}
1536
a4aea562
MB		 1537static struct blk_mq_ops nvme_mq_admin_ops = {
1538 .queue_rq = nvme_admin_queue_rq,
1539 .map_queue = blk_mq_map_queue,
1540 .init_hctx = nvme_admin_init_hctx,
2c30540b 1541 .exit_hctx = nvme_exit_hctx,
a4aea562
MB		 1542 .init_request = nvme_admin_init_request,
1543 .timeout = nvme_timeout,
1544};
1545
1546static struct blk_mq_ops nvme_mq_ops = {
1547 .queue_rq = nvme_queue_rq,
1548 .map_queue = blk_mq_map_queue,
1549 .init_hctx = nvme_init_hctx,
2c30540b 1550 .exit_hctx = nvme_exit_hctx,
a4aea562
MB		 1551 .init_request = nvme_init_request,
1552 .timeout = nvme_timeout,
1553};
1554
ea191d2f
KB		 1555static void nvme_dev_remove_admin(struct nvme_dev *dev)
1556{
1557 if (dev->admin_q && !blk_queue_dying(dev->admin_q)) {
1558 blk_cleanup_queue(dev->admin_q);
1559 blk_mq_free_tag_set(&dev->admin_tagset);
1560 }
1561}
1562
a4aea562
MB		 1563static int nvme_alloc_admin_tags(struct nvme_dev *dev)
1564{
1565 if (!dev->admin_q) {
1566 dev->admin_tagset.ops = &nvme_mq_admin_ops;
1567 dev->admin_tagset.nr_hw_queues = 1;
1568 dev->admin_tagset.queue_depth = NVME_AQ_DEPTH - 1;
1569 dev->admin_tagset.timeout = ADMIN_TIMEOUT;
1570 dev->admin_tagset.numa_node = dev_to_node(&dev->pci_dev->dev);
ac3dd5bd 1571 dev->admin_tagset.cmd_size = nvme_cmd_size(dev);
a4aea562
MB		 1572 dev->admin_tagset.driver_data = dev;
1573
1574 if (blk_mq_alloc_tag_set(&dev->admin_tagset))
1575 return -ENOMEM;
1576
1577 dev->admin_q = blk_mq_init_queue(&dev->admin_tagset);
35b489d3 1578 if (IS_ERR(dev->admin_q)) {
a4aea562
MB		 1579 blk_mq_free_tag_set(&dev->admin_tagset);
1580 return -ENOMEM;
1581 }
ea191d2f
KB		 1582 if (!blk_get_queue(dev->admin_q)) {
1583 nvme_dev_remove_admin(dev);
1584 return -ENODEV;
1585 }
0fb59cbc
KB		 1586 } else
1587 blk_mq_unfreeze_queue(dev->admin_q);
a4aea562
MB		 1588
1589 return 0;
1590}
1591
8d85fce7 1592static int nvme_configure_admin_queue(struct nvme_dev *dev)
b60503ba 1593{
ba47e386 1594 int result;
b60503ba 1595 u32 aqa;
ba47e386 1596 u64 cap = readq(&dev->bar->cap);
b60503ba 1597 struct nvme_queue *nvmeq;
1d090624
KB		 1598 unsigned page_shift = PAGE_SHIFT;
1599 unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12;
1600 unsigned dev_page_max = NVME_CAP_MPSMAX(cap) + 12;
1601
1602 if (page_shift < dev_page_min) {
1603 dev_err(&dev->pci_dev->dev,
1604 "Minimum device page size (%u) too large for "
1605 "host (%u)\n", 1 << dev_page_min,
1606 1 << page_shift);
1607 return -ENODEV;
1608 }
1609 if (page_shift > dev_page_max) {
1610 dev_info(&dev->pci_dev->dev,
1611 "Device maximum page size (%u) smaller than "
1612 "host (%u); enabling work-around\n",
1613 1 << dev_page_max, 1 << page_shift);
1614 page_shift = dev_page_max;
1615 }
b60503ba 1616
ba47e386
MW		 1617 result = nvme_disable_ctrl(dev, cap);
1618 if (result < 0)
1619 return result;
b60503ba 1620
a4aea562 1621 nvmeq = dev->queues[0];
cd638946 1622 if (!nvmeq) {
2b25d981 1623 nvmeq = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
cd638946
KB		 1624 if (!nvmeq)
1625 return -ENOMEM;
cd638946 1626 }
b60503ba
MW		 1627
1628 aqa = nvmeq->q_depth - 1;
1629 aqa |= aqa << 16;
1630
1d090624
KB		 1631 dev->page_size = 1 << page_shift;
1632
01079522 1633 dev->ctrl_config = NVME_CC_CSS_NVM;
1d090624 1634 dev->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
b60503ba 1635 dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
7f53f9d2 1636 dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
b60503ba
MW		 1637
1638 writel(aqa, &dev->bar->aqa);
1639 writeq(nvmeq->sq_dma_addr, &dev->bar->asq);
1640 writeq(nvmeq->cq_dma_addr, &dev->bar->acq);
b60503ba 1641
ba47e386 1642 result = nvme_enable_ctrl(dev, cap);
025c557a 1643 if (result)
a4aea562
MB		 1644 goto free_nvmeq;
1645
2b25d981 1646 nvmeq->cq_vector = 0;
3193f07b 1647 result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
025c557a 1648 if (result)
0fb59cbc 1649 goto free_nvmeq;
025c557a 1650
b60503ba 1651 return result;
a4aea562 1652
a4aea562
MB		 1653 free_nvmeq:
1654 nvme_free_queues(dev, 0);
1655 return result;
b60503ba
MW		 1656}
1657
5d0f6131 1658struct nvme_iod *nvme_map_user_pages(struct nvme_dev *dev, int write,
eca18b23 1659 unsigned long addr, unsigned length)
b60503ba 1660{
36c14ed9 1661 int i, err, count, nents, offset;
7fc3cdab
MW		 1662 struct scatterlist *sg;
1663 struct page **pages;
eca18b23 1664 struct nvme_iod *iod;
36c14ed9
MW		 1665
1666 if (addr & 3)
eca18b23 1667 return ERR_PTR(-EINVAL);
5460fc03 1668 if (!length || length > INT_MAX - PAGE_SIZE)
eca18b23 1669 return ERR_PTR(-EINVAL);
7fc3cdab 1670
36c14ed9 1671 offset = offset_in_page(addr);
7fc3cdab
MW		 1672 count = DIV_ROUND_UP(offset + length, PAGE_SIZE);
1673 pages = kcalloc(count, sizeof(*pages), GFP_KERNEL);
22fff826
DC		 1674 if (!pages)
1675 return ERR_PTR(-ENOMEM);
36c14ed9
MW		 1676
1677 err = get_user_pages_fast(addr, count, 1, pages);
1678 if (err < count) {
1679 count = err;
1680 err = -EFAULT;
1681 goto put_pages;
1682 }
7fc3cdab 1683
6808c5fb 1684 err = -ENOMEM;
ac3dd5bd 1685 iod = __nvme_alloc_iod(count, length, dev, 0, GFP_KERNEL);
6808c5fb
S		 1686 if (!iod)
1687 goto put_pages;
1688
eca18b23 1689 sg = iod->sg;
36c14ed9 1690 sg_init_table(sg, count);
d0ba1e49
MW		 1691 for (i = 0; i < count; i++) {
1692 sg_set_page(&sg[i], pages[i],
5460fc03
DC		 1693 min_t(unsigned, length, PAGE_SIZE - offset),
1694 offset);
d0ba1e49
MW		 1695 length -= (PAGE_SIZE - offset);
1696 offset = 0;
7fc3cdab 1697 }
fe304c43 1698 sg_mark_end(&sg[i - 1]);
1c2ad9fa 1699 iod->nents = count;
7fc3cdab 1700
7fc3cdab
MW		 1701 nents = dma_map_sg(&dev->pci_dev->dev, sg, count,
1702 write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
36c14ed9 1703 if (!nents)
eca18b23 1704 goto free_iod;
b60503ba 1705
7fc3cdab 1706 kfree(pages);
eca18b23 1707 return iod;
b60503ba 1708
eca18b23
MW		 1709 free_iod:
1710 kfree(iod);
7fc3cdab
MW		 1711 put_pages:
1712 for (i = 0; i < count; i++)
1713 put_page(pages[i]);
1714 kfree(pages);
eca18b23 1715 return ERR_PTR(err);
7fc3cdab 1716}
b60503ba 1717
5d0f6131 1718void nvme_unmap_user_pages(struct nvme_dev *dev, int write,
1c2ad9fa 1719 struct nvme_iod *iod)
7fc3cdab 1720{
1c2ad9fa 1721 int i;
b60503ba 1722
1c2ad9fa
MW		 1723 dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents,
1724 write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
7fc3cdab 1725
1c2ad9fa
MW		 1726 for (i = 0; i < iod->nents; i++)
1727 put_page(sg_page(&iod->sg[i]));
7fc3cdab 1728}
b60503ba 1729
a53295b6
MW		 1730static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
1731{
1732 struct nvme_dev *dev = ns->dev;
a53295b6
MW		 1733 struct nvme_user_io io;
1734 struct nvme_command c;
f410c680
KB		 1735 unsigned length, meta_len;
1736 int status, i;
1737 struct nvme_iod *iod, *meta_iod = NULL;
1738 dma_addr_t meta_dma_addr;
1739 void *meta, *uninitialized_var(meta_mem);
a53295b6
MW		 1740
1741 if (copy_from_user(&io, uio, sizeof(io)))
1742 return -EFAULT;
6c7d4945 1743 length = (io.nblocks + 1) << ns->lba_shift;
f410c680
KB		 1744 meta_len = (io.nblocks + 1) * ns->ms;
1745
1746 if (meta_len && ((io.metadata & 3) || !io.metadata))
1747 return -EINVAL;
6c7d4945
MW		 1748
1749 switch (io.opcode) {
1750 case nvme_cmd_write:
1751 case nvme_cmd_read:
6bbf1acd 1752 case nvme_cmd_compare:
eca18b23 1753 iod = nvme_map_user_pages(dev, io.opcode & 1, io.addr, length);
6413214c 1754 break;
6c7d4945 1755 default:
6bbf1acd 1756 return -EINVAL;
6c7d4945
MW		 1757 }
1758
eca18b23
MW		 1759 if (IS_ERR(iod))
1760 return PTR_ERR(iod);
a53295b6
MW		 1761
1762 memset(&c, 0, sizeof(c));
1763 c.rw.opcode = io.opcode;
1764 c.rw.flags = io.flags;
6c7d4945 1765 c.rw.nsid = cpu_to_le32(ns->ns_id);
a53295b6 1766 c.rw.slba = cpu_to_le64(io.slba);
6c7d4945 1767 c.rw.length = cpu_to_le16(io.nblocks);
a53295b6 1768 c.rw.control = cpu_to_le16(io.control);
1c9b5265
MW		 1769 c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
1770 c.rw.reftag = cpu_to_le32(io.reftag);
1771 c.rw.apptag = cpu_to_le16(io.apptag);
1772 c.rw.appmask = cpu_to_le16(io.appmask);
f410c680
KB		 1773
1774 if (meta_len) {
1b56749e
KB		 1775 meta_iod = nvme_map_user_pages(dev, io.opcode & 1, io.metadata,
1776 meta_len);
f410c680
KB		 1777 if (IS_ERR(meta_iod)) {
1778 status = PTR_ERR(meta_iod);
1779 meta_iod = NULL;
1780 goto unmap;
1781 }
1782
1783 meta_mem = dma_alloc_coherent(&dev->pci_dev->dev, meta_len,
1784 &meta_dma_addr, GFP_KERNEL);
1785 if (!meta_mem) {
1786 status = -ENOMEM;
1787 goto unmap;
1788 }
1789
1790 if (io.opcode & 1) {
1791 int meta_offset = 0;
1792
1793 for (i = 0; i < meta_iod->nents; i++) {
1794 meta = kmap_atomic(sg_page(&meta_iod->sg[i])) +
1795 meta_iod->sg[i].offset;
1796 memcpy(meta_mem + meta_offset, meta,
1797 meta_iod->sg[i].length);
1798 kunmap_atomic(meta);
1799 meta_offset += meta_iod->sg[i].length;
1800 }
1801 }
1802
1803 c.rw.metadata = cpu_to_le64(meta_dma_addr);
1804 }
1805
edd10d33
KB		 1806 length = nvme_setup_prps(dev, iod, length, GFP_KERNEL);
1807 c.rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
1808 c.rw.prp2 = cpu_to_le64(iod->first_dma);
a53295b6 1809
b77954cb
MW		 1810 if (length != (io.nblocks + 1) << ns->lba_shift)
1811 status = -ENOMEM;
1812 else
a4aea562 1813 status = nvme_submit_io_cmd(dev, ns, &c, NULL);
a53295b6 1814
f410c680
KB		 1815 if (meta_len) {
1816 if (status == NVME_SC_SUCCESS && !(io.opcode & 1)) {
1817 int meta_offset = 0;
1818
1819 for (i = 0; i < meta_iod->nents; i++) {
1820 meta = kmap_atomic(sg_page(&meta_iod->sg[i])) +
1821 meta_iod->sg[i].offset;
1822 memcpy(meta, meta_mem + meta_offset,
1823 meta_iod->sg[i].length);
1824 kunmap_atomic(meta);
1825 meta_offset += meta_iod->sg[i].length;
1826 }
1827 }
1828
1829 dma_free_coherent(&dev->pci_dev->dev, meta_len, meta_mem,
1830 meta_dma_addr);
1831 }
1832
1833 unmap:
1c2ad9fa 1834 nvme_unmap_user_pages(dev, io.opcode & 1, iod);
eca18b23 1835 nvme_free_iod(dev, iod);
f410c680
KB		 1836
1837 if (meta_iod) {
1838 nvme_unmap_user_pages(dev, io.opcode & 1, meta_iod);
1839 nvme_free_iod(dev, meta_iod);
1840 }
1841
a53295b6
MW		 1842 return status;
1843}
1844
a4aea562
MB		 1845static int nvme_user_cmd(struct nvme_dev *dev, struct nvme_ns *ns,
1846 struct nvme_passthru_cmd __user *ucmd)
6ee44cdc 1847{
7963e521 1848 struct nvme_passthru_cmd cmd;
6ee44cdc 1849 struct nvme_command c;
eca18b23 1850 int status, length;
c7d36ab8 1851 struct nvme_iod *uninitialized_var(iod);
94f370ca 1852 unsigned timeout;
6ee44cdc 1853
6bbf1acd
MW		 1854 if (!capable(CAP_SYS_ADMIN))
1855 return -EACCES;
1856 if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
6ee44cdc 1857 return -EFAULT;
6ee44cdc
MW		 1858
1859 memset(&c, 0, sizeof(c));
6bbf1acd
MW		 1860 c.common.opcode = cmd.opcode;
1861 c.common.flags = cmd.flags;
1862 c.common.nsid = cpu_to_le32(cmd.nsid);
1863 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1864 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1865 c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
1866 c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
1867 c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
1868 c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
1869 c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
1870 c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);
1871
1872 length = cmd.data_len;
1873 if (cmd.data_len) {
49742188
MW		 1874 iod = nvme_map_user_pages(dev, cmd.opcode & 1, cmd.addr,
1875 length);
eca18b23
MW		 1876 if (IS_ERR(iod))
1877 return PTR_ERR(iod);
edd10d33
KB		 1878 length = nvme_setup_prps(dev, iod, length, GFP_KERNEL);
1879 c.common.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
1880 c.common.prp2 = cpu_to_le64(iod->first_dma);
6bbf1acd
MW		 1881 }
1882
94f370ca
KB		 1883 timeout = cmd.timeout_ms ? msecs_to_jiffies(cmd.timeout_ms) :
1884 ADMIN_TIMEOUT;
a4aea562 1885
6bbf1acd 1886 if (length != cmd.data_len)
b77954cb 1887 status = -ENOMEM;
a4aea562
MB		 1888 else if (ns) {
1889 struct request *req;
1890
1891 req = blk_mq_alloc_request(ns->queue, WRITE,
1892 (GFP_KERNEL|__GFP_WAIT), false);
97fe3832
JA		 1893 if (IS_ERR(req))
1894 status = PTR_ERR(req);
a4aea562
MB		 1895 else {
1896 status = nvme_submit_sync_cmd(req, &c, &cmd.result,
1897 timeout);
9d135bb8 1898 blk_mq_free_request(req);
a4aea562
MB		 1899 }
1900 } else
1901 status = __nvme_submit_admin_cmd(dev, &c, &cmd.result, timeout);
eca18b23 1902
6bbf1acd 1903 if (cmd.data_len) {
1c2ad9fa 1904 nvme_unmap_user_pages(dev, cmd.opcode & 1, iod);
eca18b23 1905 nvme_free_iod(dev, iod);
6bbf1acd 1906 }
f4f117f6 1907
cf90bc48 1908 if ((status >= 0) && copy_to_user(&ucmd->result, &cmd.result,
f4f117f6
KB		 1909 sizeof(cmd.result)))
1910 status = -EFAULT;
1911
6ee44cdc
MW		 1912 return status;
1913}
1914
b60503ba
MW		 1915static int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd,
1916 unsigned long arg)
1917{
1918 struct nvme_ns *ns = bdev->bd_disk->private_data;
1919
1920 switch (cmd) {
6bbf1acd 1921 case NVME_IOCTL_ID:
c3bfe717 1922 force_successful_syscall_return();
6bbf1acd
MW		 1923 return ns->ns_id;
1924 case NVME_IOCTL_ADMIN_CMD:
a4aea562 1925 return nvme_user_cmd(ns->dev, NULL, (void __user *)arg);
7963e521 1926 case NVME_IOCTL_IO_CMD:
a4aea562 1927 return nvme_user_cmd(ns->dev, ns, (void __user *)arg);
a53295b6
MW		 1928 case NVME_IOCTL_SUBMIT_IO:
1929 return nvme_submit_io(ns, (void __user *)arg);
5d0f6131
VV		 1930 case SG_GET_VERSION_NUM:
1931 return nvme_sg_get_version_num((void __user *)arg);
1932 case SG_IO:
1933 return nvme_sg_io(ns, (void __user *)arg);
b60503ba
MW		 1934 default:
1935 return -ENOTTY;
1936 }
1937}
1938
320a3827
KB		 1939#ifdef CONFIG_COMPAT
1940static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
1941 unsigned int cmd, unsigned long arg)
1942{
320a3827
KB		 1943 switch (cmd) {
1944 case SG_IO:
e179729a 1945 return -ENOIOCTLCMD;
320a3827
KB		 1946 }
1947 return nvme_ioctl(bdev, mode, cmd, arg);
1948}
1949#else
1950#define nvme_compat_ioctl NULL
1951#endif
1952
9ac27090
KB		 1953static int nvme_open(struct block_device *bdev, fmode_t mode)
1954{
9e60352c
KB		 1955 int ret = 0;
1956 struct nvme_ns *ns;
9ac27090 1957
9e60352c
KB		 1958 spin_lock(&dev_list_lock);
1959 ns = bdev->bd_disk->private_data;
1960 if (!ns)
1961 ret = -ENXIO;
1962 else if (!kref_get_unless_zero(&ns->dev->kref))
1963 ret = -ENXIO;
1964 spin_unlock(&dev_list_lock);
1965
1966 return ret;
9ac27090
KB		 1967}
1968
1969static void nvme_free_dev(struct kref *kref);
1970
1971static void nvme_release(struct gendisk *disk, fmode_t mode)
1972{
1973 struct nvme_ns *ns = disk->private_data;
1974 struct nvme_dev *dev = ns->dev;
1975
1976 kref_put(&dev->kref, nvme_free_dev);
1977}
1978
4cc09e2d
KB		 1979static int nvme_getgeo(struct block_device *bd, struct hd_geometry *geo)
1980{
1981 /* some standard values */
1982 geo->heads = 1 << 6;
1983 geo->sectors = 1 << 5;
1984 geo->cylinders = get_capacity(bd->bd_disk) >> 11;
1985 return 0;
1986}
1987
e1e5e564
KB		 1988static void nvme_config_discard(struct nvme_ns *ns)
1989{
1990 u32 logical_block_size = queue_logical_block_size(ns->queue);
1991 ns->queue->limits.discard_zeroes_data = 0;
1992 ns->queue->limits.discard_alignment = logical_block_size;
1993 ns->queue->limits.discard_granularity = logical_block_size;
1994 ns->queue->limits.max_discard_sectors = 0xffffffff;
1995 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
1996}
1997
1998static int nvme_noop_verify(struct blk_integrity_iter *iter)
1999{
2000 return 0;
2001}
2002
2003static int nvme_noop_generate(struct blk_integrity_iter *iter)
2004{
2005 return 0;
2006}
2007
2008struct blk_integrity nvme_meta_noop = {
2009 .name = "NVME_META_NOOP",
2010 .generate_fn = nvme_noop_generate,
2011 .verify_fn = nvme_noop_verify,
2012};
2013
2014static void nvme_init_integrity(struct nvme_ns *ns)
2015{
2016 struct blk_integrity integrity;
2017
2018 switch (ns->pi_type) {
2019 case NVME_NS_DPS_PI_TYPE3:
2020 integrity = t10_pi_type3_crc;
2021 break;
2022 case NVME_NS_DPS_PI_TYPE1:
2023 case NVME_NS_DPS_PI_TYPE2:
2024 integrity = t10_pi_type1_crc;
2025 break;
2026 default:
2027 integrity = nvme_meta_noop;
2028 break;
2029 }
2030 integrity.tuple_size = ns->ms;
2031 blk_integrity_register(ns->disk, &integrity);
2032 blk_queue_max_integrity_segments(ns->queue, 1);
2033}
2034
1b9dbf7f
KB		 2035static int nvme_revalidate_disk(struct gendisk *disk)
2036{
2037 struct nvme_ns *ns = disk->private_data;
2038 struct nvme_dev *dev = ns->dev;
2039 struct nvme_id_ns *id;
2040 dma_addr_t dma_addr;
e1e5e564
KB		 2041 int lbaf, pi_type, old_ms;
2042 unsigned short bs;
1b9dbf7f
KB		 2043
2044 id = dma_alloc_coherent(&dev->pci_dev->dev, 4096, &dma_addr,
2045 GFP_KERNEL);
2046 if (!id) {
2047 dev_warn(&dev->pci_dev->dev, "%s: Memory alocation failure\n",
2048 __func__);
2049 return 0;
2050 }
e1e5e564
KB		 2051 if (nvme_identify(dev, ns->ns_id, 0, dma_addr)) {
2052 dev_warn(&dev->pci_dev->dev,
2053 "identify failed ns:%d, setting capacity to 0\n",
2054 ns->ns_id);
2055 memset(id, 0, sizeof(*id));
2056 }
1b9dbf7f 2057
e1e5e564
KB		 2058 old_ms = ns->ms;
2059 lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
1b9dbf7f 2060 ns->lba_shift = id->lbaf[lbaf].ds;
e1e5e564
KB		 2061 ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
2062
2063 /*
2064 * If identify namespace failed, use default 512 byte block size so
2065 * block layer can use before failing read/write for 0 capacity.
2066 */
2067 if (ns->lba_shift == 0)
2068 ns->lba_shift = 9;
2069 bs = 1 << ns->lba_shift;
2070
2071 /* XXX: PI implementation requires metadata equal t10 pi tuple size */
2072 pi_type = ns->ms == sizeof(struct t10_pi_tuple) ?
2073 id->dps & NVME_NS_DPS_PI_MASK : 0;
2074
2075 if (disk->integrity && (ns->pi_type != pi_type || ns->ms != old_ms ||
2076 bs != queue_logical_block_size(disk->queue) ||
2077 (ns->ms && id->flbas & NVME_NS_FLBAS_META_EXT)))
2078 blk_integrity_unregister(disk);
2079
2080 ns->pi_type = pi_type;
2081 blk_queue_logical_block_size(ns->queue, bs);
2082
2083 if (ns->ms && !disk->integrity && (disk->flags & GENHD_FL_UP) &&
2084 !(id->flbas & NVME_NS_FLBAS_META_EXT))
2085 nvme_init_integrity(ns);
2086
2087 if (id->ncap == 0 || (ns->ms && !disk->integrity))
2088 set_capacity(disk, 0);
2089 else
2090 set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));
2091
2092 if (dev->oncs & NVME_CTRL_ONCS_DSM)
2093 nvme_config_discard(ns);
1b9dbf7f 2094
1b9dbf7f
KB		 2095 dma_free_coherent(&dev->pci_dev->dev, 4096, id, dma_addr);
2096 return 0;
2097}
2098
b60503ba
MW		 2099static const struct block_device_operations nvme_fops = {
2100 .owner = THIS_MODULE,
2101 .ioctl = nvme_ioctl,
320a3827 2102 .compat_ioctl = nvme_compat_ioctl,
9ac27090
KB		 2103 .open = nvme_open,
2104 .release = nvme_release,
4cc09e2d 2105 .getgeo = nvme_getgeo,
1b9dbf7f 2106 .revalidate_disk= nvme_revalidate_disk,
b60503ba
MW		 2107};
2108
1fa6aead
MW		 2109static int nvme_kthread(void *data)
2110{
d4b4ff8e 2111 struct nvme_dev *dev, *next;
1fa6aead
MW		 2112
2113 while (!kthread_should_stop()) {
564a232c 2114 set_current_state(TASK_INTERRUPTIBLE);
1fa6aead 2115 spin_lock(&dev_list_lock);
d4b4ff8e 2116 list_for_each_entry_safe(dev, next, &dev_list, node) {
1fa6aead 2117 int i;
d4b4ff8e
KB		 2118 if (readl(&dev->bar->csts) & NVME_CSTS_CFS &&
2119 dev->initialized) {
2120 if (work_busy(&dev->reset_work))
2121 continue;
2122 list_del_init(&dev->node);
2123 dev_warn(&dev->pci_dev->dev,
a4aea562
MB		 2124 "Failed status: %x, reset controller\n",
2125 readl(&dev->bar->csts));
9ca97374 2126 dev->reset_workfn = nvme_reset_failed_dev;
d4b4ff8e
KB		 2127 queue_work(nvme_workq, &dev->reset_work);
2128 continue;
2129 }
1fa6aead 2130 for (i = 0; i < dev->queue_count; i++) {
a4aea562 2131 struct nvme_queue *nvmeq = dev->queues[i];
740216fc
MW		 2132 if (!nvmeq)
2133 continue;
1fa6aead 2134 spin_lock_irq(&nvmeq->q_lock);
bc57a0f7 2135 nvme_process_cq(nvmeq);
6fccf938
KB		 2136
2137 while ((i == 0) && (dev->event_limit > 0)) {
a4aea562 2138 if (nvme_submit_async_admin_req(dev))
6fccf938
KB		 2139 break;
2140 dev->event_limit--;
2141 }
1fa6aead
MW		 2142 spin_unlock_irq(&nvmeq->q_lock);
2143 }
2144 }
2145 spin_unlock(&dev_list_lock);
acb7aa0d 2146 schedule_timeout(round_jiffies_relative(HZ));
1fa6aead
MW		 2147 }
2148 return 0;
2149}
2150
e1e5e564 2151static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid)
b60503ba
MW		 2152{
2153 struct nvme_ns *ns;
2154 struct gendisk *disk;
a4aea562 2155 int node = dev_to_node(&dev->pci_dev->dev);
b60503ba 2156
a4aea562 2157 ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
b60503ba 2158 if (!ns)
e1e5e564
KB		 2159 return;
2160
a4aea562 2161 ns->queue = blk_mq_init_queue(&dev->tagset);
9f173b33 2162 if (IS_ERR(ns->queue))
b60503ba 2163 goto out_free_ns;
4eeb9215
MW		 2164 queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue);
2165 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
a4aea562 2166 queue_flag_set_unlocked(QUEUE_FLAG_SG_GAPS, ns->queue);
b60503ba
MW		 2167 ns->dev = dev;
2168 ns->queue->queuedata = ns;
2169
a4aea562 2170 disk = alloc_disk_node(0, node);
b60503ba
MW		 2171 if (!disk)
2172 goto out_free_queue;
a4aea562 2173
5aff9382 2174 ns->ns_id = nsid;
b60503ba 2175 ns->disk = disk;
e1e5e564
KB		 2176 ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
2177 list_add_tail(&ns->list, &dev->namespaces);
2178
e9ef4636 2179 blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
8fc23e03
KB		 2180 if (dev->max_hw_sectors)
2181 blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors);
a4aea562
MB		 2182 if (dev->stripe_size)
2183 blk_queue_chunk_sectors(ns->queue, dev->stripe_size >> 9);
a7d2ce28
KB		 2184 if (dev->vwc & NVME_CTRL_VWC_PRESENT)
2185 blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA);
b60503ba
MW		 2186
2187 disk->major = nvme_major;
469071a3 2188 disk->first_minor = 0;
b60503ba
MW		 2189 disk->fops = &nvme_fops;
2190 disk->private_data = ns;
2191 disk->queue = ns->queue;
388f037f 2192 disk->driverfs_dev = &dev->pci_dev->dev;
469071a3 2193 disk->flags = GENHD_FL_EXT_DEVT;
5aff9382 2194 sprintf(disk->disk_name, "nvme%dn%d", dev->instance, nsid);
b60503ba 2195
e1e5e564
KB		 2196 /*
2197 * Initialize capacity to 0 until we establish the namespace format and
2198 * setup integrity extentions if necessary. The revalidate_disk after
2199 * add_disk allows the driver to register with integrity if the format
2200 * requires it.
2201 */
2202 set_capacity(disk, 0);
2203 nvme_revalidate_disk(ns->disk);
2204 add_disk(ns->disk);
2205 if (ns->ms)
2206 revalidate_disk(ns->disk);
2207 return;
b60503ba
MW		 2208 out_free_queue:
2209 blk_cleanup_queue(ns->queue);
2210 out_free_ns:
2211 kfree(ns);
b60503ba
MW		 2212}
2213
42f61420
KB		 2214static void nvme_create_io_queues(struct nvme_dev *dev)
2215{
a4aea562 2216 unsigned i;
42f61420 2217
a4aea562 2218 for (i = dev->queue_count; i <= dev->max_qid; i++)
2b25d981 2219 if (!nvme_alloc_queue(dev, i, dev->q_depth))
42f61420
KB		 2220 break;
2221
a4aea562
MB		 2222 for (i = dev->online_queues; i <= dev->queue_count - 1; i++)
2223 if (nvme_create_queue(dev->queues[i], i))
42f61420
KB		 2224 break;
2225}
2226
b3b06812 2227static int set_queue_count(struct nvme_dev *dev, int count)
b60503ba
MW		 2228{
2229 int status;
2230 u32 result;
b3b06812 2231 u32 q_count = (count - 1) | ((count - 1) << 16);
b60503ba 2232
df348139 2233 status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES, q_count, 0,
bc5fc7e4 2234 &result);
27e8166c
MW		 2235 if (status < 0)
2236 return status;
2237 if (status > 0) {
2238 dev_err(&dev->pci_dev->dev, "Could not set queue count (%d)\n",
2239 status);
badc34d4 2240 return 0;
27e8166c 2241 }
b60503ba
MW		 2242 return min(result & 0xffff, result >> 16) + 1;
2243}
2244
9d713c2b
KB		 2245static size_t db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
2246{
b80d5ccc 2247 return 4096 + ((nr_io_queues + 1) * 8 * dev->db_stride);
9d713c2b
KB		 2248}
2249
8d85fce7 2250static int nvme_setup_io_queues(struct nvme_dev *dev)
b60503ba 2251{
a4aea562 2252 struct nvme_queue *adminq = dev->queues[0];
fa08a396 2253 struct pci_dev *pdev = dev->pci_dev;
42f61420 2254 int result, i, vecs, nr_io_queues, size;
b60503ba 2255
42f61420 2256 nr_io_queues = num_possible_cpus();
b348b7d5 2257 result = set_queue_count(dev, nr_io_queues);
badc34d4 2258 if (result <= 0)
1b23484b 2259 return result;
b348b7d5
MW		 2260 if (result < nr_io_queues)
2261 nr_io_queues = result;
b60503ba 2262
9d713c2b
KB		 2263 size = db_bar_size(dev, nr_io_queues);
2264 if (size > 8192) {
f1938f6e 2265 iounmap(dev->bar);
9d713c2b
KB		 2266 do {
2267 dev->bar = ioremap(pci_resource_start(pdev, 0), size);
2268 if (dev->bar)
2269 break;
2270 if (!--nr_io_queues)
2271 return -ENOMEM;
2272 size = db_bar_size(dev, nr_io_queues);
2273 } while (1);
f1938f6e 2274 dev->dbs = ((void __iomem *)dev->bar) + 4096;
5a92e700 2275 adminq->q_db = dev->dbs;
f1938f6e
MW		 2276 }
2277
9d713c2b 2278 /* Deregister the admin queue's interrupt */
3193f07b 2279 free_irq(dev->entry[0].vector, adminq);
9d713c2b 2280
e32efbfc
JA		 2281 /*
2282 * If we enable msix early due to not intx, disable it again before
2283 * setting up the full range we need.
2284 */
2285 if (!pdev->irq)
2286 pci_disable_msix(pdev);
2287
be577fab 2288 for (i = 0; i < nr_io_queues; i++)
1b23484b 2289 dev->entry[i].entry = i;
be577fab
AG		 2290 vecs = pci_enable_msix_range(pdev, dev->entry, 1, nr_io_queues);
2291 if (vecs < 0) {
2292 vecs = pci_enable_msi_range(pdev, 1, min(nr_io_queues, 32));
2293 if (vecs < 0) {
2294 vecs = 1;
2295 } else {
2296 for (i = 0; i < vecs; i++)
2297 dev->entry[i].vector = i + pdev->irq;
fa08a396
RRG		 2298 }
2299 }
2300
063a8096
MW		 2301 /*
2302 * Should investigate if there's a performance win from allocating
2303 * more queues than interrupt vectors; it might allow the submission
2304 * path to scale better, even if the receive path is limited by the
2305 * number of interrupts.
2306 */
2307 nr_io_queues = vecs;
42f61420 2308 dev->max_qid = nr_io_queues;
063a8096 2309
3193f07b 2310 result = queue_request_irq(dev, adminq, adminq->irqname);
a4aea562 2311 if (result)
22404274 2312 goto free_queues;
1b23484b 2313
cd638946 2314 /* Free previously allocated queues that are no longer usable */
42f61420 2315 nvme_free_queues(dev, nr_io_queues + 1);
a4aea562 2316 nvme_create_io_queues(dev);
9ecdc946 2317
22404274 2318 return 0;
b60503ba 2319
22404274 2320 free_queues:
a1a5ef99 2321 nvme_free_queues(dev, 1);
22404274 2322 return result;
b60503ba
MW		 2323}
2324
422ef0c7
MW		 2325/*
2326 * Return: error value if an error occurred setting up the queues or calling
2327 * Identify Device. 0 if these succeeded, even if adding some of the
2328 * namespaces failed. At the moment, these failures are silent. TBD which
2329 * failures should be reported.
2330 */
8d85fce7 2331static int nvme_dev_add(struct nvme_dev *dev)
b60503ba 2332{
68608c26 2333 struct pci_dev *pdev = dev->pci_dev;
c3bfe717
MW		 2334 int res;
2335 unsigned nn, i;
51814232 2336 struct nvme_id_ctrl *ctrl;
bc5fc7e4 2337 void *mem;
b60503ba 2338 dma_addr_t dma_addr;
159b67d7 2339 int shift = NVME_CAP_MPSMIN(readq(&dev->bar->cap)) + 12;
b60503ba 2340
e1e5e564 2341 mem = dma_alloc_coherent(&pdev->dev, 4096, &dma_addr, GFP_KERNEL);
a9ef4343
KB		 2342 if (!mem)
2343 return -ENOMEM;
b60503ba 2344
bc5fc7e4 2345 res = nvme_identify(dev, 0, 1, dma_addr);
b60503ba 2346 if (res) {
27e8166c 2347 dev_err(&pdev->dev, "Identify Controller failed (%d)\n", res);
e1e5e564
KB		 2348 dma_free_coherent(&dev->pci_dev->dev, 4096, mem, dma_addr);
2349 return -EIO;
b60503ba
MW		 2350 }
2351
bc5fc7e4 2352 ctrl = mem;
51814232 2353 nn = le32_to_cpup(&ctrl->nn);
0e5e4f0e 2354 dev->oncs = le16_to_cpup(&ctrl->oncs);
c30341dc 2355 dev->abort_limit = ctrl->acl + 1;
a7d2ce28 2356 dev->vwc = ctrl->vwc;
6fccf938 2357 dev->event_limit = min(ctrl->aerl + 1, 8);
51814232
MW		 2358 memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn));
2359 memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn));
2360 memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr));
159b67d7 2361 if (ctrl->mdts)
8fc23e03 2362 dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9);
68608c26 2363 if ((pdev->vendor == PCI_VENDOR_ID_INTEL) &&
a4aea562
MB		 2364 (pdev->device == 0x0953) && ctrl->vs[3]) {
2365 unsigned int max_hw_sectors;
2366
159b67d7 2367 dev->stripe_size = 1 << (ctrl->vs[3] + shift);
a4aea562
MB		 2368 max_hw_sectors = dev->stripe_size >> (shift - 9);
2369 if (dev->max_hw_sectors) {
2370 dev->max_hw_sectors = min(max_hw_sectors,
2371 dev->max_hw_sectors);
2372 } else
2373 dev->max_hw_sectors = max_hw_sectors;
2374 }
e1e5e564 2375 dma_free_coherent(&dev->pci_dev->dev, 4096, mem, dma_addr);
a4aea562
MB		 2376
2377 dev->tagset.ops = &nvme_mq_ops;
2378 dev->tagset.nr_hw_queues = dev->online_queues - 1;
2379 dev->tagset.timeout = NVME_IO_TIMEOUT;
2380 dev->tagset.numa_node = dev_to_node(&dev->pci_dev->dev);
2381 dev->tagset.queue_depth =
2382 min_t(int, dev->q_depth, BLK_MQ_MAX_DEPTH) - 1;
ac3dd5bd 2383 dev->tagset.cmd_size = nvme_cmd_size(dev);
a4aea562
MB		 2384 dev->tagset.flags = BLK_MQ_F_SHOULD_MERGE;
2385 dev->tagset.driver_data = dev;
2386
2387 if (blk_mq_alloc_tag_set(&dev->tagset))
e1e5e564 2388 return 0;
b60503ba 2389
e1e5e564
KB		 2390 for (i = 1; i <= nn; i++)
2391 nvme_alloc_ns(dev, i);
b60503ba 2392
e1e5e564 2393 return 0;
b60503ba
MW		 2394}
2395
0877cb0d
KB		 2396static int nvme_dev_map(struct nvme_dev *dev)
2397{
42f61420 2398 u64 cap;
0877cb0d
KB		 2399 int bars, result = -ENOMEM;
2400 struct pci_dev *pdev = dev->pci_dev;
2401
2402 if (pci_enable_device_mem(pdev))
2403 return result;
2404
2405 dev->entry[0].vector = pdev->irq;
2406 pci_set_master(pdev);
2407 bars = pci_select_bars(pdev, IORESOURCE_MEM);
be7837e8
JA		 2408 if (!bars)
2409 goto disable_pci;
2410
0877cb0d
KB		 2411 if (pci_request_selected_regions(pdev, bars, "nvme"))
2412 goto disable_pci;
2413
052d0efa
RK		 2414 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) &&
2415 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)))
2416 goto disable;
0877cb0d 2417
0877cb0d
KB		 2418 dev->bar = ioremap(pci_resource_start(pdev, 0), 8192);
2419 if (!dev->bar)
2420 goto disable;
e32efbfc 2421
0e53d180
KB		 2422 if (readl(&dev->bar->csts) == -1) {
2423 result = -ENODEV;
2424 goto unmap;
2425 }
e32efbfc
JA		 2426
2427 /*
2428 * Some devices don't advertse INTx interrupts, pre-enable a single
2429 * MSIX vec for setup. We'll adjust this later.
2430 */
2431 if (!pdev->irq) {
2432 result = pci_enable_msix(pdev, dev->entry, 1);
2433 if (result < 0)
2434 goto unmap;
2435 }
2436
42f61420
KB		 2437 cap = readq(&dev->bar->cap);
2438 dev->q_depth = min_t(int, NVME_CAP_MQES(cap) + 1, NVME_Q_DEPTH);
2439 dev->db_stride = 1 << NVME_CAP_STRIDE(cap);
0877cb0d
KB		 2440 dev->dbs = ((void __iomem *)dev->bar) + 4096;
2441
2442 return 0;
2443
0e53d180
KB		 2444 unmap:
2445 iounmap(dev->bar);
2446 dev->bar = NULL;
0877cb0d
KB		 2447 disable:
2448 pci_release_regions(pdev);
2449 disable_pci:
2450 pci_disable_device(pdev);
2451 return result;
2452}
2453
2454static void nvme_dev_unmap(struct nvme_dev *dev)
2455{
2456 if (dev->pci_dev->msi_enabled)
2457 pci_disable_msi(dev->pci_dev);
2458 else if (dev->pci_dev->msix_enabled)
2459 pci_disable_msix(dev->pci_dev);
2460
2461 if (dev->bar) {
2462 iounmap(dev->bar);
2463 dev->bar = NULL;
9a6b9458 2464 pci_release_regions(dev->pci_dev);
0877cb0d
KB		 2465 }
2466
0877cb0d
KB		 2467 if (pci_is_enabled(dev->pci_dev))
2468 pci_disable_device(dev->pci_dev);
2469}
2470
4d115420
KB		 2471struct nvme_delq_ctx {
2472 struct task_struct *waiter;
2473 struct kthread_worker *worker;
2474 atomic_t refcount;
2475};
2476
2477static void nvme_wait_dq(struct nvme_delq_ctx *dq, struct nvme_dev *dev)
2478{
2479 dq->waiter = current;
2480 mb();
2481
2482 for (;;) {
2483 set_current_state(TASK_KILLABLE);
2484 if (!atomic_read(&dq->refcount))
2485 break;
2486 if (!schedule_timeout(ADMIN_TIMEOUT) ||
2487 fatal_signal_pending(current)) {
0fb59cbc
KB		 2488 /*
2489 * Disable the controller first since we can't trust it
2490 * at this point, but leave the admin queue enabled
2491 * until all queue deletion requests are flushed.
2492 * FIXME: This may take a while if there are more h/w
2493 * queues than admin tags.
2494 */
4d115420 2495 set_current_state(TASK_RUNNING);
4d115420 2496 nvme_disable_ctrl(dev, readq(&dev->bar->cap));
0fb59cbc 2497 nvme_clear_queue(dev->queues[0]);
4d115420 2498 flush_kthread_worker(dq->worker);
0fb59cbc 2499 nvme_disable_queue(dev, 0);
4d115420
KB		 2500 return;
2501 }
2502 }
2503 set_current_state(TASK_RUNNING);
2504}
2505
2506static void nvme_put_dq(struct nvme_delq_ctx *dq)
2507{
2508 atomic_dec(&dq->refcount);
2509 if (dq->waiter)
2510 wake_up_process(dq->waiter);
2511}
2512
2513static struct nvme_delq_ctx *nvme_get_dq(struct nvme_delq_ctx *dq)
2514{
2515 atomic_inc(&dq->refcount);
2516 return dq;
2517}
2518
2519static void nvme_del_queue_end(struct nvme_queue *nvmeq)
2520{
2521 struct nvme_delq_ctx *dq = nvmeq->cmdinfo.ctx;
2522
2523 nvme_clear_queue(nvmeq);
2524 nvme_put_dq(dq);
2525}
2526
2527static int adapter_async_del_queue(struct nvme_queue *nvmeq, u8 opcode,
2528 kthread_work_func_t fn)
2529{
2530 struct nvme_command c;
2531
2532 memset(&c, 0, sizeof(c));
2533 c.delete_queue.opcode = opcode;
2534 c.delete_queue.qid = cpu_to_le16(nvmeq->qid);
2535
2536 init_kthread_work(&nvmeq->cmdinfo.work, fn);
a4aea562
MB		 2537 return nvme_submit_admin_async_cmd(nvmeq->dev, &c, &nvmeq->cmdinfo,
2538 ADMIN_TIMEOUT);
4d115420
KB		 2539}
2540
2541static void nvme_del_cq_work_handler(struct kthread_work *work)
2542{
2543 struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
2544 cmdinfo.work);
2545 nvme_del_queue_end(nvmeq);
2546}
2547
2548static int nvme_delete_cq(struct nvme_queue *nvmeq)
2549{
2550 return adapter_async_del_queue(nvmeq, nvme_admin_delete_cq,
2551 nvme_del_cq_work_handler);
2552}
2553
2554static void nvme_del_sq_work_handler(struct kthread_work *work)
2555{
2556 struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
2557 cmdinfo.work);
2558 int status = nvmeq->cmdinfo.status;
2559
2560 if (!status)
2561 status = nvme_delete_cq(nvmeq);
2562 if (status)
2563 nvme_del_queue_end(nvmeq);
2564}
2565
2566static int nvme_delete_sq(struct nvme_queue *nvmeq)
2567{
2568 return adapter_async_del_queue(nvmeq, nvme_admin_delete_sq,
2569 nvme_del_sq_work_handler);
2570}
2571
2572static void nvme_del_queue_start(struct kthread_work *work)
2573{
2574 struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
2575 cmdinfo.work);
4d115420
KB		 2576 if (nvme_delete_sq(nvmeq))
2577 nvme_del_queue_end(nvmeq);
2578}
2579
2580static void nvme_disable_io_queues(struct nvme_dev *dev)
2581{
2582 int i;
2583 DEFINE_KTHREAD_WORKER_ONSTACK(worker);
2584 struct nvme_delq_ctx dq;
2585 struct task_struct *kworker_task = kthread_run(kthread_worker_fn,
2586 &worker, "nvme%d", dev->instance);
2587
2588 if (IS_ERR(kworker_task)) {
2589 dev_err(&dev->pci_dev->dev,
2590 "Failed to create queue del task\n");
2591 for (i = dev->queue_count - 1; i > 0; i--)
2592 nvme_disable_queue(dev, i);
2593 return;
2594 }
2595
2596 dq.waiter = NULL;
2597 atomic_set(&dq.refcount, 0);
2598 dq.worker = &worker;
2599 for (i = dev->queue_count - 1; i > 0; i--) {
a4aea562 2600 struct nvme_queue *nvmeq = dev->queues[i];
4d115420
KB		 2601
2602 if (nvme_suspend_queue(nvmeq))
2603 continue;
2604 nvmeq->cmdinfo.ctx = nvme_get_dq(&dq);
2605 nvmeq->cmdinfo.worker = dq.worker;
2606 init_kthread_work(&nvmeq->cmdinfo.work, nvme_del_queue_start);
2607 queue_kthread_work(dq.worker, &nvmeq->cmdinfo.work);
2608 }
2609 nvme_wait_dq(&dq, dev);
2610 kthread_stop(kworker_task);
2611}
2612
b9afca3e
DM		 2613/*
2614* Remove the node from the device list and check
2615* for whether or not we need to stop the nvme_thread.
2616*/
2617static void nvme_dev_list_remove(struct nvme_dev *dev)
2618{
2619 struct task_struct *tmp = NULL;
2620
2621 spin_lock(&dev_list_lock);
2622 list_del_init(&dev->node);
2623 if (list_empty(&dev_list) && !IS_ERR_OR_NULL(nvme_thread)) {
2624 tmp = nvme_thread;
2625 nvme_thread = NULL;
2626 }
2627 spin_unlock(&dev_list_lock);
2628
2629 if (tmp)
2630 kthread_stop(tmp);
2631}
2632
c9d3bf88
KB		 2633static void nvme_freeze_queues(struct nvme_dev *dev)
2634{
2635 struct nvme_ns *ns;
2636
2637 list_for_each_entry(ns, &dev->namespaces, list) {
2638 blk_mq_freeze_queue_start(ns->queue);
2639
2640 spin_lock(ns->queue->queue_lock);
2641 queue_flag_set(QUEUE_FLAG_STOPPED, ns->queue);
2642 spin_unlock(ns->queue->queue_lock);
2643
2644 blk_mq_cancel_requeue_work(ns->queue);
2645 blk_mq_stop_hw_queues(ns->queue);
2646 }
2647}
2648
2649static void nvme_unfreeze_queues(struct nvme_dev *dev)
2650{
2651 struct nvme_ns *ns;
2652
2653 list_for_each_entry(ns, &dev->namespaces, list) {
2654 queue_flag_clear_unlocked(QUEUE_FLAG_STOPPED, ns->queue);
2655 blk_mq_unfreeze_queue(ns->queue);
2656 blk_mq_start_stopped_hw_queues(ns->queue, true);
2657 blk_mq_kick_requeue_list(ns->queue);
2658 }
2659}
2660
f0b50732 2661static void nvme_dev_shutdown(struct nvme_dev *dev)
b60503ba 2662{
22404274 2663 int i;
7c1b2450 2664 u32 csts = -1;
22404274 2665
d4b4ff8e 2666 dev->initialized = 0;
b9afca3e 2667 nvme_dev_list_remove(dev);
1fa6aead 2668
c9d3bf88
KB		 2669 if (dev->bar) {
2670 nvme_freeze_queues(dev);
7c1b2450 2671 csts = readl(&dev->bar->csts);
c9d3bf88 2672 }
7c1b2450 2673 if (csts & NVME_CSTS_CFS || !(csts & NVME_CSTS_RDY)) {
4d115420 2674 for (i = dev->queue_count - 1; i >= 0; i--) {
a4aea562 2675 struct nvme_queue *nvmeq = dev->queues[i];
4d115420
KB		 2676 nvme_suspend_queue(nvmeq);
2677 nvme_clear_queue(nvmeq);
2678 }
2679 } else {
2680 nvme_disable_io_queues(dev);
1894d8f1 2681 nvme_shutdown_ctrl(dev);
4d115420
KB		 2682 nvme_disable_queue(dev, 0);
2683 }
f0b50732
KB		 2684 nvme_dev_unmap(dev);
2685}
2686
2687static void nvme_dev_remove(struct nvme_dev *dev)
2688{
9ac27090 2689 struct nvme_ns *ns;
f0b50732 2690
9ac27090 2691 list_for_each_entry(ns, &dev->namespaces, list) {
e1e5e564
KB		 2692 if (ns->disk->flags & GENHD_FL_UP) {
2693 if (ns->disk->integrity)
2694 blk_integrity_unregister(ns->disk);
9ac27090 2695 del_gendisk(ns->disk);
e1e5e564 2696 }
cef6a948
KB		 2697 if (!blk_queue_dying(ns->queue)) {
2698 blk_mq_abort_requeue_list(ns->queue);
9ac27090 2699 blk_cleanup_queue(ns->queue);
cef6a948 2700 }
b60503ba 2701 }
b60503ba
MW		 2702}
2703
091b6092
MW		 2704static int nvme_setup_prp_pools(struct nvme_dev *dev)
2705{
2706 struct device *dmadev = &dev->pci_dev->dev;
2707 dev->prp_page_pool = dma_pool_create("prp list page", dmadev,
2708 PAGE_SIZE, PAGE_SIZE, 0);
2709 if (!dev->prp_page_pool)
2710 return -ENOMEM;
2711
99802a7a
MW		 2712 /* Optimisation for I/Os between 4k and 128k */
2713 dev->prp_small_pool = dma_pool_create("prp list 256", dmadev,
2714 256, 256, 0);
2715 if (!dev->prp_small_pool) {
2716 dma_pool_destroy(dev->prp_page_pool);
2717 return -ENOMEM;
2718 }
091b6092
MW		 2719 return 0;
2720}
2721
2722static void nvme_release_prp_pools(struct nvme_dev *dev)
2723{
2724 dma_pool_destroy(dev->prp_page_pool);
99802a7a 2725 dma_pool_destroy(dev->prp_small_pool);
091b6092
MW		 2726}
2727
cd58ad7d
QSA		 2728static DEFINE_IDA(nvme_instance_ida);
2729
2730static int nvme_set_instance(struct nvme_dev *dev)
b60503ba 2731{
cd58ad7d
QSA		 2732 int instance, error;
2733
2734 do {
2735 if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
2736 return -ENODEV;
2737
2738 spin_lock(&dev_list_lock);
2739 error = ida_get_new(&nvme_instance_ida, &instance);
2740 spin_unlock(&dev_list_lock);
2741 } while (error == -EAGAIN);
2742
2743 if (error)
2744 return -ENODEV;
2745
2746 dev->instance = instance;
2747 return 0;
b60503ba
MW		 2748}
2749
2750static void nvme_release_instance(struct nvme_dev *dev)
2751{
cd58ad7d
QSA		 2752 spin_lock(&dev_list_lock);
2753 ida_remove(&nvme_instance_ida, dev->instance);
2754 spin_unlock(&dev_list_lock);
b60503ba
MW		 2755}
2756
9ac27090
KB		 2757static void nvme_free_namespaces(struct nvme_dev *dev)
2758{
2759 struct nvme_ns *ns, *next;
2760
2761 list_for_each_entry_safe(ns, next, &dev->namespaces, list) {
2762 list_del(&ns->list);
9e60352c
KB		 2763
2764 spin_lock(&dev_list_lock);
2765 ns->disk->private_data = NULL;
2766 spin_unlock(&dev_list_lock);
2767
9ac27090
KB		 2768 put_disk(ns->disk);
2769 kfree(ns);
2770 }
2771}
2772
5e82e952
KB		 2773static void nvme_free_dev(struct kref *kref)
2774{
2775 struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref);
9ac27090 2776
a96d4f5c 2777 pci_dev_put(dev->pci_dev);
9ac27090 2778 nvme_free_namespaces(dev);
285dffc9 2779 nvme_release_instance(dev);
a4aea562 2780 blk_mq_free_tag_set(&dev->tagset);
ea191d2f 2781 blk_put_queue(dev->admin_q);
5e82e952
KB		 2782 kfree(dev->queues);
2783 kfree(dev->entry);
2784 kfree(dev);
2785}
2786
2787static int nvme_dev_open(struct inode *inode, struct file *f)
2788{
2789 struct nvme_dev *dev = container_of(f->private_data, struct nvme_dev,
2790 miscdev);
2791 kref_get(&dev->kref);
2792 f->private_data = dev;
2793 return 0;
2794}
2795
2796static int nvme_dev_release(struct inode *inode, struct file *f)
2797{
2798 struct nvme_dev *dev = f->private_data;
2799 kref_put(&dev->kref, nvme_free_dev);
2800 return 0;
2801}
2802
2803static long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
2804{
2805 struct nvme_dev *dev = f->private_data;
a4aea562
MB		 2806 struct nvme_ns *ns;
2807
5e82e952
KB		 2808 switch (cmd) {
2809 case NVME_IOCTL_ADMIN_CMD:
a4aea562 2810 return nvme_user_cmd(dev, NULL, (void __user *)arg);
7963e521 2811 case NVME_IOCTL_IO_CMD:
a4aea562
MB		 2812 if (list_empty(&dev->namespaces))
2813 return -ENOTTY;
2814 ns = list_first_entry(&dev->namespaces, struct nvme_ns, list);
2815 return nvme_user_cmd(dev, ns, (void __user *)arg);
5e82e952
KB		 2816 default:
2817 return -ENOTTY;
2818 }
2819}
2820
2821static const struct file_operations nvme_dev_fops = {
2822 .owner = THIS_MODULE,
2823 .open = nvme_dev_open,
2824 .release = nvme_dev_release,
2825 .unlocked_ioctl = nvme_dev_ioctl,
2826 .compat_ioctl = nvme_dev_ioctl,
2827};
2828
a4aea562
MB		 2829static void nvme_set_irq_hints(struct nvme_dev *dev)
2830{
2831 struct nvme_queue *nvmeq;
2832 int i;
2833
2834 for (i = 0; i < dev->online_queues; i++) {
2835 nvmeq = dev->queues[i];
2836
2837 if (!nvmeq->hctx)
2838 continue;
2839
2840 irq_set_affinity_hint(dev->entry[nvmeq->cq_vector].vector,
2841 nvmeq->hctx->cpumask);
2842 }
2843}
2844
f0b50732
KB		 2845static int nvme_dev_start(struct nvme_dev *dev)
2846{
2847 int result;
b9afca3e 2848 bool start_thread = false;
f0b50732
KB		 2849
2850 result = nvme_dev_map(dev);
2851 if (result)
2852 return result;
2853
2854 result = nvme_configure_admin_queue(dev);
2855 if (result)
2856 goto unmap;
2857
2858 spin_lock(&dev_list_lock);
b9afca3e
DM		 2859 if (list_empty(&dev_list) && IS_ERR_OR_NULL(nvme_thread)) {
2860 start_thread = true;
2861 nvme_thread = NULL;
2862 }
f0b50732
KB		 2863 list_add(&dev->node, &dev_list);
2864 spin_unlock(&dev_list_lock);
2865
b9afca3e
DM		 2866 if (start_thread) {
2867 nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");
387caa5a 2868 wake_up_all(&nvme_kthread_wait);
b9afca3e
DM		 2869 } else
2870 wait_event_killable(nvme_kthread_wait, nvme_thread);
2871
2872 if (IS_ERR_OR_NULL(nvme_thread)) {
2873 result = nvme_thread ? PTR_ERR(nvme_thread) : -EINTR;
2874 goto disable;
2875 }
a4aea562
MB		 2876
2877 nvme_init_queue(dev->queues[0], 0);
0fb59cbc
KB		 2878 result = nvme_alloc_admin_tags(dev);
2879 if (result)
2880 goto disable;
b9afca3e 2881
f0b50732 2882 result = nvme_setup_io_queues(dev);
badc34d4 2883 if (result)
0fb59cbc 2884 goto free_tags;
f0b50732 2885
a4aea562
MB		 2886 nvme_set_irq_hints(dev);
2887
d82e8bfd 2888 return result;
f0b50732 2889
0fb59cbc
KB		 2890 free_tags:
2891 nvme_dev_remove_admin(dev);
f0b50732 2892 disable:
a1a5ef99 2893 nvme_disable_queue(dev, 0);
b9afca3e 2894 nvme_dev_list_remove(dev);
f0b50732
KB		 2895 unmap:
2896 nvme_dev_unmap(dev);
2897 return result;
2898}
2899
9a6b9458
KB		 2900static int nvme_remove_dead_ctrl(void *arg)
2901{
2902 struct nvme_dev *dev = (struct nvme_dev *)arg;
2903 struct pci_dev *pdev = dev->pci_dev;
2904
2905 if (pci_get_drvdata(pdev))
c81f4975 2906 pci_stop_and_remove_bus_device_locked(pdev);
9a6b9458
KB		 2907 kref_put(&dev->kref, nvme_free_dev);
2908 return 0;
2909}
2910
2911static void nvme_remove_disks(struct work_struct *ws)
2912{
9a6b9458
KB		 2913 struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work);
2914
5a92e700 2915 nvme_free_queues(dev, 1);
302c6727 2916 nvme_dev_remove(dev);
9a6b9458
KB		 2917}
2918
2919static int nvme_dev_resume(struct nvme_dev *dev)
2920{
2921 int ret;
2922
2923 ret = nvme_dev_start(dev);
badc34d4 2924 if (ret)
9a6b9458 2925 return ret;
badc34d4 2926 if (dev->online_queues < 2) {
9a6b9458 2927 spin_lock(&dev_list_lock);
9ca97374 2928 dev->reset_workfn = nvme_remove_disks;
9a6b9458
KB		 2929 queue_work(nvme_workq, &dev->reset_work);
2930 spin_unlock(&dev_list_lock);
c9d3bf88
KB		 2931 } else {
2932 nvme_unfreeze_queues(dev);
2933 nvme_set_irq_hints(dev);
9a6b9458 2934 }
d4b4ff8e 2935 dev->initialized = 1;
9a6b9458
KB		 2936 return 0;
2937}
2938
2939static void nvme_dev_reset(struct nvme_dev *dev)
2940{
2941 nvme_dev_shutdown(dev);
2942 if (nvme_dev_resume(dev)) {
a4aea562 2943 dev_warn(&dev->pci_dev->dev, "Device failed to resume\n");
9a6b9458
KB		 2944 kref_get(&dev->kref);
2945 if (IS_ERR(kthread_run(nvme_remove_dead_ctrl, dev, "nvme%d",
2946 dev->instance))) {
2947 dev_err(&dev->pci_dev->dev,
2948 "Failed to start controller remove task\n");
2949 kref_put(&dev->kref, nvme_free_dev);
2950 }
2951 }
2952}
2953
2954static void nvme_reset_failed_dev(struct work_struct *ws)
2955{
2956 struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work);
2957 nvme_dev_reset(dev);
2958}
2959
9ca97374
TH		 2960static void nvme_reset_workfn(struct work_struct *work)
2961{
2962 struct nvme_dev *dev = container_of(work, struct nvme_dev, reset_work);
2963 dev->reset_workfn(work);
2964}
2965
8d85fce7 2966static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
b60503ba 2967{
a4aea562 2968 int node, result = -ENOMEM;
b60503ba
MW		 2969 struct nvme_dev *dev;
2970
a4aea562
MB		 2971 node = dev_to_node(&pdev->dev);
2972 if (node == NUMA_NO_NODE)
2973 set_dev_node(&pdev->dev, 0);
2974
2975 dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
b60503ba
MW		 2976 if (!dev)
2977 return -ENOMEM;
a4aea562
MB		 2978 dev->entry = kzalloc_node(num_possible_cpus() * sizeof(*dev->entry),
2979 GFP_KERNEL, node);
b60503ba
MW		 2980 if (!dev->entry)
2981 goto free;
a4aea562
MB		 2982 dev->queues = kzalloc_node((num_possible_cpus() + 1) * sizeof(void *),
2983 GFP_KERNEL, node);
b60503ba
MW		 2984 if (!dev->queues)
2985 goto free;
2986
2987 INIT_LIST_HEAD(&dev->namespaces);
9ca97374
TH		 2988 dev->reset_workfn = nvme_reset_failed_dev;
2989 INIT_WORK(&dev->reset_work, nvme_reset_workfn);
a96d4f5c 2990 dev->pci_dev = pci_dev_get(pdev);
9a6b9458 2991 pci_set_drvdata(pdev, dev);
cd58ad7d
QSA		 2992 result = nvme_set_instance(dev);
2993 if (result)
a96d4f5c 2994 goto put_pci;
b60503ba 2995
091b6092
MW		 2996 result = nvme_setup_prp_pools(dev);
2997 if (result)
0877cb0d 2998 goto release;
091b6092 2999
fb35e914 3000 kref_init(&dev->kref);
f0b50732 3001 result = nvme_dev_start(dev);
badc34d4 3002 if (result)
0877cb0d 3003 goto release_pools;
b60503ba 3004
badc34d4
KB		 3005 if (dev->online_queues > 1)
3006 result = nvme_dev_add(dev);
d82e8bfd 3007 if (result)
f0b50732 3008 goto shutdown;
740216fc 3009
5e82e952
KB		 3010 scnprintf(dev->name, sizeof(dev->name), "nvme%d", dev->instance);
3011 dev->miscdev.minor = MISC_DYNAMIC_MINOR;
3012 dev->miscdev.parent = &pdev->dev;
3013 dev->miscdev.name = dev->name;
3014 dev->miscdev.fops = &nvme_dev_fops;
3015 result = misc_register(&dev->miscdev);
3016 if (result)
3017 goto remove;
3018
a4aea562
MB		 3019 nvme_set_irq_hints(dev);
3020
d4b4ff8e 3021 dev->initialized = 1;
b60503ba
MW		 3022 return 0;
3023
5e82e952
KB		 3024 remove:
3025 nvme_dev_remove(dev);
a4aea562 3026 nvme_dev_remove_admin(dev);
9ac27090 3027 nvme_free_namespaces(dev);
f0b50732
KB		 3028 shutdown:
3029 nvme_dev_shutdown(dev);
0877cb0d 3030 release_pools:
a1a5ef99 3031 nvme_free_queues(dev, 0);
091b6092 3032 nvme_release_prp_pools(dev);
0877cb0d
KB		 3033 release:
3034 nvme_release_instance(dev);
a96d4f5c
KB		 3035 put_pci:
3036 pci_dev_put(dev->pci_dev);
b60503ba
MW		 3037 free:
3038 kfree(dev->queues);
3039 kfree(dev->entry);
3040 kfree(dev);
3041 return result;
3042}
3043
f0d54a54
KB		 3044static void nvme_reset_notify(struct pci_dev *pdev, bool prepare)
3045{
a6739479 3046 struct nvme_dev *dev = pci_get_drvdata(pdev);
f0d54a54 3047
a6739479
KB		 3048 if (prepare)
3049 nvme_dev_shutdown(dev);
3050 else
3051 nvme_dev_resume(dev);
f0d54a54
KB		 3052}
3053
09ece142
KB		 3054static void nvme_shutdown(struct pci_dev *pdev)
3055{
3056 struct nvme_dev *dev = pci_get_drvdata(pdev);
3057 nvme_dev_shutdown(dev);
3058}
3059
8d85fce7 3060static void nvme_remove(struct pci_dev *pdev)
b60503ba
MW		 3061{
3062 struct nvme_dev *dev = pci_get_drvdata(pdev);
9a6b9458
KB		 3063
3064 spin_lock(&dev_list_lock);
3065 list_del_init(&dev->node);
3066 spin_unlock(&dev_list_lock);
3067
3068 pci_set_drvdata(pdev, NULL);
3069 flush_work(&dev->reset_work);
5e82e952 3070 misc_deregister(&dev->miscdev);
9a6b9458 3071 nvme_dev_shutdown(dev);
c9d3bf88 3072 nvme_dev_remove(dev);
a4aea562 3073 nvme_dev_remove_admin(dev);
a1a5ef99 3074 nvme_free_queues(dev, 0);
9a6b9458 3075 nvme_release_prp_pools(dev);
5e82e952 3076 kref_put(&dev->kref, nvme_free_dev);
b60503ba
MW		 3077}
3078
3079/* These functions are yet to be implemented */
3080#define nvme_error_detected NULL
3081#define nvme_dump_registers NULL
3082#define nvme_link_reset NULL
3083#define nvme_slot_reset NULL
3084#define nvme_error_resume NULL
cd638946 3085
671a6018 3086#ifdef CONFIG_PM_SLEEP
cd638946
KB		 3087static int nvme_suspend(struct device *dev)
3088{
3089 struct pci_dev *pdev = to_pci_dev(dev);
3090 struct nvme_dev *ndev = pci_get_drvdata(pdev);
3091
3092 nvme_dev_shutdown(ndev);
3093 return 0;
3094}
3095
3096static int nvme_resume(struct device *dev)
3097{
3098 struct pci_dev *pdev = to_pci_dev(dev);
3099 struct nvme_dev *ndev = pci_get_drvdata(pdev);
cd638946 3100
9a6b9458 3101 if (nvme_dev_resume(ndev) && !work_busy(&ndev->reset_work)) {
9ca97374 3102 ndev->reset_workfn = nvme_reset_failed_dev;
9a6b9458
KB		 3103 queue_work(nvme_workq, &ndev->reset_work);
3104 }
3105 return 0;
cd638946 3106}
671a6018 3107#endif
cd638946
KB		 3108
3109static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);
b60503ba 3110
1d352035 3111static const struct pci_error_handlers nvme_err_handler = {
b60503ba
MW		 3112 .error_detected = nvme_error_detected,
3113 .mmio_enabled = nvme_dump_registers,
3114 .link_reset = nvme_link_reset,
3115 .slot_reset = nvme_slot_reset,
3116 .resume = nvme_error_resume,
f0d54a54 3117 .reset_notify = nvme_reset_notify,
b60503ba
MW		 3118};
3119
3120/* Move to pci_ids.h later */
3121#define PCI_CLASS_STORAGE_EXPRESS 0x010802
3122
6eb0d698 3123static const struct pci_device_id nvme_id_table[] = {
b60503ba
MW		 3124 { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
3125 { 0, }
3126};
3127MODULE_DEVICE_TABLE(pci, nvme_id_table);
3128
3129static struct pci_driver nvme_driver = {
3130 .name = "nvme",
3131 .id_table = nvme_id_table,
3132 .probe = nvme_probe,
8d85fce7 3133 .remove = nvme_remove,
09ece142 3134 .shutdown = nvme_shutdown,
cd638946
KB		 3135 .driver = {
3136 .pm = &nvme_dev_pm_ops,
3137 },
b60503ba
MW		 3138 .err_handler = &nvme_err_handler,
3139};
3140
3141static int __init nvme_init(void)
3142{
0ac13140 3143 int result;
1fa6aead 3144
b9afca3e 3145 init_waitqueue_head(&nvme_kthread_wait);
b60503ba 3146
9a6b9458
KB		 3147 nvme_workq = create_singlethread_workqueue("nvme");
3148 if (!nvme_workq)
b9afca3e 3149 return -ENOMEM;
9a6b9458 3150
5c42ea16
KB		 3151 result = register_blkdev(nvme_major, "nvme");
3152 if (result < 0)
9a6b9458 3153 goto kill_workq;
5c42ea16 3154 else if (result > 0)
0ac13140 3155 nvme_major = result;
b60503ba 3156
f3db22fe
KB		 3157 result = pci_register_driver(&nvme_driver);
3158 if (result)
a4aea562 3159 goto unregister_blkdev;
1fa6aead 3160 return 0;
b60503ba 3161
1fa6aead 3162 unregister_blkdev:
b60503ba 3163 unregister_blkdev(nvme_major, "nvme");
9a6b9458
KB		 3164 kill_workq:
3165 destroy_workqueue(nvme_workq);
b60503ba
MW		 3166 return result;
3167}
3168
3169static void __exit nvme_exit(void)
3170{
3171 pci_unregister_driver(&nvme_driver);
f3db22fe 3172 unregister_hotcpu_notifier(&nvme_nb);
b60503ba 3173 unregister_blkdev(nvme_major, "nvme");
9a6b9458 3174 destroy_workqueue(nvme_workq);
b9afca3e 3175 BUG_ON(nvme_thread && !IS_ERR(nvme_thread));
21bd78bc 3176 _nvme_check_size();
b60503ba
MW		 3177}
3178
3179MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
3180MODULE_LICENSE("GPL");
c78b4713 3181MODULE_VERSION("1.0");
b60503ba
MW		 3182module_init(nvme_init);
3183module_exit(nvme_exit);
Linux kernel - Deliverables
This page took 0.444529 seconds and 5 git commands to generate.