2 * nvme-lightnvm.c - LightNVM NVMe device
4 * Copyright (C) 2014-2015 IT University of Copenhagen
5 * Initial release: Matias Bjorling <mb@lightnvm.io>
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License version
9 * 2 as published by the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; see the file COPYING. If not, write to
18 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
27 #include <linux/nvme.h>
28 #include <linux/bitops.h>
29 #include <linux/lightnvm.h>
30 #include <linux/vmalloc.h>
32 enum nvme_nvm_admin_opcode
{
33 nvme_nvm_admin_identity
= 0xe2,
34 nvme_nvm_admin_get_l2p_tbl
= 0xea,
35 nvme_nvm_admin_get_bb_tbl
= 0xf2,
36 nvme_nvm_admin_set_bb_tbl
= 0xf1,
39 struct nvme_nvm_hb_rw
{
55 struct nvme_nvm_ph_rw
{
71 struct nvme_nvm_identity
{
83 struct nvme_nvm_l2ptbl
{
96 struct nvme_nvm_bbtbl
{
110 struct nvme_nvm_erase_blk
{
125 struct nvme_nvm_command
{
127 struct nvme_common_command common
;
128 struct nvme_nvm_identity identity
;
129 struct nvme_nvm_hb_rw hb_rw
;
130 struct nvme_nvm_ph_rw ph_rw
;
131 struct nvme_nvm_l2ptbl l2p
;
132 struct nvme_nvm_bbtbl get_bb
;
133 struct nvme_nvm_bbtbl set_bb
;
134 struct nvme_nvm_erase_blk erase
;
138 struct nvme_nvm_id_group
{
161 struct nvme_nvm_addr_format
{
184 struct nvme_nvm_addr_format ppaf
;
187 struct nvme_nvm_id_group groups
[4];
191 * Check we didn't inadvertently grow the command struct
193 static inline void _nvme_nvm_check_size(void)
195 BUILD_BUG_ON(sizeof(struct nvme_nvm_identity
) != 64);
196 BUILD_BUG_ON(sizeof(struct nvme_nvm_hb_rw
) != 64);
197 BUILD_BUG_ON(sizeof(struct nvme_nvm_ph_rw
) != 64);
198 BUILD_BUG_ON(sizeof(struct nvme_nvm_bbtbl
) != 64);
199 BUILD_BUG_ON(sizeof(struct nvme_nvm_l2ptbl
) != 64);
200 BUILD_BUG_ON(sizeof(struct nvme_nvm_erase_blk
) != 64);
201 BUILD_BUG_ON(sizeof(struct nvme_nvm_id_group
) != 960);
202 BUILD_BUG_ON(sizeof(struct nvme_nvm_addr_format
) != 128);
203 BUILD_BUG_ON(sizeof(struct nvme_nvm_id
) != 4096);
206 static int init_grps(struct nvm_id
*nvm_id
, struct nvme_nvm_id
*nvme_nvm_id
)
208 struct nvme_nvm_id_group
*src
;
209 struct nvm_id_group
*dst
;
212 end
= min_t(u32
, 4, nvm_id
->cgrps
);
214 for (i
= 0; i
< end
; i
++) {
215 src
= &nvme_nvm_id
->groups
[i
];
216 dst
= &nvm_id
->groups
[i
];
218 dst
->mtype
= src
->mtype
;
219 dst
->fmtype
= src
->fmtype
;
220 dst
->num_ch
= src
->num_ch
;
221 dst
->num_lun
= src
->num_lun
;
222 dst
->num_pln
= src
->num_pln
;
224 dst
->num_pg
= le16_to_cpu(src
->num_pg
);
225 dst
->num_blk
= le16_to_cpu(src
->num_blk
);
226 dst
->fpg_sz
= le16_to_cpu(src
->fpg_sz
);
227 dst
->csecs
= le16_to_cpu(src
->csecs
);
228 dst
->sos
= le16_to_cpu(src
->sos
);
230 dst
->trdt
= le32_to_cpu(src
->trdt
);
231 dst
->trdm
= le32_to_cpu(src
->trdm
);
232 dst
->tprt
= le32_to_cpu(src
->tprt
);
233 dst
->tprm
= le32_to_cpu(src
->tprm
);
234 dst
->tbet
= le32_to_cpu(src
->tbet
);
235 dst
->tbem
= le32_to_cpu(src
->tbem
);
236 dst
->mpos
= le32_to_cpu(src
->mpos
);
238 dst
->cpar
= le16_to_cpu(src
->cpar
);
244 static int nvme_nvm_identity(struct request_queue
*q
, struct nvm_id
*nvm_id
)
246 struct nvme_ns
*ns
= q
->queuedata
;
247 struct nvme_nvm_id
*nvme_nvm_id
;
248 struct nvme_nvm_command c
= {};
251 c
.identity
.opcode
= nvme_nvm_admin_identity
;
252 c
.identity
.nsid
= cpu_to_le32(ns
->ns_id
);
253 c
.identity
.chnl_off
= 0;
255 nvme_nvm_id
= kmalloc(sizeof(struct nvme_nvm_id
), GFP_KERNEL
);
259 ret
= nvme_submit_sync_cmd(q
, (struct nvme_command
*)&c
, nvme_nvm_id
,
260 sizeof(struct nvme_nvm_id
));
266 nvm_id
->ver_id
= nvme_nvm_id
->ver_id
;
267 nvm_id
->vmnt
= nvme_nvm_id
->vmnt
;
268 nvm_id
->cgrps
= nvme_nvm_id
->cgrps
;
269 nvm_id
->cap
= le32_to_cpu(nvme_nvm_id
->cap
);
270 nvm_id
->dom
= le32_to_cpu(nvme_nvm_id
->dom
);
272 ret
= init_grps(nvm_id
, nvme_nvm_id
);
278 static int nvme_nvm_get_l2p_tbl(struct request_queue
*q
, u64 slba
, u32 nlb
,
279 nvm_l2p_update_fn
*update_l2p
, void *priv
)
281 struct nvme_ns
*ns
= q
->queuedata
;
282 struct nvme_dev
*dev
= ns
->dev
;
283 struct nvme_nvm_command c
= {};
284 u32 len
= queue_max_hw_sectors(q
) << 9;
285 u32 nlb_pr_rq
= len
/ sizeof(u64
);
290 c
.l2p
.opcode
= nvme_nvm_admin_get_l2p_tbl
;
291 c
.l2p
.nsid
= cpu_to_le32(ns
->ns_id
);
292 entries
= kmalloc(len
, GFP_KERNEL
);
297 u32 cmd_nlb
= min(nlb_pr_rq
, nlb
);
299 c
.l2p
.slba
= cpu_to_le64(cmd_slba
);
300 c
.l2p
.nlb
= cpu_to_le32(cmd_nlb
);
302 ret
= nvme_submit_sync_cmd(q
, (struct nvme_command
*)&c
,
305 dev_err(dev
->dev
, "L2P table transfer failed (%d)\n",
311 if (update_l2p(cmd_slba
, cmd_nlb
, entries
, priv
)) {
325 static int nvme_nvm_get_bb_tbl(struct request_queue
*q
, int lunid
,
326 unsigned int nr_blocks
,
327 nvm_bb_update_fn
*update_bbtbl
, void *priv
)
329 struct nvme_ns
*ns
= q
->queuedata
;
330 struct nvme_dev
*dev
= ns
->dev
;
331 struct nvme_nvm_command c
= {};
336 c
.get_bb
.opcode
= nvme_nvm_admin_get_bb_tbl
;
337 c
.get_bb
.nsid
= cpu_to_le32(ns
->ns_id
);
338 c
.get_bb
.lbb
= cpu_to_le32(lunid
);
339 bb_bitmap_size
= ((nr_blocks
>> 15) + 1) * PAGE_SIZE
;
340 bb_bitmap
= kmalloc(bb_bitmap_size
, GFP_KERNEL
);
344 bitmap_zero(bb_bitmap
, nr_blocks
);
346 ret
= nvme_submit_sync_cmd(q
, (struct nvme_command
*)&c
, bb_bitmap
,
349 dev_err(dev
->dev
, "get bad block table failed (%d)\n", ret
);
354 ret
= update_bbtbl(lunid
, bb_bitmap
, nr_blocks
, priv
);
365 static inline void nvme_nvm_rqtocmd(struct request
*rq
, struct nvm_rq
*rqd
,
366 struct nvme_ns
*ns
, struct nvme_nvm_command
*c
)
368 c
->ph_rw
.opcode
= rqd
->opcode
;
369 c
->ph_rw
.nsid
= cpu_to_le32(ns
->ns_id
);
370 c
->ph_rw
.spba
= cpu_to_le64(rqd
->ppa_addr
.ppa
);
371 c
->ph_rw
.control
= cpu_to_le16(rqd
->flags
);
372 c
->ph_rw
.length
= cpu_to_le16(rqd
->nr_pages
- 1);
374 if (rqd
->opcode
== NVM_OP_HBWRITE
|| rqd
->opcode
== NVM_OP_HBREAD
)
375 c
->hb_rw
.slba
= cpu_to_le64(nvme_block_nr(ns
,
376 rqd
->bio
->bi_iter
.bi_sector
));
379 static void nvme_nvm_end_io(struct request
*rq
, int error
)
381 struct nvm_rq
*rqd
= rq
->end_io_data
;
382 struct nvm_dev
*dev
= rqd
->dev
;
384 if (dev
->mt
->end_io(rqd
, error
))
385 pr_err("nvme: err status: %x result: %lx\n",
386 rq
->errors
, (unsigned long)rq
->special
);
389 blk_mq_free_request(rq
);
392 static int nvme_nvm_submit_io(struct request_queue
*q
, struct nvm_rq
*rqd
)
394 struct nvme_ns
*ns
= q
->queuedata
;
396 struct bio
*bio
= rqd
->bio
;
397 struct nvme_nvm_command
*cmd
;
399 rq
= blk_mq_alloc_request(q
, bio_rw(bio
), GFP_KERNEL
, 0);
403 cmd
= kzalloc(sizeof(struct nvme_nvm_command
), GFP_KERNEL
);
405 blk_mq_free_request(rq
);
409 rq
->cmd_type
= REQ_TYPE_DRV_PRIV
;
410 rq
->ioprio
= bio_prio(bio
);
412 if (bio_has_data(bio
))
413 rq
->nr_phys_segments
= bio_phys_segments(q
, bio
);
415 rq
->__data_len
= bio
->bi_iter
.bi_size
;
416 rq
->bio
= rq
->biotail
= bio
;
418 nvme_nvm_rqtocmd(rq
, rqd
, ns
, cmd
);
420 rq
->cmd
= (unsigned char *)cmd
;
421 rq
->cmd_len
= sizeof(struct nvme_nvm_command
);
422 rq
->special
= (void *)0;
424 rq
->end_io_data
= rqd
;
426 blk_execute_rq_nowait(q
, NULL
, rq
, 0, nvme_nvm_end_io
);
431 static int nvme_nvm_erase_block(struct request_queue
*q
, struct nvm_rq
*rqd
)
433 struct nvme_ns
*ns
= q
->queuedata
;
434 struct nvme_nvm_command c
= {};
436 c
.erase
.opcode
= NVM_OP_ERASE
;
437 c
.erase
.nsid
= cpu_to_le32(ns
->ns_id
);
438 c
.erase
.spba
= cpu_to_le64(rqd
->ppa_addr
.ppa
);
439 c
.erase
.length
= cpu_to_le16(rqd
->nr_pages
- 1);
441 return nvme_submit_sync_cmd(q
, (struct nvme_command
*)&c
, NULL
, 0);
444 static void *nvme_nvm_create_dma_pool(struct request_queue
*q
, char *name
)
446 struct nvme_ns
*ns
= q
->queuedata
;
447 struct nvme_dev
*dev
= ns
->dev
;
449 return dma_pool_create(name
, dev
->dev
, PAGE_SIZE
, PAGE_SIZE
, 0);
452 static void nvme_nvm_destroy_dma_pool(void *pool
)
454 struct dma_pool
*dma_pool
= pool
;
456 dma_pool_destroy(dma_pool
);
459 static void *nvme_nvm_dev_dma_alloc(struct request_queue
*q
, void *pool
,
460 gfp_t mem_flags
, dma_addr_t
*dma_handler
)
462 return dma_pool_alloc(pool
, mem_flags
, dma_handler
);
465 static void nvme_nvm_dev_dma_free(void *pool
, void *ppa_list
,
466 dma_addr_t dma_handler
)
468 dma_pool_free(pool
, ppa_list
, dma_handler
);
471 static struct nvm_dev_ops nvme_nvm_dev_ops
= {
472 .identity
= nvme_nvm_identity
,
474 .get_l2p_tbl
= nvme_nvm_get_l2p_tbl
,
476 .get_bb_tbl
= nvme_nvm_get_bb_tbl
,
478 .submit_io
= nvme_nvm_submit_io
,
479 .erase_block
= nvme_nvm_erase_block
,
481 .create_dma_pool
= nvme_nvm_create_dma_pool
,
482 .destroy_dma_pool
= nvme_nvm_destroy_dma_pool
,
483 .dev_dma_alloc
= nvme_nvm_dev_dma_alloc
,
484 .dev_dma_free
= nvme_nvm_dev_dma_free
,
489 int nvme_nvm_register(struct request_queue
*q
, char *disk_name
)
491 return nvm_register(q
, disk_name
, &nvme_nvm_dev_ops
);
494 void nvme_nvm_unregister(struct request_queue
*q
, char *disk_name
)
496 nvm_unregister(disk_name
);
499 int nvme_nvm_ns_supported(struct nvme_ns
*ns
, struct nvme_id_ns
*id
)
501 struct nvme_dev
*dev
= ns
->dev
;
502 struct pci_dev
*pdev
= to_pci_dev(dev
->dev
);
504 /* QEMU NVMe simulator - PCI ID + Vendor specific bit */
505 if (pdev
->vendor
== PCI_VENDOR_ID_INTEL
&& pdev
->device
== 0x5845 &&
509 /* CNEX Labs - PCI ID + Vendor specific bit */
510 if (pdev
->vendor
== 0x1d1d && pdev
->device
== 0x2807 &&
517 int nvme_nvm_register(struct request_queue
*q
, char *disk_name
)
521 void nvme_nvm_unregister(struct request_queue
*q
, char *disk_name
) {};
522 int nvme_nvm_ns_supported(struct nvme_ns
*ns
, struct nvme_id_ns
*id
)
526 #endif /* CONFIG_NVM */