2 * CXL Flash Device Driver
4 * Written by: Manoj N. Kumar <manoj@linux.vnet.ibm.com>, IBM Corporation
5 * Matthew R. Ochs <mrochs@linux.vnet.ibm.com>, IBM Corporation
7 * Copyright (C) 2015 IBM Corporation
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License
11 * as published by the Free Software Foundation; either version
12 * 2 of the License, or (at your option) any later version.
15 #include <linux/delay.h>
16 #include <linux/file.h>
17 #include <linux/syscalls.h>
19 #include <asm/unaligned.h>
21 #include <scsi/scsi.h>
22 #include <scsi/scsi_host.h>
23 #include <scsi/scsi_cmnd.h>
24 #include <scsi/scsi_eh.h>
25 #include <uapi/scsi/cxlflash_ioctl.h>
29 #include "superpipe.h"
31 struct cxlflash_global global
;
34 * marshal_det_to_rele() - translate detach to release structure
35 * @detach: Destination structure for the translate/copy.
36 * @rele: Source structure from which to translate/copy.
38 static void marshal_det_to_rele(struct dk_cxlflash_detach
*detach
,
39 struct dk_cxlflash_release
*release
)
41 release
->hdr
= detach
->hdr
;
42 release
->context_id
= detach
->context_id
;
46 * cxlflash_free_errpage() - frees resources associated with global error page
48 void cxlflash_free_errpage(void)
51 mutex_lock(&global
.mutex
);
52 if (global
.err_page
) {
53 __free_page(global
.err_page
);
54 global
.err_page
= NULL
;
56 mutex_unlock(&global
.mutex
);
60 * cxlflash_stop_term_user_contexts() - stops/terminates known user contexts
61 * @cfg: Internal structure associated with the host.
63 * When the host needs to go down, all users must be quiesced and their
64 * memory freed. This is accomplished by putting the contexts in error
65 * state which will notify the user and let them 'drive' the tear-down.
66 * Meanwhile, this routine camps until all user contexts have been removed.
68 void cxlflash_stop_term_user_contexts(struct cxlflash_cfg
*cfg
)
70 struct device
*dev
= &cfg
->dev
->dev
;
73 cxlflash_mark_contexts_error(cfg
);
78 for (i
= 0; i
< MAX_CONTEXT
; i
++)
79 if (cfg
->ctx_tbl
[i
]) {
84 if (!found
&& list_empty(&cfg
->ctx_err_recovery
))
87 dev_dbg(dev
, "%s: Wait for user contexts to quiesce...\n",
89 wake_up_all(&cfg
->limbo_waitq
);
95 * find_error_context() - locates a context by cookie on the error recovery list
96 * @cfg: Internal structure associated with the host.
97 * @rctxid: Desired context by id.
98 * @file: Desired context by file.
100 * Return: Found context on success, NULL on failure
102 static struct ctx_info
*find_error_context(struct cxlflash_cfg
*cfg
, u64 rctxid
,
105 struct ctx_info
*ctxi
;
107 list_for_each_entry(ctxi
, &cfg
->ctx_err_recovery
, list
)
108 if ((ctxi
->ctxid
== rctxid
) || (ctxi
->file
== file
))
115 * get_context() - obtains a validated and locked context reference
116 * @cfg: Internal structure associated with the host.
117 * @rctxid: Desired context (raw, un-decoded format).
118 * @arg: LUN information or file associated with request.
119 * @ctx_ctrl: Control information to 'steer' desired lookup.
121 * NOTE: despite the name pid, in linux, current->pid actually refers
122 * to the lightweight process id (tid) and can change if the process is
123 * multi threaded. The tgid remains constant for the process and only changes
124 * when the process of fork. For all intents and purposes, think of tgid
125 * as a pid in the traditional sense.
127 * Return: Validated context on success, NULL on failure
129 struct ctx_info
*get_context(struct cxlflash_cfg
*cfg
, u64 rctxid
,
130 void *arg
, enum ctx_ctrl ctx_ctrl
)
132 struct device
*dev
= &cfg
->dev
->dev
;
133 struct ctx_info
*ctxi
= NULL
;
134 struct lun_access
*lun_access
= NULL
;
135 struct file
*file
= NULL
;
136 struct llun_info
*lli
= arg
;
137 u64 ctxid
= DECODE_CTXID(rctxid
);
139 pid_t pid
= current
->tgid
, ctxpid
= 0;
141 if (ctx_ctrl
& CTX_CTRL_FILE
) {
143 file
= (struct file
*)arg
;
146 if (ctx_ctrl
& CTX_CTRL_CLONE
)
147 pid
= current
->parent
->tgid
;
149 if (likely(ctxid
< MAX_CONTEXT
)) {
151 rc
= mutex_lock_interruptible(&cfg
->ctx_tbl_list_mutex
);
155 ctxi
= cfg
->ctx_tbl
[ctxid
];
157 if ((file
&& (ctxi
->file
!= file
)) ||
158 (!file
&& (ctxi
->ctxid
!= rctxid
)))
161 if ((ctx_ctrl
& CTX_CTRL_ERR
) ||
162 (!ctxi
&& (ctx_ctrl
& CTX_CTRL_ERR_FALLBACK
)))
163 ctxi
= find_error_context(cfg
, rctxid
, file
);
165 mutex_unlock(&cfg
->ctx_tbl_list_mutex
);
170 * Need to acquire ownership of the context while still
171 * under the table/list lock to serialize with a remove
172 * thread. Use the 'try' to avoid stalling the
173 * table/list lock for a single context.
175 * Note that the lock order is:
177 * cfg->ctx_tbl_list_mutex -> ctxi->mutex
179 * Therefore release ctx_tbl_list_mutex before retrying.
181 rc
= mutex_trylock(&ctxi
->mutex
);
182 mutex_unlock(&cfg
->ctx_tbl_list_mutex
);
184 break; /* got the context's lock! */
191 if (likely(!(ctx_ctrl
& CTX_CTRL_NOPID
)))
196 list_for_each_entry(lun_access
, &ctxi
->luns
, list
)
197 if (lun_access
->lli
== lli
)
204 dev_dbg(dev
, "%s: rctxid=%016llX ctxinfo=%p ctxpid=%u pid=%u "
205 "ctx_ctrl=%u\n", __func__
, rctxid
, ctxi
, ctxpid
, pid
,
211 mutex_unlock(&ctxi
->mutex
);
217 * put_context() - release a context that was retrieved from get_context()
218 * @ctxi: Context to release.
220 * For now, releasing the context equates to unlocking it's mutex.
222 void put_context(struct ctx_info
*ctxi
)
224 mutex_unlock(&ctxi
->mutex
);
228 * afu_attach() - attach a context to the AFU
229 * @cfg: Internal structure associated with the host.
230 * @ctxi: Context to attach.
232 * Upon setting the context capabilities, they must be confirmed with
233 * a read back operation as the context might have been closed since
234 * the mailbox was unlocked. When this occurs, registration is failed.
236 * Return: 0 on success, -errno on failure
238 static int afu_attach(struct cxlflash_cfg
*cfg
, struct ctx_info
*ctxi
)
240 struct device
*dev
= &cfg
->dev
->dev
;
241 struct afu
*afu
= cfg
->afu
;
242 struct sisl_ctrl_map
*ctrl_map
= ctxi
->ctrl_map
;
246 /* Unlock cap and restrict user to read/write cmds in translated mode */
247 readq_be(&ctrl_map
->mbox_r
);
248 val
= (SISL_CTX_CAP_READ_CMD
| SISL_CTX_CAP_WRITE_CMD
);
249 writeq_be(val
, &ctrl_map
->ctx_cap
);
250 val
= readq_be(&ctrl_map
->ctx_cap
);
251 if (val
!= (SISL_CTX_CAP_READ_CMD
| SISL_CTX_CAP_WRITE_CMD
)) {
252 dev_err(dev
, "%s: ctx may be closed val=%016llX\n",
258 /* Set up MMIO registers pointing to the RHT */
259 writeq_be((u64
)ctxi
->rht_start
, &ctrl_map
->rht_start
);
260 val
= SISL_RHT_CNT_ID((u64
)MAX_RHT_PER_CONTEXT
, (u64
)(afu
->ctx_hndl
));
261 writeq_be(val
, &ctrl_map
->rht_cnt_id
);
263 dev_dbg(dev
, "%s: returning rc=%d\n", __func__
, rc
);
268 * read_cap16() - issues a SCSI READ_CAP16 command
269 * @sdev: SCSI device associated with LUN.
270 * @lli: LUN destined for capacity request.
272 * Return: 0 on success, -errno on failure
274 static int read_cap16(struct scsi_device
*sdev
, struct llun_info
*lli
)
276 struct cxlflash_cfg
*cfg
= (struct cxlflash_cfg
*)sdev
->host
->hostdata
;
277 struct device
*dev
= &cfg
->dev
->dev
;
278 struct glun_info
*gli
= lli
->parent
;
281 u8
*sense_buf
= NULL
;
285 u32 tout
= (MC_DISCOVERY_TIMEOUT
* HZ
);
288 cmd_buf
= kzalloc(CMD_BUFSIZE
, GFP_KERNEL
);
289 scsi_cmd
= kzalloc(MAX_COMMAND_SIZE
, GFP_KERNEL
);
290 sense_buf
= kzalloc(SCSI_SENSE_BUFFERSIZE
, GFP_KERNEL
);
291 if (unlikely(!cmd_buf
|| !scsi_cmd
|| !sense_buf
)) {
296 scsi_cmd
[0] = SERVICE_ACTION_IN_16
; /* read cap(16) */
297 scsi_cmd
[1] = SAI_READ_CAPACITY_16
; /* service action */
298 put_unaligned_be32(CMD_BUFSIZE
, &scsi_cmd
[10]);
300 dev_dbg(dev
, "%s: %ssending cmd(0x%x)\n", __func__
,
301 retry_cnt
? "re" : "", scsi_cmd
[0]);
303 result
= scsi_execute(sdev
, scsi_cmd
, DMA_FROM_DEVICE
, cmd_buf
,
304 CMD_BUFSIZE
, sense_buf
, tout
, 5, 0, NULL
);
306 if (driver_byte(result
) == DRIVER_SENSE
) {
307 result
&= ~(0xFF<<24); /* DRIVER_SENSE is not an error */
308 if (result
& SAM_STAT_CHECK_CONDITION
) {
309 struct scsi_sense_hdr sshdr
;
311 scsi_normalize_sense(sense_buf
, SCSI_SENSE_BUFFERSIZE
,
313 switch (sshdr
.sense_key
) {
315 case RECOVERED_ERROR
:
318 result
&= ~SAM_STAT_CHECK_CONDITION
;
322 case 0x29: /* Power on Reset or Device Reset */
324 case 0x2A: /* Device capacity changed */
325 case 0x3F: /* Report LUNs changed */
326 /* Retry the command once more */
327 if (retry_cnt
++ < 1) {
342 dev_err(dev
, "%s: command failed, result=0x%x\n",
349 * Read cap was successful, grab values from the buffer;
350 * note that we don't need to worry about unaligned access
351 * as the buffer is allocated on an aligned boundary.
353 mutex_lock(&gli
->mutex
);
354 gli
->max_lba
= be64_to_cpu(*((u64
*)&cmd_buf
[0]));
355 gli
->blk_len
= be32_to_cpu(*((u32
*)&cmd_buf
[8]));
356 mutex_unlock(&gli
->mutex
);
363 dev_dbg(dev
, "%s: maxlba=%lld blklen=%d rc=%d\n",
364 __func__
, gli
->max_lba
, gli
->blk_len
, rc
);
369 * get_rhte() - obtains validated resource handle table entry reference
370 * @ctxi: Context owning the resource handle.
371 * @rhndl: Resource handle associated with entry.
372 * @lli: LUN associated with request.
374 * Return: Validated RHTE on success, NULL on failure
376 struct sisl_rht_entry
*get_rhte(struct ctx_info
*ctxi
, res_hndl_t rhndl
,
377 struct llun_info
*lli
)
379 struct sisl_rht_entry
*rhte
= NULL
;
381 if (unlikely(!ctxi
->rht_start
)) {
382 pr_debug("%s: Context does not have allocated RHT!\n",
387 if (unlikely(rhndl
>= MAX_RHT_PER_CONTEXT
)) {
388 pr_debug("%s: Bad resource handle! (%d)\n", __func__
, rhndl
);
392 if (unlikely(ctxi
->rht_lun
[rhndl
] != lli
)) {
393 pr_debug("%s: Bad resource handle LUN! (%d)\n",
398 rhte
= &ctxi
->rht_start
[rhndl
];
399 if (unlikely(rhte
->nmask
== 0)) {
400 pr_debug("%s: Unopened resource handle! (%d)\n",
411 * rhte_checkout() - obtains free/empty resource handle table entry
412 * @ctxi: Context owning the resource handle.
413 * @lli: LUN associated with request.
415 * Return: Free RHTE on success, NULL on failure
417 struct sisl_rht_entry
*rhte_checkout(struct ctx_info
*ctxi
,
418 struct llun_info
*lli
)
420 struct sisl_rht_entry
*rhte
= NULL
;
423 /* Find a free RHT entry */
424 for (i
= 0; i
< MAX_RHT_PER_CONTEXT
; i
++)
425 if (ctxi
->rht_start
[i
].nmask
== 0) {
426 rhte
= &ctxi
->rht_start
[i
];
432 ctxi
->rht_lun
[i
] = lli
;
434 pr_debug("%s: returning rhte=%p (%d)\n", __func__
, rhte
, i
);
439 * rhte_checkin() - releases a resource handle table entry
440 * @ctxi: Context owning the resource handle.
441 * @rhte: RHTE to release.
443 void rhte_checkin(struct ctx_info
*ctxi
,
444 struct sisl_rht_entry
*rhte
)
446 u32 rsrc_handle
= rhte
- ctxi
->rht_start
;
451 ctxi
->rht_lun
[rsrc_handle
] = NULL
;
455 * rhte_format1() - populates a RHTE for format 1
456 * @rhte: RHTE to populate.
457 * @lun_id: LUN ID of LUN associated with RHTE.
458 * @perm: Desired permissions for RHTE.
459 * @port_sel: Port selection mask
461 static void rht_format1(struct sisl_rht_entry
*rhte
, u64 lun_id
, u32 perm
,
465 * Populate the Format 1 RHT entry for direct access (physical
466 * LUN) using the synchronization sequence defined in the
467 * SISLite specification.
469 struct sisl_rht_entry_f1 dummy
= { 0 };
470 struct sisl_rht_entry_f1
*rhte_f1
= (struct sisl_rht_entry_f1
*)rhte
;
472 memset(rhte_f1
, 0, sizeof(*rhte_f1
));
473 rhte_f1
->fp
= SISL_RHT_FP(1U, 0);
474 dma_wmb(); /* Make setting of format bit visible */
476 rhte_f1
->lun_id
= lun_id
;
477 dma_wmb(); /* Make setting of LUN id visible */
480 * Use a dummy RHT Format 1 entry to build the second dword
481 * of the entry that must be populated in a single write when
482 * enabled (valid bit set to TRUE).
485 dummy
.fp
= SISL_RHT_FP(1U, perm
);
486 dummy
.port_sel
= port_sel
;
487 rhte_f1
->dw
= dummy
.dw
;
489 dma_wmb(); /* Make remaining RHT entry fields visible */
493 * cxlflash_lun_attach() - attaches a user to a LUN and manages the LUN's mode
494 * @gli: LUN to attach.
495 * @mode: Desired mode of the LUN.
496 * @locked: Mutex status on current thread.
498 * Return: 0 on success, -errno on failure
500 int cxlflash_lun_attach(struct glun_info
*gli
, enum lun_mode mode
, bool locked
)
505 mutex_lock(&gli
->mutex
);
507 if (gli
->mode
== MODE_NONE
)
509 else if (gli
->mode
!= mode
) {
510 pr_debug("%s: LUN operating in mode %d, requested mode %d\n",
511 __func__
, gli
->mode
, mode
);
517 WARN_ON(gli
->users
<= 0);
519 pr_debug("%s: Returning rc=%d gli->mode=%u gli->users=%u\n",
520 __func__
, rc
, gli
->mode
, gli
->users
);
522 mutex_unlock(&gli
->mutex
);
527 * cxlflash_lun_detach() - detaches a user from a LUN and resets the LUN's mode
528 * @gli: LUN to detach.
530 void cxlflash_lun_detach(struct glun_info
*gli
)
532 mutex_lock(&gli
->mutex
);
533 WARN_ON(gli
->mode
== MODE_NONE
);
534 if (--gli
->users
== 0)
535 gli
->mode
= MODE_NONE
;
536 pr_debug("%s: gli->users=%u\n", __func__
, gli
->users
);
537 WARN_ON(gli
->users
< 0);
538 mutex_unlock(&gli
->mutex
);
542 * _cxlflash_disk_release() - releases the specified resource entry
543 * @sdev: SCSI device associated with LUN.
544 * @ctxi: Context owning resources.
545 * @release: Release ioctl data structure.
547 * Note that the AFU sync should _not_ be performed when the context is sitting
548 * on the error recovery list. A context on the error recovery list is not known
549 * to the AFU due to reset. When the context is recovered, it will be reattached
550 * and made known again to the AFU.
552 * Return: 0 on success, -errno on failure
554 int _cxlflash_disk_release(struct scsi_device
*sdev
,
555 struct ctx_info
*ctxi
,
556 struct dk_cxlflash_release
*release
)
558 struct cxlflash_cfg
*cfg
= (struct cxlflash_cfg
*)sdev
->host
->hostdata
;
559 struct device
*dev
= &cfg
->dev
->dev
;
560 struct llun_info
*lli
= sdev
->hostdata
;
561 struct glun_info
*gli
= lli
->parent
;
562 struct afu
*afu
= cfg
->afu
;
563 bool put_ctx
= false;
565 res_hndl_t rhndl
= release
->rsrc_handle
;
568 u64 ctxid
= DECODE_CTXID(release
->context_id
),
569 rctxid
= release
->context_id
;
571 struct sisl_rht_entry
*rhte
;
572 struct sisl_rht_entry_f1
*rhte_f1
;
574 dev_dbg(dev
, "%s: ctxid=%llu rhndl=0x%llx gli->mode=%u gli->users=%u\n",
575 __func__
, ctxid
, release
->rsrc_handle
, gli
->mode
, gli
->users
);
578 ctxi
= get_context(cfg
, rctxid
, lli
, CTX_CTRL_ERR_FALLBACK
);
579 if (unlikely(!ctxi
)) {
580 dev_dbg(dev
, "%s: Bad context! (%llu)\n",
589 rhte
= get_rhte(ctxi
, rhndl
, lli
);
590 if (unlikely(!rhte
)) {
591 dev_dbg(dev
, "%s: Bad resource handle! (%d)\n",
600 * Clear the Format 1 RHT entry for direct access
601 * (physical LUN) using the synchronization sequence
602 * defined in the SISLite specification.
604 rhte_f1
= (struct sisl_rht_entry_f1
*)rhte
;
607 dma_wmb(); /* Make revocation of RHT entry visible */
610 dma_wmb(); /* Make clearing of LUN id visible */
613 dma_wmb(); /* Make RHT entry bottom-half clearing visible */
615 if (!ctxi
->err_recovery_active
)
616 cxlflash_afu_sync(afu
, ctxid
, rhndl
, AFU_HW_SYNC
);
619 WARN(1, "Unsupported LUN mode!");
623 rhte_checkin(ctxi
, rhte
);
624 cxlflash_lun_detach(gli
);
629 dev_dbg(dev
, "%s: returning rc=%d\n", __func__
, rc
);
633 int cxlflash_disk_release(struct scsi_device
*sdev
,
634 struct dk_cxlflash_release
*release
)
636 return _cxlflash_disk_release(sdev
, NULL
, release
);
640 * destroy_context() - releases a context
641 * @cfg: Internal structure associated with the host.
642 * @ctxi: Context to release.
644 * Note that the rht_lun member of the context was cut from a single
645 * allocation when the context was created and therefore does not need
646 * to be explicitly freed. Also note that we conditionally check for the
647 * existence of the context control map before clearing the RHT registers
648 * and context capabilities because it is possible to destroy a context
649 * while the context is in the error state (previous mapping was removed
650 * [so we don't have to worry about clearing] and context is waiting for
653 static void destroy_context(struct cxlflash_cfg
*cfg
,
654 struct ctx_info
*ctxi
)
656 struct afu
*afu
= cfg
->afu
;
658 WARN_ON(!list_empty(&ctxi
->luns
));
660 /* Clear RHT registers and drop all capabilities for this context */
661 if (afu
->afu_map
&& ctxi
->ctrl_map
) {
662 writeq_be(0, &ctxi
->ctrl_map
->rht_start
);
663 writeq_be(0, &ctxi
->ctrl_map
->rht_cnt_id
);
664 writeq_be(0, &ctxi
->ctrl_map
->ctx_cap
);
667 /* Free memory associated with context */
668 free_page((ulong
)ctxi
->rht_start
);
669 kfree(ctxi
->rht_lun
);
671 atomic_dec_if_positive(&cfg
->num_user_contexts
);
675 * create_context() - allocates and initializes a context
676 * @cfg: Internal structure associated with the host.
677 * @ctx: Previously obtained CXL context reference.
678 * @ctxid: Previously obtained process element associated with CXL context.
679 * @adap_fd: Previously obtained adapter fd associated with CXL context.
680 * @file: Previously obtained file associated with CXL context.
681 * @perms: User-specified permissions.
683 * The context's mutex is locked when an allocated context is returned.
685 * Return: Allocated context on success, NULL on failure
687 static struct ctx_info
*create_context(struct cxlflash_cfg
*cfg
,
688 struct cxl_context
*ctx
, int ctxid
,
689 int adap_fd
, struct file
*file
,
692 struct device
*dev
= &cfg
->dev
->dev
;
693 struct afu
*afu
= cfg
->afu
;
694 struct ctx_info
*ctxi
= NULL
;
695 struct llun_info
**lli
= NULL
;
696 struct sisl_rht_entry
*rhte
;
698 ctxi
= kzalloc(sizeof(*ctxi
), GFP_KERNEL
);
699 lli
= kzalloc((MAX_RHT_PER_CONTEXT
* sizeof(*lli
)), GFP_KERNEL
);
700 if (unlikely(!ctxi
|| !lli
)) {
701 dev_err(dev
, "%s: Unable to allocate context!\n", __func__
);
705 rhte
= (struct sisl_rht_entry
*)get_zeroed_page(GFP_KERNEL
);
706 if (unlikely(!rhte
)) {
707 dev_err(dev
, "%s: Unable to allocate RHT!\n", __func__
);
712 ctxi
->rht_start
= rhte
;
713 ctxi
->rht_perms
= perms
;
715 ctxi
->ctrl_map
= &afu
->afu_map
->ctrls
[ctxid
].ctrl
;
716 ctxi
->ctxid
= ENCODE_CTXID(ctxi
, ctxid
);
718 ctxi
->pid
= current
->tgid
; /* tgid = pid */
721 mutex_init(&ctxi
->mutex
);
722 INIT_LIST_HEAD(&ctxi
->luns
);
723 INIT_LIST_HEAD(&ctxi
->list
); /* initialize for list_empty() */
725 atomic_inc(&cfg
->num_user_contexts
);
726 mutex_lock(&ctxi
->mutex
);
738 * _cxlflash_disk_detach() - detaches a LUN from a context
739 * @sdev: SCSI device associated with LUN.
740 * @ctxi: Context owning resources.
741 * @detach: Detach ioctl data structure.
743 * As part of the detach, all per-context resources associated with the LUN
744 * are cleaned up. When detaching the last LUN for a context, the context
745 * itself is cleaned up and released.
747 * Return: 0 on success, -errno on failure
749 static int _cxlflash_disk_detach(struct scsi_device
*sdev
,
750 struct ctx_info
*ctxi
,
751 struct dk_cxlflash_detach
*detach
)
753 struct cxlflash_cfg
*cfg
= (struct cxlflash_cfg
*)sdev
->host
->hostdata
;
754 struct device
*dev
= &cfg
->dev
->dev
;
755 struct llun_info
*lli
= sdev
->hostdata
;
756 struct lun_access
*lun_access
, *t
;
757 struct dk_cxlflash_release rel
;
758 bool put_ctx
= false;
763 u64 ctxid
= DECODE_CTXID(detach
->context_id
),
764 rctxid
= detach
->context_id
;
766 dev_dbg(dev
, "%s: ctxid=%llu\n", __func__
, ctxid
);
769 ctxi
= get_context(cfg
, rctxid
, lli
, CTX_CTRL_ERR_FALLBACK
);
770 if (unlikely(!ctxi
)) {
771 dev_dbg(dev
, "%s: Bad context! (%llu)\n",
780 /* Cleanup outstanding resources tied to this LUN */
782 marshal_det_to_rele(detach
, &rel
);
783 for (i
= 0; i
< MAX_RHT_PER_CONTEXT
; i
++) {
784 if (ctxi
->rht_lun
[i
] == lli
) {
786 _cxlflash_disk_release(sdev
, ctxi
, &rel
);
789 /* No need to loop further if we're done */
790 if (ctxi
->rht_out
== 0)
795 /* Take our LUN out of context, free the node */
796 list_for_each_entry_safe(lun_access
, t
, &ctxi
->luns
, list
)
797 if (lun_access
->lli
== lli
) {
798 list_del(&lun_access
->list
);
804 /* Tear down context following last LUN cleanup */
805 if (list_empty(&ctxi
->luns
)) {
806 ctxi
->unavail
= true;
807 mutex_unlock(&ctxi
->mutex
);
808 mutex_lock(&cfg
->ctx_tbl_list_mutex
);
809 mutex_lock(&ctxi
->mutex
);
811 /* Might not have been in error list so conditionally remove */
812 if (!list_empty(&ctxi
->list
))
813 list_del(&ctxi
->list
);
814 cfg
->ctx_tbl
[ctxid
] = NULL
;
815 mutex_unlock(&cfg
->ctx_tbl_list_mutex
);
816 mutex_unlock(&ctxi
->mutex
);
819 destroy_context(cfg
, ctxi
);
824 * As a last step, clean up external resources when not
825 * already on an external cleanup thread, i.e.: close(adap_fd).
827 * NOTE: this will free up the context from the CXL services,
828 * allowing it to dole out the same context_id on a future
829 * (or even currently in-flight) disk_attach operation.
838 dev_dbg(dev
, "%s: returning rc=%d\n", __func__
, rc
);
842 static int cxlflash_disk_detach(struct scsi_device
*sdev
,
843 struct dk_cxlflash_detach
*detach
)
845 return _cxlflash_disk_detach(sdev
, NULL
, detach
);
849 * cxlflash_cxl_release() - release handler for adapter file descriptor
850 * @inode: File-system inode associated with fd.
851 * @file: File installed with adapter file descriptor.
853 * This routine is the release handler for the fops registered with
854 * the CXL services on an initial attach for a context. It is called
855 * when a close is performed on the adapter file descriptor returned
856 * to the user. Programmatically, the user is not required to perform
857 * the close, as it is handled internally via the detach ioctl when
858 * a context is being removed. Note that nothing prevents the user
859 * from performing a close, but the user should be aware that doing
860 * so is considered catastrophic and subsequent usage of the superpipe
861 * API with previously saved off tokens will fail.
863 * When initiated from an external close (either by the user or via
864 * a process tear down), the routine derives the context reference
865 * and calls detach for each LUN associated with the context. The
866 * final detach operation will cause the context itself to be freed.
867 * Note that the saved off lfd is reset prior to calling detach to
868 * signify that the final detach should not perform a close.
870 * When initiated from a detach operation as part of the tear down
871 * of a context, the context is first completely freed and then the
872 * close is performed. This routine will fail to derive the context
873 * reference (due to the context having already been freed) and then
874 * call into the CXL release entry point.
876 * Thus, with exception to when the CXL process element (context id)
877 * lookup fails (a case that should theoretically never occur), every
878 * call into this routine results in a complete freeing of a context.
880 * As part of the detach, all per-context resources associated with the LUN
881 * are cleaned up. When detaching the last LUN for a context, the context
882 * itself is cleaned up and released.
884 * Return: 0 on success
886 static int cxlflash_cxl_release(struct inode
*inode
, struct file
*file
)
888 struct cxl_context
*ctx
= cxl_fops_get_context(file
);
889 struct cxlflash_cfg
*cfg
= container_of(file
->f_op
, struct cxlflash_cfg
,
891 struct device
*dev
= &cfg
->dev
->dev
;
892 struct ctx_info
*ctxi
= NULL
;
893 struct dk_cxlflash_detach detach
= { { 0 }, 0 };
894 struct lun_access
*lun_access
, *t
;
895 enum ctx_ctrl ctrl
= CTX_CTRL_ERR_FALLBACK
| CTX_CTRL_FILE
;
898 ctxid
= cxl_process_element(ctx
);
899 if (unlikely(ctxid
< 0)) {
900 dev_err(dev
, "%s: Context %p was closed! (%d)\n",
901 __func__
, ctx
, ctxid
);
905 ctxi
= get_context(cfg
, ctxid
, file
, ctrl
);
906 if (unlikely(!ctxi
)) {
907 ctxi
= get_context(cfg
, ctxid
, file
, ctrl
| CTX_CTRL_CLONE
);
909 dev_dbg(dev
, "%s: Context %d already free!\n",
914 dev_dbg(dev
, "%s: Another process owns context %d!\n",
920 dev_dbg(dev
, "%s: close(%d) for context %d\n",
921 __func__
, ctxi
->lfd
, ctxid
);
923 /* Reset the file descriptor to indicate we're on a close() thread */
925 detach
.context_id
= ctxi
->ctxid
;
926 list_for_each_entry_safe(lun_access
, t
, &ctxi
->luns
, list
)
927 _cxlflash_disk_detach(lun_access
->sdev
, ctxi
, &detach
);
929 cxl_fd_release(inode
, file
);
931 dev_dbg(dev
, "%s: returning\n", __func__
);
936 * unmap_context() - clears a previously established mapping
937 * @ctxi: Context owning the mapping.
939 * This routine is used to switch between the error notification page
940 * (dummy page of all 1's) and the real mapping (established by the CXL
943 static void unmap_context(struct ctx_info
*ctxi
)
945 unmap_mapping_range(ctxi
->file
->f_mapping
, 0, 0, 1);
949 * get_err_page() - obtains and allocates the error notification page
951 * Return: error notification page on success, NULL on failure
953 static struct page
*get_err_page(void)
955 struct page
*err_page
= global
.err_page
;
957 if (unlikely(!err_page
)) {
958 err_page
= alloc_page(GFP_KERNEL
);
959 if (unlikely(!err_page
)) {
960 pr_err("%s: Unable to allocate err_page!\n", __func__
);
964 memset(page_address(err_page
), -1, PAGE_SIZE
);
966 /* Serialize update w/ other threads to avoid a leak */
967 mutex_lock(&global
.mutex
);
968 if (likely(!global
.err_page
))
969 global
.err_page
= err_page
;
971 __free_page(err_page
);
972 err_page
= global
.err_page
;
974 mutex_unlock(&global
.mutex
);
978 pr_debug("%s: returning err_page=%p\n", __func__
, err_page
);
983 * cxlflash_mmap_fault() - mmap fault handler for adapter file descriptor
984 * @vma: VM area associated with mapping.
985 * @vmf: VM fault associated with current fault.
987 * To support error notification via MMIO, faults are 'caught' by this routine
988 * that was inserted before passing back the adapter file descriptor on attach.
989 * When a fault occurs, this routine evaluates if error recovery is active and
990 * if so, installs the error page to 'notify' the user about the error state.
991 * During normal operation, the fault is simply handled by the original fault
992 * handler that was installed by CXL services as part of initializing the
993 * adapter file descriptor. The VMA's page protection bits are toggled to
994 * indicate cached/not-cached depending on the memory backing the fault.
996 * Return: 0 on success, VM_FAULT_SIGBUS on failure
998 static int cxlflash_mmap_fault(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
1000 struct file
*file
= vma
->vm_file
;
1001 struct cxl_context
*ctx
= cxl_fops_get_context(file
);
1002 struct cxlflash_cfg
*cfg
= container_of(file
->f_op
, struct cxlflash_cfg
,
1004 struct device
*dev
= &cfg
->dev
->dev
;
1005 struct ctx_info
*ctxi
= NULL
;
1006 struct page
*err_page
= NULL
;
1007 enum ctx_ctrl ctrl
= CTX_CTRL_ERR_FALLBACK
| CTX_CTRL_FILE
;
1011 ctxid
= cxl_process_element(ctx
);
1012 if (unlikely(ctxid
< 0)) {
1013 dev_err(dev
, "%s: Context %p was closed! (%d)\n",
1014 __func__
, ctx
, ctxid
);
1018 ctxi
= get_context(cfg
, ctxid
, file
, ctrl
);
1019 if (unlikely(!ctxi
)) {
1020 dev_dbg(dev
, "%s: Bad context! (%d)\n", __func__
, ctxid
);
1024 dev_dbg(dev
, "%s: fault(%d) for context %d\n",
1025 __func__
, ctxi
->lfd
, ctxid
);
1027 if (likely(!ctxi
->err_recovery_active
)) {
1028 vma
->vm_page_prot
= pgprot_noncached(vma
->vm_page_prot
);
1029 rc
= ctxi
->cxl_mmap_vmops
->fault(vma
, vmf
);
1031 dev_dbg(dev
, "%s: err recovery active, use err_page!\n",
1034 err_page
= get_err_page();
1035 if (unlikely(!err_page
)) {
1036 dev_err(dev
, "%s: Could not obtain error page!\n",
1038 rc
= VM_FAULT_RETRY
;
1043 vmf
->page
= err_page
;
1044 vma
->vm_page_prot
= pgprot_cached(vma
->vm_page_prot
);
1050 dev_dbg(dev
, "%s: returning rc=%d\n", __func__
, rc
);
1054 rc
= VM_FAULT_SIGBUS
;
1059 * Local MMAP vmops to 'catch' faults
1061 static const struct vm_operations_struct cxlflash_mmap_vmops
= {
1062 .fault
= cxlflash_mmap_fault
,
1066 * cxlflash_cxl_mmap() - mmap handler for adapter file descriptor
1067 * @file: File installed with adapter file descriptor.
1068 * @vma: VM area associated with mapping.
1070 * Installs local mmap vmops to 'catch' faults for error notification support.
1072 * Return: 0 on success, -errno on failure
1074 static int cxlflash_cxl_mmap(struct file
*file
, struct vm_area_struct
*vma
)
1076 struct cxl_context
*ctx
= cxl_fops_get_context(file
);
1077 struct cxlflash_cfg
*cfg
= container_of(file
->f_op
, struct cxlflash_cfg
,
1079 struct device
*dev
= &cfg
->dev
->dev
;
1080 struct ctx_info
*ctxi
= NULL
;
1081 enum ctx_ctrl ctrl
= CTX_CTRL_ERR_FALLBACK
| CTX_CTRL_FILE
;
1085 ctxid
= cxl_process_element(ctx
);
1086 if (unlikely(ctxid
< 0)) {
1087 dev_err(dev
, "%s: Context %p was closed! (%d)\n",
1088 __func__
, ctx
, ctxid
);
1093 ctxi
= get_context(cfg
, ctxid
, file
, ctrl
);
1094 if (unlikely(!ctxi
)) {
1095 dev_dbg(dev
, "%s: Bad context! (%d)\n", __func__
, ctxid
);
1100 dev_dbg(dev
, "%s: mmap(%d) for context %d\n",
1101 __func__
, ctxi
->lfd
, ctxid
);
1103 rc
= cxl_fd_mmap(file
, vma
);
1105 /* Insert ourself in the mmap fault handler path */
1106 ctxi
->cxl_mmap_vmops
= vma
->vm_ops
;
1107 vma
->vm_ops
= &cxlflash_mmap_vmops
;
1117 * Local fops for adapter file descriptor
1119 static const struct file_operations cxlflash_cxl_fops
= {
1120 .owner
= THIS_MODULE
,
1121 .mmap
= cxlflash_cxl_mmap
,
1122 .release
= cxlflash_cxl_release
,
1126 * cxlflash_mark_contexts_error() - move contexts to error state and list
1127 * @cfg: Internal structure associated with the host.
1129 * A context is only moved over to the error list when there are no outstanding
1130 * references to it. This ensures that a running operation has completed.
1132 * Return: 0 on success, -errno on failure
1134 int cxlflash_mark_contexts_error(struct cxlflash_cfg
*cfg
)
1137 struct ctx_info
*ctxi
= NULL
;
1139 mutex_lock(&cfg
->ctx_tbl_list_mutex
);
1141 for (i
= 0; i
< MAX_CONTEXT
; i
++) {
1142 ctxi
= cfg
->ctx_tbl
[i
];
1144 mutex_lock(&ctxi
->mutex
);
1145 cfg
->ctx_tbl
[i
] = NULL
;
1146 list_add(&ctxi
->list
, &cfg
->ctx_err_recovery
);
1147 ctxi
->err_recovery_active
= true;
1148 ctxi
->ctrl_map
= NULL
;
1149 unmap_context(ctxi
);
1150 mutex_unlock(&ctxi
->mutex
);
1154 mutex_unlock(&cfg
->ctx_tbl_list_mutex
);
1161 static const struct file_operations null_fops
= {
1162 .owner
= THIS_MODULE
,
1166 * cxlflash_disk_attach() - attach a LUN to a context
1167 * @sdev: SCSI device associated with LUN.
1168 * @attach: Attach ioctl data structure.
1170 * Creates a context and attaches LUN to it. A LUN can only be attached
1171 * one time to a context (subsequent attaches for the same context/LUN pair
1172 * are not supported). Additional LUNs can be attached to a context by
1173 * specifying the 'reuse' flag defined in the cxlflash_ioctl.h header.
1175 * Return: 0 on success, -errno on failure
1177 static int cxlflash_disk_attach(struct scsi_device
*sdev
,
1178 struct dk_cxlflash_attach
*attach
)
1180 struct cxlflash_cfg
*cfg
= (struct cxlflash_cfg
*)sdev
->host
->hostdata
;
1181 struct device
*dev
= &cfg
->dev
->dev
;
1182 struct afu
*afu
= cfg
->afu
;
1183 struct llun_info
*lli
= sdev
->hostdata
;
1184 struct glun_info
*gli
= lli
->parent
;
1185 struct cxl_ioctl_start_work
*work
;
1186 struct ctx_info
*ctxi
= NULL
;
1187 struct lun_access
*lun_access
= NULL
;
1194 struct cxl_context
*ctx
;
1198 /* On first attach set fileops */
1199 if (atomic_read(&cfg
->num_user_contexts
) == 0)
1200 cfg
->cxl_fops
= cxlflash_cxl_fops
;
1202 if (attach
->num_interrupts
> 4) {
1203 dev_dbg(dev
, "%s: Cannot support this many interrupts %llu\n",
1204 __func__
, attach
->num_interrupts
);
1209 if (gli
->max_lba
== 0) {
1210 dev_dbg(dev
, "%s: No capacity info for this LUN (%016llX)\n",
1211 __func__
, lli
->lun_id
[sdev
->channel
]);
1212 rc
= read_cap16(sdev
, lli
);
1214 dev_err(dev
, "%s: Invalid device! (%d)\n",
1219 dev_dbg(dev
, "%s: LBA = %016llX\n", __func__
, gli
->max_lba
);
1220 dev_dbg(dev
, "%s: BLK_LEN = %08X\n", __func__
, gli
->blk_len
);
1223 if (attach
->hdr
.flags
& DK_CXLFLASH_ATTACH_REUSE_CONTEXT
) {
1224 rctxid
= attach
->context_id
;
1225 ctxi
= get_context(cfg
, rctxid
, NULL
, 0);
1227 dev_dbg(dev
, "%s: Bad context! (%016llX)\n",
1233 list_for_each_entry(lun_access
, &ctxi
->luns
, list
)
1234 if (lun_access
->lli
== lli
) {
1235 dev_dbg(dev
, "%s: Already attached!\n",
1242 lun_access
= kzalloc(sizeof(*lun_access
), GFP_KERNEL
);
1243 if (unlikely(!lun_access
)) {
1244 dev_err(dev
, "%s: Unable to allocate lun_access!\n", __func__
);
1249 lun_access
->lli
= lli
;
1250 lun_access
->sdev
= sdev
;
1252 /* Non-NULL context indicates reuse */
1254 dev_dbg(dev
, "%s: Reusing context for LUN! (%016llX)\n",
1256 list_add(&lun_access
->list
, &ctxi
->luns
);
1261 ctx
= cxl_dev_context_init(cfg
->dev
);
1262 if (unlikely(IS_ERR_OR_NULL(ctx
))) {
1263 dev_err(dev
, "%s: Could not initialize context %p\n",
1269 ctxid
= cxl_process_element(ctx
);
1270 if (unlikely((ctxid
> MAX_CONTEXT
) || (ctxid
< 0))) {
1271 dev_err(dev
, "%s: ctxid (%d) invalid!\n", __func__
, ctxid
);
1276 file
= cxl_get_fd(ctx
, &cfg
->cxl_fops
, &fd
);
1277 if (unlikely(fd
< 0)) {
1279 dev_err(dev
, "%s: Could not get file descriptor\n", __func__
);
1283 /* Translate read/write O_* flags from fcntl.h to AFU permission bits */
1284 perms
= SISL_RHT_PERM(attach
->hdr
.flags
+ 1);
1286 ctxi
= create_context(cfg
, ctx
, ctxid
, fd
, file
, perms
);
1287 if (unlikely(!ctxi
)) {
1288 dev_err(dev
, "%s: Failed to create context! (%d)\n",
1294 work
->num_interrupts
= attach
->num_interrupts
;
1295 work
->flags
= CXL_START_WORK_NUM_IRQS
;
1297 rc
= cxl_start_work(ctx
, work
);
1299 dev_dbg(dev
, "%s: Could not start context rc=%d\n",
1304 rc
= afu_attach(cfg
, ctxi
);
1306 dev_err(dev
, "%s: Could not attach AFU rc %d\n", __func__
, rc
);
1311 * No error paths after this point. Once the fd is installed it's
1312 * visible to user space and can't be undone safely on this thread.
1313 * There is no need to worry about a deadlock here because no one
1314 * knows about us yet; we can be the only one holding our mutex.
1316 list_add(&lun_access
->list
, &ctxi
->luns
);
1317 mutex_unlock(&ctxi
->mutex
);
1318 mutex_lock(&cfg
->ctx_tbl_list_mutex
);
1319 mutex_lock(&ctxi
->mutex
);
1320 cfg
->ctx_tbl
[ctxid
] = ctxi
;
1321 mutex_unlock(&cfg
->ctx_tbl_list_mutex
);
1322 fd_install(fd
, file
);
1325 attach
->hdr
.return_flags
= 0;
1326 attach
->context_id
= ctxi
->ctxid
;
1327 attach
->block_size
= gli
->blk_len
;
1328 attach
->mmio_size
= sizeof(afu
->afu_map
->hosts
[0].harea
);
1329 attach
->last_lba
= gli
->max_lba
;
1330 attach
->max_xfer
= (sdev
->host
->max_sectors
* 512) / gli
->blk_len
;
1333 attach
->adap_fd
= fd
;
1338 dev_dbg(dev
, "%s: returning ctxid=%d fd=%d bs=%lld rc=%d llba=%lld\n",
1339 __func__
, ctxid
, fd
, attach
->block_size
, rc
, attach
->last_lba
);
1343 cxl_stop_context(ctx
);
1346 destroy_context(cfg
, ctxi
);
1350 * Here, we're overriding the fops with a dummy all-NULL fops because
1351 * fput() calls the release fop, which will cause us to mistakenly
1352 * call into the CXL code. Rather than try to add yet more complexity
1353 * to that routine (cxlflash_cxl_release) we should try to fix the
1356 file
->f_op
= &null_fops
;
1361 cxl_release_context(ctx
);
1368 * recover_context() - recovers a context in error
1369 * @cfg: Internal structure associated with the host.
1370 * @ctxi: Context to release.
1372 * Restablishes the state for a context-in-error.
1374 * Return: 0 on success, -errno on failure
1376 static int recover_context(struct cxlflash_cfg
*cfg
, struct ctx_info
*ctxi
)
1378 struct device
*dev
= &cfg
->dev
->dev
;
1380 int old_fd
, fd
= -1;
1383 struct cxl_context
*ctx
;
1384 struct afu
*afu
= cfg
->afu
;
1386 ctx
= cxl_dev_context_init(cfg
->dev
);
1387 if (unlikely(IS_ERR_OR_NULL(ctx
))) {
1388 dev_err(dev
, "%s: Could not initialize context %p\n",
1394 ctxid
= cxl_process_element(ctx
);
1395 if (unlikely((ctxid
> MAX_CONTEXT
) || (ctxid
< 0))) {
1396 dev_err(dev
, "%s: ctxid (%d) invalid!\n", __func__
, ctxid
);
1401 file
= cxl_get_fd(ctx
, &cfg
->cxl_fops
, &fd
);
1402 if (unlikely(fd
< 0)) {
1404 dev_err(dev
, "%s: Could not get file descriptor\n", __func__
);
1408 rc
= cxl_start_work(ctx
, &ctxi
->work
);
1410 dev_dbg(dev
, "%s: Could not start context rc=%d\n",
1415 /* Update with new MMIO area based on updated context id */
1416 ctxi
->ctrl_map
= &afu
->afu_map
->ctrls
[ctxid
].ctrl
;
1418 rc
= afu_attach(cfg
, ctxi
);
1420 dev_err(dev
, "%s: Could not attach AFU rc %d\n", __func__
, rc
);
1425 * No error paths after this point. Once the fd is installed it's
1426 * visible to user space and can't be undone safely on this thread.
1429 ctxi
->ctxid
= ENCODE_CTXID(ctxi
, ctxid
);
1435 * Put context back in table (note the reinit of the context list);
1436 * we must first drop the context's mutex and then acquire it in
1437 * order with the table/list mutex to avoid a deadlock - safe to do
1438 * here because no one can find us at this moment in time.
1440 mutex_unlock(&ctxi
->mutex
);
1441 mutex_lock(&cfg
->ctx_tbl_list_mutex
);
1442 mutex_lock(&ctxi
->mutex
);
1443 list_del_init(&ctxi
->list
);
1444 cfg
->ctx_tbl
[ctxid
] = ctxi
;
1445 mutex_unlock(&cfg
->ctx_tbl_list_mutex
);
1446 fd_install(fd
, file
);
1448 /* Release the original adapter fd and associated CXL resources */
1451 dev_dbg(dev
, "%s: returning ctxid=%d fd=%d rc=%d\n",
1452 __func__
, ctxid
, fd
, rc
);
1456 cxl_stop_context(ctx
);
1461 cxl_release_context(ctx
);
1466 * check_state() - checks and responds to the current adapter state
1467 * @cfg: Internal structure associated with the host.
1469 * This routine can block and should only be used on process context.
1470 * Note that when waking up from waiting in limbo, the state is unknown
1471 * and must be checked again before proceeding.
1473 * Return: 0 on success, -errno on failure
1475 static int check_state(struct cxlflash_cfg
*cfg
)
1477 struct device
*dev
= &cfg
->dev
->dev
;
1481 switch (cfg
->state
) {
1483 dev_dbg(dev
, "%s: Limbo, going to wait...\n", __func__
);
1484 rc
= wait_event_interruptible(cfg
->limbo_waitq
,
1485 cfg
->state
!= STATE_LIMBO
);
1489 case STATE_FAILTERM
:
1490 dev_dbg(dev
, "%s: Failed/Terminating!\n", __func__
);
1501 * cxlflash_afu_recover() - initiates AFU recovery
1502 * @sdev: SCSI device associated with LUN.
1503 * @recover: Recover ioctl data structure.
1505 * Only a single recovery is allowed at a time to avoid exhausting CXL
1506 * resources (leading to recovery failure) in the event that we're up
1507 * against the maximum number of contexts limit. For similar reasons,
1508 * a context recovery is retried if there are multiple recoveries taking
1509 * place at the same time and the failure was due to CXL services being
1510 * unable to keep up.
1512 * Because a user can detect an error condition before the kernel, it is
1513 * quite possible for this routine to act as the kernel's EEH detection
1514 * source (MMIO read of mbox_r). Because of this, there is a window of
1515 * time where an EEH might have been detected but not yet 'serviced'
1516 * (callback invoked, causing the device to enter limbo state). To avoid
1517 * looping in this routine during that window, a 1 second sleep is in place
1518 * between the time the MMIO failure is detected and the time a wait on the
1519 * limbo wait queue is attempted via check_state().
1521 * Return: 0 on success, -errno on failure
1523 static int cxlflash_afu_recover(struct scsi_device
*sdev
,
1524 struct dk_cxlflash_recover_afu
*recover
)
1526 struct cxlflash_cfg
*cfg
= (struct cxlflash_cfg
*)sdev
->host
->hostdata
;
1527 struct device
*dev
= &cfg
->dev
->dev
;
1528 struct llun_info
*lli
= sdev
->hostdata
;
1529 struct afu
*afu
= cfg
->afu
;
1530 struct ctx_info
*ctxi
= NULL
;
1531 struct mutex
*mutex
= &cfg
->ctx_recovery_mutex
;
1532 u64 ctxid
= DECODE_CTXID(recover
->context_id
),
1533 rctxid
= recover
->context_id
;
1535 int lretry
= 20; /* up to 2 seconds */
1538 atomic_inc(&cfg
->recovery_threads
);
1539 rc
= mutex_lock_interruptible(mutex
);
1543 dev_dbg(dev
, "%s: reason 0x%016llX rctxid=%016llX\n",
1544 __func__
, recover
->reason
, rctxid
);
1547 /* Ensure that this process is attached to the context */
1548 ctxi
= get_context(cfg
, rctxid
, lli
, CTX_CTRL_ERR_FALLBACK
);
1549 if (unlikely(!ctxi
)) {
1550 dev_dbg(dev
, "%s: Bad context! (%llu)\n", __func__
, ctxid
);
1555 if (ctxi
->err_recovery_active
) {
1557 rc
= recover_context(cfg
, ctxi
);
1559 dev_err(dev
, "%s: Recovery failed for context %llu (rc=%d)\n",
1560 __func__
, ctxid
, rc
);
1561 if ((rc
== -ENODEV
) &&
1562 ((atomic_read(&cfg
->recovery_threads
) > 1) ||
1564 dev_dbg(dev
, "%s: Going to try again!\n",
1566 mutex_unlock(mutex
);
1568 rc
= mutex_lock_interruptible(mutex
);
1577 ctxi
->err_recovery_active
= false;
1578 recover
->context_id
= ctxi
->ctxid
;
1579 recover
->adap_fd
= ctxi
->lfd
;
1580 recover
->mmio_size
= sizeof(afu
->afu_map
->hosts
[0].harea
);
1581 recover
->hdr
.return_flags
|=
1582 DK_CXLFLASH_RECOVER_AFU_CONTEXT_RESET
;
1586 /* Test if in error state */
1587 reg
= readq_be(&afu
->ctrl_map
->mbox_r
);
1589 dev_dbg(dev
, "%s: MMIO read fail! Wait for recovery...\n",
1591 mutex_unlock(&ctxi
->mutex
);
1594 rc
= check_state(cfg
);
1600 dev_dbg(dev
, "%s: MMIO working, no recovery required!\n", __func__
);
1604 mutex_unlock(mutex
);
1605 atomic_dec_if_positive(&cfg
->recovery_threads
);
1610 * process_sense() - evaluates and processes sense data
1611 * @sdev: SCSI device associated with LUN.
1612 * @verify: Verify ioctl data structure.
1614 * Return: 0 on success, -errno on failure
1616 static int process_sense(struct scsi_device
*sdev
,
1617 struct dk_cxlflash_verify
*verify
)
1619 struct cxlflash_cfg
*cfg
= (struct cxlflash_cfg
*)sdev
->host
->hostdata
;
1620 struct device
*dev
= &cfg
->dev
->dev
;
1621 struct llun_info
*lli
= sdev
->hostdata
;
1622 struct glun_info
*gli
= lli
->parent
;
1623 u64 prev_lba
= gli
->max_lba
;
1624 struct scsi_sense_hdr sshdr
= { 0 };
1627 rc
= scsi_normalize_sense((const u8
*)&verify
->sense_data
,
1628 DK_CXLFLASH_VERIFY_SENSE_LEN
, &sshdr
);
1630 dev_err(dev
, "%s: Failed to normalize sense data!\n", __func__
);
1635 switch (sshdr
.sense_key
) {
1637 case RECOVERED_ERROR
:
1641 case UNIT_ATTENTION
:
1642 switch (sshdr
.asc
) {
1643 case 0x29: /* Power on Reset or Device Reset */
1645 case 0x2A: /* Device settings/capacity changed */
1646 rc
= read_cap16(sdev
, lli
);
1651 if (prev_lba
!= gli
->max_lba
)
1652 dev_dbg(dev
, "%s: Capacity changed old=%lld "
1653 "new=%lld\n", __func__
, prev_lba
,
1656 case 0x3F: /* Report LUNs changed, Rescan. */
1657 scsi_scan_host(cfg
->host
);
1669 dev_dbg(dev
, "%s: sense_key %x asc %x ascq %x rc %d\n", __func__
,
1670 sshdr
.sense_key
, sshdr
.asc
, sshdr
.ascq
, rc
);
1675 * cxlflash_disk_verify() - verifies a LUN is the same and handle size changes
1676 * @sdev: SCSI device associated with LUN.
1677 * @verify: Verify ioctl data structure.
1679 * Return: 0 on success, -errno on failure
1681 static int cxlflash_disk_verify(struct scsi_device
*sdev
,
1682 struct dk_cxlflash_verify
*verify
)
1685 struct ctx_info
*ctxi
= NULL
;
1686 struct cxlflash_cfg
*cfg
= (struct cxlflash_cfg
*)sdev
->host
->hostdata
;
1687 struct device
*dev
= &cfg
->dev
->dev
;
1688 struct llun_info
*lli
= sdev
->hostdata
;
1689 struct glun_info
*gli
= lli
->parent
;
1690 struct sisl_rht_entry
*rhte
= NULL
;
1691 res_hndl_t rhndl
= verify
->rsrc_handle
;
1692 u64 ctxid
= DECODE_CTXID(verify
->context_id
),
1693 rctxid
= verify
->context_id
;
1696 dev_dbg(dev
, "%s: ctxid=%llu rhndl=%016llX, hint=%016llX, "
1697 "flags=%016llX\n", __func__
, ctxid
, verify
->rsrc_handle
,
1698 verify
->hint
, verify
->hdr
.flags
);
1700 ctxi
= get_context(cfg
, rctxid
, lli
, 0);
1701 if (unlikely(!ctxi
)) {
1702 dev_dbg(dev
, "%s: Bad context! (%llu)\n", __func__
, ctxid
);
1707 rhte
= get_rhte(ctxi
, rhndl
, lli
);
1708 if (unlikely(!rhte
)) {
1709 dev_dbg(dev
, "%s: Bad resource handle! (%d)\n",
1716 * Look at the hint/sense to see if it requires us to redrive
1717 * inquiry (i.e. the Unit attention is due to the WWN changing).
1719 if (verify
->hint
& DK_CXLFLASH_VERIFY_HINT_SENSE
) {
1720 rc
= process_sense(sdev
, verify
);
1722 dev_err(dev
, "%s: Failed to validate sense data (%d)\n",
1728 switch (gli
->mode
) {
1730 last_lba
= gli
->max_lba
;
1733 WARN(1, "Unsupported LUN mode!");
1736 verify
->last_lba
= last_lba
;
1741 dev_dbg(dev
, "%s: returning rc=%d llba=%llX\n",
1742 __func__
, rc
, verify
->last_lba
);
1747 * decode_ioctl() - translates an encoded ioctl to an easily identifiable string
1748 * @cmd: The ioctl command to decode.
1750 * Return: A string identifying the decoded ioctl.
1752 static char *decode_ioctl(int cmd
)
1755 case DK_CXLFLASH_ATTACH
:
1756 return __stringify_1(DK_CXLFLASH_ATTACH
);
1757 case DK_CXLFLASH_USER_DIRECT
:
1758 return __stringify_1(DK_CXLFLASH_USER_DIRECT
);
1759 case DK_CXLFLASH_RELEASE
:
1760 return __stringify_1(DK_CXLFLASH_RELEASE
);
1761 case DK_CXLFLASH_DETACH
:
1762 return __stringify_1(DK_CXLFLASH_DETACH
);
1763 case DK_CXLFLASH_VERIFY
:
1764 return __stringify_1(DK_CXLFLASH_VERIFY
);
1765 case DK_CXLFLASH_RECOVER_AFU
:
1766 return __stringify_1(DK_CXLFLASH_RECOVER_AFU
);
1767 case DK_CXLFLASH_MANAGE_LUN
:
1768 return __stringify_1(DK_CXLFLASH_MANAGE_LUN
);
1775 * cxlflash_disk_direct_open() - opens a direct (physical) disk
1776 * @sdev: SCSI device associated with LUN.
1777 * @arg: UDirect ioctl data structure.
1779 * On successful return, the user is informed of the resource handle
1780 * to be used to identify the direct lun and the size (in blocks) of
1781 * the direct lun in last LBA format.
1783 * Return: 0 on success, -errno on failure
1785 static int cxlflash_disk_direct_open(struct scsi_device
*sdev
, void *arg
)
1787 struct cxlflash_cfg
*cfg
= (struct cxlflash_cfg
*)sdev
->host
->hostdata
;
1788 struct device
*dev
= &cfg
->dev
->dev
;
1789 struct afu
*afu
= cfg
->afu
;
1790 struct llun_info
*lli
= sdev
->hostdata
;
1791 struct glun_info
*gli
= lli
->parent
;
1793 struct dk_cxlflash_udirect
*pphys
= (struct dk_cxlflash_udirect
*)arg
;
1795 u64 ctxid
= DECODE_CTXID(pphys
->context_id
),
1796 rctxid
= pphys
->context_id
;
1799 u64 rsrc_handle
= -1;
1800 u32 port
= CHAN2PORT(sdev
->channel
);
1804 struct ctx_info
*ctxi
= NULL
;
1805 struct sisl_rht_entry
*rhte
= NULL
;
1807 pr_debug("%s: ctxid=%llu ls=0x%llx\n", __func__
, ctxid
, lun_size
);
1809 rc
= cxlflash_lun_attach(gli
, MODE_PHYSICAL
, false);
1811 dev_dbg(dev
, "%s: Failed to attach to LUN! (PHYSICAL)\n",
1816 ctxi
= get_context(cfg
, rctxid
, lli
, 0);
1817 if (unlikely(!ctxi
)) {
1818 dev_dbg(dev
, "%s: Bad context! (%llu)\n", __func__
, ctxid
);
1823 rhte
= rhte_checkout(ctxi
, lli
);
1824 if (unlikely(!rhte
)) {
1825 dev_dbg(dev
, "%s: too many opens for this context\n", __func__
);
1826 rc
= -EMFILE
; /* too many opens */
1830 rsrc_handle
= (rhte
- ctxi
->rht_start
);
1832 rht_format1(rhte
, lli
->lun_id
[sdev
->channel
], ctxi
->rht_perms
, port
);
1833 cxlflash_afu_sync(afu
, ctxid
, rsrc_handle
, AFU_LW_SYNC
);
1835 last_lba
= gli
->max_lba
;
1836 pphys
->hdr
.return_flags
= 0;
1837 pphys
->last_lba
= last_lba
;
1838 pphys
->rsrc_handle
= rsrc_handle
;
1843 dev_dbg(dev
, "%s: returning handle 0x%llx rc=%d llba %lld\n",
1844 __func__
, rsrc_handle
, rc
, last_lba
);
1848 cxlflash_lun_detach(gli
);
1853 * ioctl_common() - common IOCTL handler for driver
1854 * @sdev: SCSI device associated with LUN.
1855 * @cmd: IOCTL command.
1857 * Handles common fencing operations that are valid for multiple ioctls. Always
1858 * allow through ioctls that are cleanup oriented in nature, even when operating
1859 * in a failed/terminating state.
1861 * Return: 0 on success, -errno on failure
1863 static int ioctl_common(struct scsi_device
*sdev
, int cmd
)
1865 struct cxlflash_cfg
*cfg
= (struct cxlflash_cfg
*)sdev
->host
->hostdata
;
1866 struct device
*dev
= &cfg
->dev
->dev
;
1867 struct llun_info
*lli
= sdev
->hostdata
;
1870 if (unlikely(!lli
)) {
1871 dev_dbg(dev
, "%s: Unknown LUN\n", __func__
);
1876 rc
= check_state(cfg
);
1877 if (unlikely(rc
) && (cfg
->state
== STATE_FAILTERM
)) {
1879 case DK_CXLFLASH_RELEASE
:
1880 case DK_CXLFLASH_DETACH
:
1881 dev_dbg(dev
, "%s: Command override! (%d)\n",
1892 * cxlflash_ioctl() - IOCTL handler for driver
1893 * @sdev: SCSI device associated with LUN.
1894 * @cmd: IOCTL command.
1895 * @arg: Userspace ioctl data structure.
1897 * Return: 0 on success, -errno on failure
1899 int cxlflash_ioctl(struct scsi_device
*sdev
, int cmd
, void __user
*arg
)
1901 typedef int (*sioctl
) (struct scsi_device
*, void *);
1903 struct cxlflash_cfg
*cfg
= (struct cxlflash_cfg
*)sdev
->host
->hostdata
;
1904 struct device
*dev
= &cfg
->dev
->dev
;
1905 struct afu
*afu
= cfg
->afu
;
1906 struct dk_cxlflash_hdr
*hdr
;
1907 char buf
[sizeof(union cxlflash_ioctls
)];
1909 bool known_ioctl
= false;
1912 struct Scsi_Host
*shost
= sdev
->host
;
1913 sioctl do_ioctl
= NULL
;
1915 static const struct {
1918 } ioctl_tbl
[] = { /* NOTE: order matters here */
1919 {sizeof(struct dk_cxlflash_attach
), (sioctl
)cxlflash_disk_attach
},
1920 {sizeof(struct dk_cxlflash_udirect
), cxlflash_disk_direct_open
},
1921 {sizeof(struct dk_cxlflash_release
), (sioctl
)cxlflash_disk_release
},
1922 {sizeof(struct dk_cxlflash_detach
), (sioctl
)cxlflash_disk_detach
},
1923 {sizeof(struct dk_cxlflash_verify
), (sioctl
)cxlflash_disk_verify
},
1924 {sizeof(struct dk_cxlflash_recover_afu
), (sioctl
)cxlflash_afu_recover
},
1925 {sizeof(struct dk_cxlflash_manage_lun
), (sioctl
)cxlflash_manage_lun
},
1928 /* Restrict command set to physical support only for internal LUN */
1929 if (afu
->internal_lun
)
1931 case DK_CXLFLASH_RELEASE
:
1932 dev_dbg(dev
, "%s: %s not supported for lun_mode=%d\n",
1933 __func__
, decode_ioctl(cmd
), afu
->internal_lun
);
1935 goto cxlflash_ioctl_exit
;
1939 case DK_CXLFLASH_ATTACH
:
1940 case DK_CXLFLASH_USER_DIRECT
:
1941 case DK_CXLFLASH_RELEASE
:
1942 case DK_CXLFLASH_DETACH
:
1943 case DK_CXLFLASH_VERIFY
:
1944 case DK_CXLFLASH_RECOVER_AFU
:
1945 dev_dbg(dev
, "%s: %s (%08X) on dev(%d/%d/%d/%llu)\n",
1946 __func__
, decode_ioctl(cmd
), cmd
, shost
->host_no
,
1947 sdev
->channel
, sdev
->id
, sdev
->lun
);
1948 rc
= ioctl_common(sdev
, cmd
);
1950 goto cxlflash_ioctl_exit
;
1954 case DK_CXLFLASH_MANAGE_LUN
:
1956 idx
= _IOC_NR(cmd
) - _IOC_NR(DK_CXLFLASH_ATTACH
);
1957 size
= ioctl_tbl
[idx
].size
;
1958 do_ioctl
= ioctl_tbl
[idx
].ioctl
;
1960 if (likely(do_ioctl
))
1966 goto cxlflash_ioctl_exit
;
1969 if (unlikely(copy_from_user(&buf
, arg
, size
))) {
1970 dev_err(dev
, "%s: copy_from_user() fail! "
1971 "size=%lu cmd=%d (%s) arg=%p\n",
1972 __func__
, size
, cmd
, decode_ioctl(cmd
), arg
);
1974 goto cxlflash_ioctl_exit
;
1977 hdr
= (struct dk_cxlflash_hdr
*)&buf
;
1978 if (hdr
->version
!= DK_CXLFLASH_VERSION_0
) {
1979 dev_dbg(dev
, "%s: Version %u not supported for %s\n",
1980 __func__
, hdr
->version
, decode_ioctl(cmd
));
1982 goto cxlflash_ioctl_exit
;
1985 if (hdr
->rsvd
[0] || hdr
->rsvd
[1] || hdr
->rsvd
[2] || hdr
->return_flags
) {
1986 dev_dbg(dev
, "%s: Reserved/rflags populated!\n", __func__
);
1988 goto cxlflash_ioctl_exit
;
1991 rc
= do_ioctl(sdev
, (void *)&buf
);
1993 if (unlikely(copy_to_user(arg
, &buf
, size
))) {
1994 dev_err(dev
, "%s: copy_to_user() fail! "
1995 "size=%lu cmd=%d (%s) arg=%p\n",
1996 __func__
, size
, cmd
, decode_ioctl(cmd
), arg
);
2000 /* fall through to exit */
2002 cxlflash_ioctl_exit
:
2003 if (unlikely(rc
&& known_ioctl
))
2004 dev_err(dev
, "%s: ioctl %s (%08X) on dev(%d/%d/%d/%llu) "
2005 "returned rc %d\n", __func__
,
2006 decode_ioctl(cmd
), cmd
, shost
->host_no
,
2007 sdev
->channel
, sdev
->id
, sdev
->lun
, rc
);
2009 dev_dbg(dev
, "%s: ioctl %s (%08X) on dev(%d/%d/%d/%llu) "
2010 "returned rc %d\n", __func__
, decode_ioctl(cmd
),
2011 cmd
, shost
->host_no
, sdev
->channel
, sdev
->id
,