2 * drivers/pci/pcie/aer/aerdrv_core.c
4 * This file is subject to the terms and conditions of the GNU General Public
5 * License. See the file "COPYING" in the main directory of this archive
8 * This file implements the core part of PCI-Express AER. When an pci-express
9 * error is delivered, an error message will be collected and printed to
10 * console, then, an error recovery procedure will be executed by following
11 * the pci error recovery rules.
13 * Copyright (C) 2006 Intel Corp.
14 * Tom Long Nguyen (tom.l.nguyen@intel.com)
15 * Zhang Yanmin (yanmin.zhang@intel.com)
19 #include <linux/module.h>
20 #include <linux/pci.h>
21 #include <linux/kernel.h>
22 #include <linux/errno.h>
24 #include <linux/suspend.h>
25 #include <linux/acpi.h>
26 #include <linux/pci-acpi.h>
27 #include <linux/delay.h>
31 module_param(forceload
, bool, 0);
33 #define PCI_CFG_SPACE_SIZE (0x100)
34 int pci_find_aer_capability(struct pci_dev
*dev
)
39 /* Check if it's a pci-express device */
40 pos
= pci_find_capability(dev
, PCI_CAP_ID_EXP
);
44 /* Check if it supports pci-express AER */
45 pos
= PCI_CFG_SPACE_SIZE
;
47 if (pci_read_config_dword(dev
, pos
, ®32
))
50 /* some broken boards return ~0 */
51 if (reg32
== 0xffffffff)
54 if (PCI_EXT_CAP_ID(reg32
) == PCI_EXT_CAP_ID_ERR
)
63 int pci_enable_pcie_error_reporting(struct pci_dev
*dev
)
68 pos
= pci_find_capability(dev
, PCI_CAP_ID_EXP
);
72 pci_read_config_word(dev
, pos
+PCI_EXP_DEVCTL
, ®16
);
75 PCI_EXP_DEVCTL_NFERE
|
78 pci_write_config_word(dev
, pos
+PCI_EXP_DEVCTL
,
83 int pci_disable_pcie_error_reporting(struct pci_dev
*dev
)
88 pos
= pci_find_capability(dev
, PCI_CAP_ID_EXP
);
92 pci_read_config_word(dev
, pos
+PCI_EXP_DEVCTL
, ®16
);
93 reg16
= reg16
& ~(PCI_EXP_DEVCTL_CERE
|
94 PCI_EXP_DEVCTL_NFERE
|
97 pci_write_config_word(dev
, pos
+PCI_EXP_DEVCTL
,
102 int pci_cleanup_aer_uncorrect_error_status(struct pci_dev
*dev
)
107 pos
= pci_find_aer_capability(dev
);
111 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
, &status
);
112 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_SEVER
, &mask
);
113 if (dev
->error_state
== pci_channel_io_normal
)
114 status
&= ~mask
; /* Clear corresponding nonfatal bits */
116 status
&= mask
; /* Clear corresponding fatal bits */
117 pci_write_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
, status
);
122 static int find_device_iter(struct device
*device
, void *data
)
125 u16 id
= *(unsigned long *)data
;
126 u8 secondary
, subordinate
, d_bus
= id
>> 8;
128 if (device
->bus
== &pci_bus_type
) {
129 dev
= to_pci_dev(device
);
130 if (id
== ((dev
->bus
->number
<< 8) | dev
->devfn
)) {
134 *(unsigned long*)data
= (unsigned long)device
;
139 * If device is P2P, check if it is an upstream?
141 if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
) {
142 pci_read_config_byte(dev
, PCI_SECONDARY_BUS
,
144 pci_read_config_byte(dev
, PCI_SUBORDINATE_BUS
,
146 if (d_bus
>= secondary
&& d_bus
<= subordinate
) {
147 *(unsigned long*)data
= (unsigned long)device
;
157 * find_source_device - search through device hierarchy for source device
158 * @p_dev: pointer to Root Port pci_dev data structure
159 * @id: device ID of agent who sends an error message to this Root Port
161 * Invoked when error is detected at the Root Port.
163 static struct device
* find_source_device(struct pci_dev
*parent
, u16 id
)
165 struct pci_dev
*dev
= parent
;
166 struct device
*device
;
167 unsigned long device_addr
;
170 /* Is Root Port an agent that sends error message? */
171 if (id
== ((dev
->bus
->number
<< 8) | dev
->devfn
))
176 if ((status
= device_for_each_child(&dev
->dev
,
177 &device_addr
, find_device_iter
))) {
178 device
= (struct device
*)device_addr
;
179 dev
= to_pci_dev(device
);
180 if (id
== ((dev
->bus
->number
<< 8) | dev
->devfn
))
188 static void report_error_detected(struct pci_dev
*dev
, void *data
)
190 pci_ers_result_t vote
;
191 struct pci_error_handlers
*err_handler
;
192 struct aer_broadcast_data
*result_data
;
193 result_data
= (struct aer_broadcast_data
*) data
;
195 dev
->error_state
= result_data
->state
;
198 !dev
->driver
->err_handler
||
199 !dev
->driver
->err_handler
->error_detected
) {
200 if (result_data
->state
== pci_channel_io_frozen
&&
201 !(dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
)) {
203 * In case of fatal recovery, if one of down-
204 * stream device has no driver. We might be
205 * unable to recover because a later insmod
206 * of a driver for this device is unaware of
209 printk(KERN_DEBUG
"Device ID[%s] has %s\n",
210 dev
->dev
.bus_id
, (dev
->driver
) ?
211 "no AER-aware driver" : "no driver");
216 err_handler
= dev
->driver
->err_handler
;
217 vote
= err_handler
->error_detected(dev
, result_data
->state
);
218 result_data
->result
= merge_result(result_data
->result
, vote
);
222 static void report_mmio_enabled(struct pci_dev
*dev
, void *data
)
224 pci_ers_result_t vote
;
225 struct pci_error_handlers
*err_handler
;
226 struct aer_broadcast_data
*result_data
;
227 result_data
= (struct aer_broadcast_data
*) data
;
230 !dev
->driver
->err_handler
||
231 !dev
->driver
->err_handler
->mmio_enabled
)
234 err_handler
= dev
->driver
->err_handler
;
235 vote
= err_handler
->mmio_enabled(dev
);
236 result_data
->result
= merge_result(result_data
->result
, vote
);
240 static void report_slot_reset(struct pci_dev
*dev
, void *data
)
242 pci_ers_result_t vote
;
243 struct pci_error_handlers
*err_handler
;
244 struct aer_broadcast_data
*result_data
;
245 result_data
= (struct aer_broadcast_data
*) data
;
248 !dev
->driver
->err_handler
||
249 !dev
->driver
->err_handler
->slot_reset
)
252 err_handler
= dev
->driver
->err_handler
;
253 vote
= err_handler
->slot_reset(dev
);
254 result_data
->result
= merge_result(result_data
->result
, vote
);
258 static void report_resume(struct pci_dev
*dev
, void *data
)
260 struct pci_error_handlers
*err_handler
;
262 dev
->error_state
= pci_channel_io_normal
;
265 !dev
->driver
->err_handler
||
266 !dev
->driver
->err_handler
->slot_reset
)
269 err_handler
= dev
->driver
->err_handler
;
270 err_handler
->resume(dev
);
275 * broadcast_error_message - handle message broadcast to downstream drivers
276 * @device: pointer to from where in a hierarchy message is broadcasted down
277 * @api: callback to be broadcasted
278 * @state: error state
280 * Invoked during error recovery process. Once being invoked, the content
281 * of error severity will be broadcasted to all downstream drivers in a
282 * hierarchy in question.
284 static pci_ers_result_t
broadcast_error_message(struct pci_dev
*dev
,
285 enum pci_channel_state state
,
287 void (*cb
)(struct pci_dev
*, void *))
289 struct aer_broadcast_data result_data
;
291 printk(KERN_DEBUG
"Broadcast %s message\n", error_mesg
);
292 result_data
.state
= state
;
293 if (cb
== report_error_detected
)
294 result_data
.result
= PCI_ERS_RESULT_CAN_RECOVER
;
296 result_data
.result
= PCI_ERS_RESULT_RECOVERED
;
298 if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
) {
300 * If the error is reported by a bridge, we think this error
301 * is related to the downstream link of the bridge, so we
302 * do error recovery on all subordinates of the bridge instead
303 * of the bridge and clear the error status of the bridge.
305 if (cb
== report_error_detected
)
306 dev
->error_state
= state
;
307 pci_walk_bus(dev
->subordinate
, cb
, &result_data
);
308 if (cb
== report_resume
) {
309 pci_cleanup_aer_uncorrect_error_status(dev
);
310 dev
->error_state
= pci_channel_io_normal
;
315 * If the error is reported by an end point, we think this
316 * error is related to the upstream link of the end point.
318 pci_walk_bus(dev
->bus
, cb
, &result_data
);
321 return result_data
.result
;
324 struct find_aer_service_data
{
325 struct pcie_port_service_driver
*aer_driver
;
329 static int find_aer_service_iter(struct device
*device
, void *data
)
331 struct device_driver
*driver
;
332 struct pcie_port_service_driver
*service_driver
;
333 struct pcie_device
*pcie_dev
;
334 struct find_aer_service_data
*result
;
336 result
= (struct find_aer_service_data
*) data
;
338 if (device
->bus
== &pcie_port_bus_type
) {
339 pcie_dev
= to_pcie_device(device
);
340 if (pcie_dev
->id
.port_type
== PCIE_SW_DOWNSTREAM_PORT
)
341 result
->is_downstream
= 1;
343 driver
= device
->driver
;
345 service_driver
= to_service_driver(driver
);
346 if (service_driver
->id_table
->service_type
==
347 PCIE_PORT_SERVICE_AER
) {
348 result
->aer_driver
= service_driver
;
357 static void find_aer_service(struct pci_dev
*dev
,
358 struct find_aer_service_data
*data
)
360 device_for_each_child(&dev
->dev
, data
, find_aer_service_iter
);
363 static pci_ers_result_t
reset_link(struct pcie_device
*aerdev
,
366 struct pci_dev
*udev
;
367 pci_ers_result_t status
;
368 struct find_aer_service_data data
;
370 if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
)
373 udev
= dev
->bus
->self
;
375 data
.is_downstream
= 0;
376 data
.aer_driver
= NULL
;
377 find_aer_service(udev
, &data
);
380 * Use the aer driver of the error agent firstly.
381 * If it hasn't the aer driver, use the root port's
383 if (!data
.aer_driver
|| !data
.aer_driver
->reset_link
) {
384 if (data
.is_downstream
&&
385 aerdev
->device
.driver
&&
386 to_service_driver(aerdev
->device
.driver
)->reset_link
) {
388 to_service_driver(aerdev
->device
.driver
);
390 printk(KERN_DEBUG
"No link-reset support to Device ID"
393 return PCI_ERS_RESULT_DISCONNECT
;
397 status
= data
.aer_driver
->reset_link(udev
);
398 if (status
!= PCI_ERS_RESULT_RECOVERED
) {
399 printk(KERN_DEBUG
"Link reset at upstream Device ID"
402 return PCI_ERS_RESULT_DISCONNECT
;
409 * do_recovery - handle nonfatal/fatal error recovery process
410 * @aerdev: pointer to a pcie_device data structure of root port
411 * @dev: pointer to a pci_dev data structure of agent detecting an error
412 * @severity: error severity type
414 * Invoked when an error is nonfatal/fatal. Once being invoked, broadcast
415 * error detected message to all downstream drivers within a hierarchy in
416 * question and return the returned code.
418 static pci_ers_result_t
do_recovery(struct pcie_device
*aerdev
,
422 pci_ers_result_t status
, result
= PCI_ERS_RESULT_RECOVERED
;
423 enum pci_channel_state state
;
425 if (severity
== AER_FATAL
)
426 state
= pci_channel_io_frozen
;
428 state
= pci_channel_io_normal
;
430 status
= broadcast_error_message(dev
,
433 report_error_detected
);
435 if (severity
== AER_FATAL
) {
436 result
= reset_link(aerdev
, dev
);
437 if (result
!= PCI_ERS_RESULT_RECOVERED
) {
438 /* TODO: Should panic here? */
443 if (status
== PCI_ERS_RESULT_CAN_RECOVER
)
444 status
= broadcast_error_message(dev
,
447 report_mmio_enabled
);
449 if (status
== PCI_ERS_RESULT_NEED_RESET
) {
451 * TODO: Should call platform-specific
452 * functions to reset slot before calling
453 * drivers' slot_reset callbacks?
455 status
= broadcast_error_message(dev
,
461 if (status
== PCI_ERS_RESULT_RECOVERED
)
462 broadcast_error_message(dev
,
471 * handle_error_source - handle logging error into an event log
472 * @aerdev: pointer to pcie_device data structure of the root port
473 * @dev: pointer to pci_dev data structure of error source device
474 * @info: comprehensive error information
476 * Invoked when an error being detected by Root Port.
478 static void handle_error_source(struct pcie_device
* aerdev
,
480 struct aer_err_info info
)
482 pci_ers_result_t status
= 0;
485 if (info
.severity
== AER_CORRECTABLE
) {
487 * Correctable error does not need software intevention.
488 * No need to go through error recovery process.
490 pos
= pci_find_aer_capability(dev
);
492 pci_write_config_dword(dev
, pos
+ PCI_ERR_COR_STATUS
,
495 status
= do_recovery(aerdev
, dev
, info
.severity
);
496 if (status
== PCI_ERS_RESULT_RECOVERED
) {
497 printk(KERN_DEBUG
"AER driver successfully recovered\n");
499 /* TODO: Should kernel panic here? */
500 printk(KERN_DEBUG
"AER driver didn't recover\n");
506 * aer_enable_rootport - enable Root Port's interrupts when receiving messages
507 * @rpc: pointer to a Root Port data structure
509 * Invoked when PCIE bus loads AER service driver.
511 void aer_enable_rootport(struct aer_rpc
*rpc
)
513 struct pci_dev
*pdev
= rpc
->rpd
->port
;
518 pos
= pci_find_capability(pdev
, PCI_CAP_ID_EXP
);
519 /* Clear PCIE Capability's Device Status */
520 pci_read_config_word(pdev
, pos
+PCI_EXP_DEVSTA
, ®16
);
521 pci_write_config_word(pdev
, pos
+PCI_EXP_DEVSTA
, reg16
);
523 /* Disable system error generation in response to error messages */
524 pci_read_config_word(pdev
, pos
+ PCI_EXP_RTCTL
, ®16
);
525 reg16
&= ~(SYSTEM_ERROR_INTR_ON_MESG_MASK
);
526 pci_write_config_word(pdev
, pos
+ PCI_EXP_RTCTL
, reg16
);
528 aer_pos
= pci_find_aer_capability(pdev
);
529 /* Clear error status */
530 pci_read_config_dword(pdev
, aer_pos
+ PCI_ERR_ROOT_STATUS
, ®32
);
531 pci_write_config_dword(pdev
, aer_pos
+ PCI_ERR_ROOT_STATUS
, reg32
);
532 pci_read_config_dword(pdev
, aer_pos
+ PCI_ERR_COR_STATUS
, ®32
);
533 pci_write_config_dword(pdev
, aer_pos
+ PCI_ERR_COR_STATUS
, reg32
);
534 pci_read_config_dword(pdev
, aer_pos
+ PCI_ERR_UNCOR_STATUS
, ®32
);
535 pci_write_config_dword(pdev
, aer_pos
+ PCI_ERR_UNCOR_STATUS
, reg32
);
537 /* Enable Root Port device reporting error itself */
538 pci_read_config_word(pdev
, pos
+PCI_EXP_DEVCTL
, ®16
);
540 PCI_EXP_DEVCTL_CERE
|
541 PCI_EXP_DEVCTL_NFERE
|
542 PCI_EXP_DEVCTL_FERE
|
544 pci_write_config_word(pdev
, pos
+PCI_EXP_DEVCTL
,
547 /* Enable Root Port's interrupt in response to error messages */
548 pci_write_config_dword(pdev
,
549 aer_pos
+ PCI_ERR_ROOT_COMMAND
,
550 ROOT_PORT_INTR_ON_MESG_MASK
);
554 * disable_root_aer - disable Root Port's interrupts when receiving messages
555 * @rpc: pointer to a Root Port data structure
557 * Invoked when PCIE bus unloads AER service driver.
559 static void disable_root_aer(struct aer_rpc
*rpc
)
561 struct pci_dev
*pdev
= rpc
->rpd
->port
;
565 pos
= pci_find_aer_capability(pdev
);
566 /* Disable Root's interrupt in response to error messages */
567 pci_write_config_dword(pdev
, pos
+ PCI_ERR_ROOT_COMMAND
, 0);
569 /* Clear Root's error status reg */
570 pci_read_config_dword(pdev
, pos
+ PCI_ERR_ROOT_STATUS
, ®32
);
571 pci_write_config_dword(pdev
, pos
+ PCI_ERR_ROOT_STATUS
, reg32
);
575 * get_e_source - retrieve an error source
576 * @rpc: pointer to the root port which holds an error
578 * Invoked by DPC handler to consume an error.
580 static struct aer_err_source
* get_e_source(struct aer_rpc
*rpc
)
582 struct aer_err_source
*e_source
;
585 /* Lock access to Root error producer/consumer index */
586 spin_lock_irqsave(&rpc
->e_lock
, flags
);
587 if (rpc
->prod_idx
== rpc
->cons_idx
) {
588 spin_unlock_irqrestore(&rpc
->e_lock
, flags
);
591 e_source
= &rpc
->e_sources
[rpc
->cons_idx
];
593 if (rpc
->cons_idx
== AER_ERROR_SOURCES_MAX
)
595 spin_unlock_irqrestore(&rpc
->e_lock
, flags
);
600 static int get_device_error_info(struct pci_dev
*dev
, struct aer_err_info
*info
)
604 pos
= pci_find_aer_capability(dev
);
606 /* The device might not support AER */
610 if (info
->severity
== AER_CORRECTABLE
) {
611 pci_read_config_dword(dev
, pos
+ PCI_ERR_COR_STATUS
,
613 if (!(info
->status
& ERR_CORRECTABLE_ERROR_MASK
))
614 return AER_UNSUCCESS
;
615 } else if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
||
616 info
->severity
== AER_NONFATAL
) {
618 /* Link is still healthy for IO reads */
619 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
,
621 if (!(info
->status
& ERR_UNCORRECTABLE_ERROR_MASK
))
622 return AER_UNSUCCESS
;
624 if (info
->status
& AER_LOG_TLP_MASKS
) {
625 info
->flags
|= AER_TLP_HEADER_VALID_FLAG
;
626 pci_read_config_dword(dev
,
627 pos
+ PCI_ERR_HEADER_LOG
, &info
->tlp
.dw0
);
628 pci_read_config_dword(dev
,
629 pos
+ PCI_ERR_HEADER_LOG
+ 4, &info
->tlp
.dw1
);
630 pci_read_config_dword(dev
,
631 pos
+ PCI_ERR_HEADER_LOG
+ 8, &info
->tlp
.dw2
);
632 pci_read_config_dword(dev
,
633 pos
+ PCI_ERR_HEADER_LOG
+ 12, &info
->tlp
.dw3
);
641 * aer_isr_one_error - consume an error detected by root port
642 * @p_device: pointer to error root port service device
643 * @e_src: pointer to an error source
645 static void aer_isr_one_error(struct pcie_device
*p_device
,
646 struct aer_err_source
*e_src
)
648 struct device
*s_device
;
649 struct aer_err_info e_info
= {0, 0, 0,};
654 * There is a possibility that both correctable error and
655 * uncorrectable error being logged. Report correctable error first.
657 for (i
= 1; i
& ROOT_ERR_STATUS_MASKS
; i
<<= 2) {
660 if (!(e_src
->status
& i
))
663 /* Init comprehensive error information */
664 if (i
& PCI_ERR_ROOT_COR_RCV
) {
665 id
= ERR_COR_ID(e_src
->id
);
666 e_info
.severity
= AER_CORRECTABLE
;
668 id
= ERR_UNCOR_ID(e_src
->id
);
669 e_info
.severity
= ((e_src
->status
>> 6) & 1);
672 (PCI_ERR_ROOT_MULTI_COR_RCV
|
673 PCI_ERR_ROOT_MULTI_UNCOR_RCV
))
674 e_info
.flags
|= AER_MULTI_ERROR_VALID_FLAG
;
675 if (!(s_device
= find_source_device(p_device
->port
, id
))) {
676 printk(KERN_DEBUG
"%s->can't find device of ID%04x\n",
680 if (get_device_error_info(to_pci_dev(s_device
), &e_info
) ==
682 aer_print_error(to_pci_dev(s_device
), &e_info
);
683 handle_error_source(p_device
,
684 to_pci_dev(s_device
),
691 * aer_isr - consume errors detected by root port
692 * @context: pointer to a private data of pcie device
694 * Invoked, as DPC, when root port records new detected error
696 void aer_isr(void *context
)
698 struct pcie_device
*p_device
= (struct pcie_device
*) context
;
699 struct aer_rpc
*rpc
= get_service_data(p_device
);
700 struct aer_err_source
*e_src
;
702 mutex_lock(&rpc
->rpc_mutex
);
703 e_src
= get_e_source(rpc
);
705 aer_isr_one_error(p_device
, e_src
);
706 e_src
= get_e_source(rpc
);
708 mutex_unlock(&rpc
->rpc_mutex
);
710 wake_up(&rpc
->wait_release
);
714 * aer_delete_rootport - disable root port aer and delete service data
715 * @rpc: pointer to a root port device being deleted
717 * Invoked when AER service unloaded on a specific Root Port
719 void aer_delete_rootport(struct aer_rpc
*rpc
)
721 /* Disable root port AER itself */
722 disable_root_aer(rpc
);
728 * aer_init - provide AER initialization
729 * @dev: pointer to AER pcie device
731 * Invoked when AER service driver is loaded.
733 int aer_init(struct pcie_device
*dev
)
737 /* Run _OSC Method */
738 status
= aer_osc_setup(dev
->port
);
740 if(status
!= OSC_METHOD_RUN_SUCCESS
) {
741 printk(KERN_DEBUG
"%s: AER service init fails - %s\n",
743 (status
== OSC_METHOD_NOT_SUPPORTED
) ?
744 "No ACPI _OSC support" : "Run ACPI _OSC fails");
753 EXPORT_SYMBOL_GPL(pci_find_aer_capability
);
754 EXPORT_SYMBOL_GPL(pci_enable_pcie_error_reporting
);
755 EXPORT_SYMBOL_GPL(pci_disable_pcie_error_reporting
);
756 EXPORT_SYMBOL_GPL(pci_cleanup_aer_uncorrect_error_status
);