2 * drivers/pci/pcie/aer/aerdrv_core.c
4 * This file is subject to the terms and conditions of the GNU General Public
5 * License. See the file "COPYING" in the main directory of this archive
8 * This file implements the core part of PCI-Express AER. When an pci-express
9 * error is delivered, an error message will be collected and printed to
10 * console, then, an error recovery procedure will be executed by following
11 * the pci error recovery rules.
13 * Copyright (C) 2006 Intel Corp.
14 * Tom Long Nguyen (tom.l.nguyen@intel.com)
15 * Zhang Yanmin (yanmin.zhang@intel.com)
19 #include <linux/module.h>
20 #include <linux/pci.h>
21 #include <linux/kernel.h>
22 #include <linux/errno.h>
24 #include <linux/suspend.h>
25 #include <linux/delay.h>
29 module_param(forceload
, bool, 0);
31 int pci_enable_pcie_error_reporting(struct pci_dev
*dev
)
36 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_ERR
);
40 pos
= pci_find_capability(dev
, PCI_CAP_ID_EXP
);
44 pci_read_config_word(dev
, pos
+PCI_EXP_DEVCTL
, ®16
);
47 PCI_EXP_DEVCTL_NFERE
|
50 pci_write_config_word(dev
, pos
+PCI_EXP_DEVCTL
,
55 int pci_disable_pcie_error_reporting(struct pci_dev
*dev
)
60 pos
= pci_find_capability(dev
, PCI_CAP_ID_EXP
);
64 pci_read_config_word(dev
, pos
+PCI_EXP_DEVCTL
, ®16
);
65 reg16
= reg16
& ~(PCI_EXP_DEVCTL_CERE
|
66 PCI_EXP_DEVCTL_NFERE
|
69 pci_write_config_word(dev
, pos
+PCI_EXP_DEVCTL
,
74 int pci_cleanup_aer_uncorrect_error_status(struct pci_dev
*dev
)
79 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_ERR
);
83 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
, &status
);
84 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_SEVER
, &mask
);
85 if (dev
->error_state
== pci_channel_io_normal
)
86 status
&= ~mask
; /* Clear corresponding nonfatal bits */
88 status
&= mask
; /* Clear corresponding fatal bits */
89 pci_write_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
, status
);
95 int pci_cleanup_aer_correct_error_status(struct pci_dev
*dev
)
100 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_ERR
);
104 pci_read_config_dword(dev
, pos
+ PCI_ERR_COR_STATUS
, &status
);
105 pci_write_config_dword(dev
, pos
+ PCI_ERR_COR_STATUS
, status
);
112 static void set_device_error_reporting(struct pci_dev
*dev
, void *data
)
114 bool enable
= *((bool *)data
);
116 if (dev
->pcie_type
== PCIE_RC_PORT
||
117 dev
->pcie_type
== PCIE_SW_UPSTREAM_PORT
||
118 dev
->pcie_type
== PCIE_SW_DOWNSTREAM_PORT
) {
120 pci_enable_pcie_error_reporting(dev
);
122 pci_disable_pcie_error_reporting(dev
);
126 pcie_set_ecrc_checking(dev
);
130 * set_downstream_devices_error_reporting - enable/disable the error reporting bits on the root port and its downstream ports.
131 * @dev: pointer to root port's pci_dev data structure
132 * @enable: true = enable error reporting, false = disable error reporting.
134 static void set_downstream_devices_error_reporting(struct pci_dev
*dev
,
137 set_device_error_reporting(dev
, &enable
);
139 if (!dev
->subordinate
)
141 pci_walk_bus(dev
->subordinate
, set_device_error_reporting
, &enable
);
144 static int find_device_iter(struct device
*device
, void *data
)
147 u16 id
= *(unsigned long *)data
;
148 u8 secondary
, subordinate
, d_bus
= id
>> 8;
150 if (device
->bus
== &pci_bus_type
) {
151 dev
= to_pci_dev(device
);
152 if (id
== ((dev
->bus
->number
<< 8) | dev
->devfn
)) {
156 *(unsigned long*)data
= (unsigned long)device
;
161 * If device is P2P, check if it is an upstream?
163 if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
) {
164 pci_read_config_byte(dev
, PCI_SECONDARY_BUS
,
166 pci_read_config_byte(dev
, PCI_SUBORDINATE_BUS
,
168 if (d_bus
>= secondary
&& d_bus
<= subordinate
) {
169 *(unsigned long*)data
= (unsigned long)device
;
179 * find_source_device - search through device hierarchy for source device
180 * @parent: pointer to Root Port pci_dev data structure
181 * @id: device ID of agent who sends an error message to this Root Port
183 * Invoked when error is detected at the Root Port.
185 static struct device
* find_source_device(struct pci_dev
*parent
, u16 id
)
187 struct pci_dev
*dev
= parent
;
188 struct device
*device
;
189 unsigned long device_addr
;
192 /* Is Root Port an agent that sends error message? */
193 if (id
== ((dev
->bus
->number
<< 8) | dev
->devfn
))
198 if ((status
= device_for_each_child(&dev
->dev
,
199 &device_addr
, find_device_iter
))) {
200 device
= (struct device
*)device_addr
;
201 dev
= to_pci_dev(device
);
202 if (id
== ((dev
->bus
->number
<< 8) | dev
->devfn
))
210 static void report_error_detected(struct pci_dev
*dev
, void *data
)
212 pci_ers_result_t vote
;
213 struct pci_error_handlers
*err_handler
;
214 struct aer_broadcast_data
*result_data
;
215 result_data
= (struct aer_broadcast_data
*) data
;
217 dev
->error_state
= result_data
->state
;
220 !dev
->driver
->err_handler
||
221 !dev
->driver
->err_handler
->error_detected
) {
222 if (result_data
->state
== pci_channel_io_frozen
&&
223 !(dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
)) {
225 * In case of fatal recovery, if one of down-
226 * stream device has no driver. We might be
227 * unable to recover because a later insmod
228 * of a driver for this device is unaware of
231 dev_printk(KERN_DEBUG
, &dev
->dev
, "device has %s\n",
233 "no AER-aware driver" : "no driver");
238 err_handler
= dev
->driver
->err_handler
;
239 vote
= err_handler
->error_detected(dev
, result_data
->state
);
240 result_data
->result
= merge_result(result_data
->result
, vote
);
244 static void report_mmio_enabled(struct pci_dev
*dev
, void *data
)
246 pci_ers_result_t vote
;
247 struct pci_error_handlers
*err_handler
;
248 struct aer_broadcast_data
*result_data
;
249 result_data
= (struct aer_broadcast_data
*) data
;
252 !dev
->driver
->err_handler
||
253 !dev
->driver
->err_handler
->mmio_enabled
)
256 err_handler
= dev
->driver
->err_handler
;
257 vote
= err_handler
->mmio_enabled(dev
);
258 result_data
->result
= merge_result(result_data
->result
, vote
);
262 static void report_slot_reset(struct pci_dev
*dev
, void *data
)
264 pci_ers_result_t vote
;
265 struct pci_error_handlers
*err_handler
;
266 struct aer_broadcast_data
*result_data
;
267 result_data
= (struct aer_broadcast_data
*) data
;
270 !dev
->driver
->err_handler
||
271 !dev
->driver
->err_handler
->slot_reset
)
274 err_handler
= dev
->driver
->err_handler
;
275 vote
= err_handler
->slot_reset(dev
);
276 result_data
->result
= merge_result(result_data
->result
, vote
);
280 static void report_resume(struct pci_dev
*dev
, void *data
)
282 struct pci_error_handlers
*err_handler
;
284 dev
->error_state
= pci_channel_io_normal
;
287 !dev
->driver
->err_handler
||
288 !dev
->driver
->err_handler
->resume
)
291 err_handler
= dev
->driver
->err_handler
;
292 err_handler
->resume(dev
);
297 * broadcast_error_message - handle message broadcast to downstream drivers
298 * @dev: pointer to from where in a hierarchy message is broadcasted down
299 * @state: error state
300 * @error_mesg: message to print
301 * @cb: callback to be broadcasted
303 * Invoked during error recovery process. Once being invoked, the content
304 * of error severity will be broadcasted to all downstream drivers in a
305 * hierarchy in question.
307 static pci_ers_result_t
broadcast_error_message(struct pci_dev
*dev
,
308 enum pci_channel_state state
,
310 void (*cb
)(struct pci_dev
*, void *))
312 struct aer_broadcast_data result_data
;
314 dev_printk(KERN_DEBUG
, &dev
->dev
, "broadcast %s message\n", error_mesg
);
315 result_data
.state
= state
;
316 if (cb
== report_error_detected
)
317 result_data
.result
= PCI_ERS_RESULT_CAN_RECOVER
;
319 result_data
.result
= PCI_ERS_RESULT_RECOVERED
;
321 if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
) {
323 * If the error is reported by a bridge, we think this error
324 * is related to the downstream link of the bridge, so we
325 * do error recovery on all subordinates of the bridge instead
326 * of the bridge and clear the error status of the bridge.
328 if (cb
== report_error_detected
)
329 dev
->error_state
= state
;
330 pci_walk_bus(dev
->subordinate
, cb
, &result_data
);
331 if (cb
== report_resume
) {
332 pci_cleanup_aer_uncorrect_error_status(dev
);
333 dev
->error_state
= pci_channel_io_normal
;
338 * If the error is reported by an end point, we think this
339 * error is related to the upstream link of the end point.
341 pci_walk_bus(dev
->bus
, cb
, &result_data
);
344 return result_data
.result
;
347 struct find_aer_service_data
{
348 struct pcie_port_service_driver
*aer_driver
;
352 static int find_aer_service_iter(struct device
*device
, void *data
)
354 struct device_driver
*driver
;
355 struct pcie_port_service_driver
*service_driver
;
356 struct find_aer_service_data
*result
;
358 result
= (struct find_aer_service_data
*) data
;
360 if (device
->bus
== &pcie_port_bus_type
) {
361 struct pcie_port_data
*port_data
;
363 port_data
= pci_get_drvdata(to_pcie_device(device
)->port
);
364 if (port_data
->port_type
== PCIE_SW_DOWNSTREAM_PORT
)
365 result
->is_downstream
= 1;
367 driver
= device
->driver
;
369 service_driver
= to_service_driver(driver
);
370 if (service_driver
->service
== PCIE_PORT_SERVICE_AER
) {
371 result
->aer_driver
= service_driver
;
380 static void find_aer_service(struct pci_dev
*dev
,
381 struct find_aer_service_data
*data
)
384 retval
= device_for_each_child(&dev
->dev
, data
, find_aer_service_iter
);
387 static pci_ers_result_t
reset_link(struct pcie_device
*aerdev
,
390 struct pci_dev
*udev
;
391 pci_ers_result_t status
;
392 struct find_aer_service_data data
;
394 if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
)
397 udev
= dev
->bus
->self
;
399 data
.is_downstream
= 0;
400 data
.aer_driver
= NULL
;
401 find_aer_service(udev
, &data
);
404 * Use the aer driver of the error agent firstly.
405 * If it hasn't the aer driver, use the root port's
407 if (!data
.aer_driver
|| !data
.aer_driver
->reset_link
) {
408 if (data
.is_downstream
&&
409 aerdev
->device
.driver
&&
410 to_service_driver(aerdev
->device
.driver
)->reset_link
) {
412 to_service_driver(aerdev
->device
.driver
);
414 dev_printk(KERN_DEBUG
, &dev
->dev
, "no link-reset "
416 return PCI_ERS_RESULT_DISCONNECT
;
420 status
= data
.aer_driver
->reset_link(udev
);
421 if (status
!= PCI_ERS_RESULT_RECOVERED
) {
422 dev_printk(KERN_DEBUG
, &dev
->dev
, "link reset at upstream "
423 "device %s failed\n", pci_name(udev
));
424 return PCI_ERS_RESULT_DISCONNECT
;
431 * do_recovery - handle nonfatal/fatal error recovery process
432 * @aerdev: pointer to a pcie_device data structure of root port
433 * @dev: pointer to a pci_dev data structure of agent detecting an error
434 * @severity: error severity type
436 * Invoked when an error is nonfatal/fatal. Once being invoked, broadcast
437 * error detected message to all downstream drivers within a hierarchy in
438 * question and return the returned code.
440 static pci_ers_result_t
do_recovery(struct pcie_device
*aerdev
,
444 pci_ers_result_t status
, result
= PCI_ERS_RESULT_RECOVERED
;
445 enum pci_channel_state state
;
447 if (severity
== AER_FATAL
)
448 state
= pci_channel_io_frozen
;
450 state
= pci_channel_io_normal
;
452 status
= broadcast_error_message(dev
,
455 report_error_detected
);
457 if (severity
== AER_FATAL
) {
458 result
= reset_link(aerdev
, dev
);
459 if (result
!= PCI_ERS_RESULT_RECOVERED
) {
460 /* TODO: Should panic here? */
465 if (status
== PCI_ERS_RESULT_CAN_RECOVER
)
466 status
= broadcast_error_message(dev
,
469 report_mmio_enabled
);
471 if (status
== PCI_ERS_RESULT_NEED_RESET
) {
473 * TODO: Should call platform-specific
474 * functions to reset slot before calling
475 * drivers' slot_reset callbacks?
477 status
= broadcast_error_message(dev
,
483 if (status
== PCI_ERS_RESULT_RECOVERED
)
484 broadcast_error_message(dev
,
493 * handle_error_source - handle logging error into an event log
494 * @aerdev: pointer to pcie_device data structure of the root port
495 * @dev: pointer to pci_dev data structure of error source device
496 * @info: comprehensive error information
498 * Invoked when an error being detected by Root Port.
500 static void handle_error_source(struct pcie_device
* aerdev
,
502 struct aer_err_info info
)
504 pci_ers_result_t status
= 0;
507 if (info
.severity
== AER_CORRECTABLE
) {
509 * Correctable error does not need software intevention.
510 * No need to go through error recovery process.
512 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_ERR
);
514 pci_write_config_dword(dev
, pos
+ PCI_ERR_COR_STATUS
,
517 status
= do_recovery(aerdev
, dev
, info
.severity
);
518 if (status
== PCI_ERS_RESULT_RECOVERED
) {
519 dev_printk(KERN_DEBUG
, &dev
->dev
, "AER driver "
520 "successfully recovered\n");
522 /* TODO: Should kernel panic here? */
523 dev_printk(KERN_DEBUG
, &dev
->dev
, "AER driver didn't "
530 * aer_enable_rootport - enable Root Port's interrupts when receiving messages
531 * @rpc: pointer to a Root Port data structure
533 * Invoked when PCIE bus loads AER service driver.
535 void aer_enable_rootport(struct aer_rpc
*rpc
)
537 struct pci_dev
*pdev
= rpc
->rpd
->port
;
542 pos
= pci_find_capability(pdev
, PCI_CAP_ID_EXP
);
543 /* Clear PCIE Capability's Device Status */
544 pci_read_config_word(pdev
, pos
+PCI_EXP_DEVSTA
, ®16
);
545 pci_write_config_word(pdev
, pos
+PCI_EXP_DEVSTA
, reg16
);
547 /* Disable system error generation in response to error messages */
548 pci_read_config_word(pdev
, pos
+ PCI_EXP_RTCTL
, ®16
);
549 reg16
&= ~(SYSTEM_ERROR_INTR_ON_MESG_MASK
);
550 pci_write_config_word(pdev
, pos
+ PCI_EXP_RTCTL
, reg16
);
552 aer_pos
= pci_find_ext_capability(pdev
, PCI_EXT_CAP_ID_ERR
);
553 /* Clear error status */
554 pci_read_config_dword(pdev
, aer_pos
+ PCI_ERR_ROOT_STATUS
, ®32
);
555 pci_write_config_dword(pdev
, aer_pos
+ PCI_ERR_ROOT_STATUS
, reg32
);
556 pci_read_config_dword(pdev
, aer_pos
+ PCI_ERR_COR_STATUS
, ®32
);
557 pci_write_config_dword(pdev
, aer_pos
+ PCI_ERR_COR_STATUS
, reg32
);
558 pci_read_config_dword(pdev
, aer_pos
+ PCI_ERR_UNCOR_STATUS
, ®32
);
559 pci_write_config_dword(pdev
, aer_pos
+ PCI_ERR_UNCOR_STATUS
, reg32
);
562 * Enable error reporting for the root port device and downstream port
565 set_downstream_devices_error_reporting(pdev
, true);
567 /* Enable Root Port's interrupt in response to error messages */
568 pci_write_config_dword(pdev
,
569 aer_pos
+ PCI_ERR_ROOT_COMMAND
,
570 ROOT_PORT_INTR_ON_MESG_MASK
);
574 * disable_root_aer - disable Root Port's interrupts when receiving messages
575 * @rpc: pointer to a Root Port data structure
577 * Invoked when PCIE bus unloads AER service driver.
579 static void disable_root_aer(struct aer_rpc
*rpc
)
581 struct pci_dev
*pdev
= rpc
->rpd
->port
;
586 * Disable error reporting for the root port device and downstream port
589 set_downstream_devices_error_reporting(pdev
, false);
591 pos
= pci_find_ext_capability(pdev
, PCI_EXT_CAP_ID_ERR
);
592 /* Disable Root's interrupt in response to error messages */
593 pci_write_config_dword(pdev
, pos
+ PCI_ERR_ROOT_COMMAND
, 0);
595 /* Clear Root's error status reg */
596 pci_read_config_dword(pdev
, pos
+ PCI_ERR_ROOT_STATUS
, ®32
);
597 pci_write_config_dword(pdev
, pos
+ PCI_ERR_ROOT_STATUS
, reg32
);
601 * get_e_source - retrieve an error source
602 * @rpc: pointer to the root port which holds an error
604 * Invoked by DPC handler to consume an error.
606 static struct aer_err_source
* get_e_source(struct aer_rpc
*rpc
)
608 struct aer_err_source
*e_source
;
611 /* Lock access to Root error producer/consumer index */
612 spin_lock_irqsave(&rpc
->e_lock
, flags
);
613 if (rpc
->prod_idx
== rpc
->cons_idx
) {
614 spin_unlock_irqrestore(&rpc
->e_lock
, flags
);
617 e_source
= &rpc
->e_sources
[rpc
->cons_idx
];
619 if (rpc
->cons_idx
== AER_ERROR_SOURCES_MAX
)
621 spin_unlock_irqrestore(&rpc
->e_lock
, flags
);
626 static int get_device_error_info(struct pci_dev
*dev
, struct aer_err_info
*info
)
630 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_ERR
);
632 /* The device might not support AER */
636 if (info
->severity
== AER_CORRECTABLE
) {
637 pci_read_config_dword(dev
, pos
+ PCI_ERR_COR_STATUS
,
639 if (!(info
->status
& ERR_CORRECTABLE_ERROR_MASK
))
640 return AER_UNSUCCESS
;
641 } else if (dev
->hdr_type
& PCI_HEADER_TYPE_BRIDGE
||
642 info
->severity
== AER_NONFATAL
) {
644 /* Link is still healthy for IO reads */
645 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
,
647 if (!(info
->status
& ERR_UNCORRECTABLE_ERROR_MASK
))
648 return AER_UNSUCCESS
;
650 if (info
->status
& AER_LOG_TLP_MASKS
) {
651 info
->flags
|= AER_TLP_HEADER_VALID_FLAG
;
652 pci_read_config_dword(dev
,
653 pos
+ PCI_ERR_HEADER_LOG
, &info
->tlp
.dw0
);
654 pci_read_config_dword(dev
,
655 pos
+ PCI_ERR_HEADER_LOG
+ 4, &info
->tlp
.dw1
);
656 pci_read_config_dword(dev
,
657 pos
+ PCI_ERR_HEADER_LOG
+ 8, &info
->tlp
.dw2
);
658 pci_read_config_dword(dev
,
659 pos
+ PCI_ERR_HEADER_LOG
+ 12, &info
->tlp
.dw3
);
667 * aer_isr_one_error - consume an error detected by root port
668 * @p_device: pointer to error root port service device
669 * @e_src: pointer to an error source
671 static void aer_isr_one_error(struct pcie_device
*p_device
,
672 struct aer_err_source
*e_src
)
674 struct device
*s_device
;
675 struct aer_err_info e_info
= {0, 0, 0,};
680 * There is a possibility that both correctable error and
681 * uncorrectable error being logged. Report correctable error first.
683 for (i
= 1; i
& ROOT_ERR_STATUS_MASKS
; i
<<= 2) {
686 if (!(e_src
->status
& i
))
689 /* Init comprehensive error information */
690 if (i
& PCI_ERR_ROOT_COR_RCV
) {
691 id
= ERR_COR_ID(e_src
->id
);
692 e_info
.severity
= AER_CORRECTABLE
;
694 id
= ERR_UNCOR_ID(e_src
->id
);
695 e_info
.severity
= ((e_src
->status
>> 6) & 1);
698 (PCI_ERR_ROOT_MULTI_COR_RCV
|
699 PCI_ERR_ROOT_MULTI_UNCOR_RCV
))
700 e_info
.flags
|= AER_MULTI_ERROR_VALID_FLAG
;
701 if (!(s_device
= find_source_device(p_device
->port
, id
))) {
702 printk(KERN_DEBUG
"%s->can't find device of ID%04x\n",
706 if (get_device_error_info(to_pci_dev(s_device
), &e_info
) ==
708 aer_print_error(to_pci_dev(s_device
), &e_info
);
709 handle_error_source(p_device
,
710 to_pci_dev(s_device
),
717 * aer_isr - consume errors detected by root port
718 * @work: definition of this work item
720 * Invoked, as DPC, when root port records new detected error
722 void aer_isr(struct work_struct
*work
)
724 struct aer_rpc
*rpc
= container_of(work
, struct aer_rpc
, dpc_handler
);
725 struct pcie_device
*p_device
= rpc
->rpd
;
726 struct aer_err_source
*e_src
;
728 mutex_lock(&rpc
->rpc_mutex
);
729 e_src
= get_e_source(rpc
);
731 aer_isr_one_error(p_device
, e_src
);
732 e_src
= get_e_source(rpc
);
734 mutex_unlock(&rpc
->rpc_mutex
);
736 wake_up(&rpc
->wait_release
);
740 * aer_delete_rootport - disable root port aer and delete service data
741 * @rpc: pointer to a root port device being deleted
743 * Invoked when AER service unloaded on a specific Root Port
745 void aer_delete_rootport(struct aer_rpc
*rpc
)
747 /* Disable root port AER itself */
748 disable_root_aer(rpc
);
754 * aer_init - provide AER initialization
755 * @dev: pointer to AER pcie device
757 * Invoked when AER service driver is loaded.
759 int aer_init(struct pcie_device
*dev
)
761 if (aer_osc_setup(dev
) && !forceload
)
767 EXPORT_SYMBOL_GPL(pci_enable_pcie_error_reporting
);
768 EXPORT_SYMBOL_GPL(pci_disable_pcie_error_reporting
);
769 EXPORT_SYMBOL_GPL(pci_cleanup_aer_uncorrect_error_status
);