2 * File: drivers/pci/pcie/aspm.c
3 * Enabling PCIe link L0s/L1 state and Clock Power Management
5 * Copyright (C) 2007 Intel
6 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com)
7 * Copyright (C) Shaohua Li (shaohua.li@intel.com)
10 #include <linux/kernel.h>
11 #include <linux/module.h>
12 #include <linux/moduleparam.h>
13 #include <linux/pci.h>
14 #include <linux/pci_regs.h>
15 #include <linux/errno.h>
17 #include <linux/init.h>
18 #include <linux/slab.h>
19 #include <linux/jiffies.h>
20 #include <linux/delay.h>
21 #include <linux/pci-aspm.h>
24 #ifdef MODULE_PARAM_PREFIX
25 #undef MODULE_PARAM_PREFIX
27 #define MODULE_PARAM_PREFIX "pcie_aspm."
29 /* Note: those are not register definitions */
30 #define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */
31 #define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */
32 #define ASPM_STATE_L1 (4) /* L1 state */
33 #define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW)
34 #define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1)
37 u32 l0s
; /* L0s latency (nsec) */
38 u32 l1
; /* L1 latency (nsec) */
41 struct pcie_link_state
{
42 struct pci_dev
*pdev
; /* Upstream component of the Link */
43 struct pcie_link_state
*root
; /* pointer to the root port link */
44 struct pcie_link_state
*parent
; /* pointer to the parent Link state */
45 struct list_head sibling
; /* node in link_list */
46 struct list_head children
; /* list of child link states */
47 struct list_head link
; /* node in parent's children list */
50 u32 aspm_support
:3; /* Supported ASPM state */
51 u32 aspm_enabled
:3; /* Enabled ASPM state */
52 u32 aspm_capable
:3; /* Capable ASPM state with latency */
53 u32 aspm_default
:3; /* Default ASPM state by BIOS */
54 u32 aspm_disable
:3; /* Disabled ASPM state */
57 u32 clkpm_capable
:1; /* Clock PM capable? */
58 u32 clkpm_enabled
:1; /* Current Clock PM state */
59 u32 clkpm_default
:1; /* Default Clock PM state by BIOS */
62 struct aspm_latency latency_up
; /* Upstream direction exit latency */
63 struct aspm_latency latency_dw
; /* Downstream direction exit latency */
65 * Endpoint acceptable latencies. A pcie downstream port only
66 * has one slot under it, so at most there are 8 functions.
68 struct aspm_latency acceptable
[8];
71 static int aspm_disabled
, aspm_force
, aspm_clear_state
;
72 static DEFINE_MUTEX(aspm_lock
);
73 static LIST_HEAD(link_list
);
75 #define POLICY_DEFAULT 0 /* BIOS default setting */
76 #define POLICY_PERFORMANCE 1 /* high performance */
77 #define POLICY_POWERSAVE 2 /* high power saving */
78 static int aspm_policy
;
79 static const char *policy_str
[] = {
80 [POLICY_DEFAULT
] = "default",
81 [POLICY_PERFORMANCE
] = "performance",
82 [POLICY_POWERSAVE
] = "powersave"
85 #define LINK_RETRAIN_TIMEOUT HZ
87 static int policy_to_aspm_state(struct pcie_link_state
*link
)
89 switch (aspm_policy
) {
90 case POLICY_PERFORMANCE
:
91 /* Disable ASPM and Clock PM */
93 case POLICY_POWERSAVE
:
94 /* Enable ASPM L0s/L1 */
95 return ASPM_STATE_ALL
;
97 return link
->aspm_default
;
102 static int policy_to_clkpm_state(struct pcie_link_state
*link
)
104 switch (aspm_policy
) {
105 case POLICY_PERFORMANCE
:
106 /* Disable ASPM and Clock PM */
108 case POLICY_POWERSAVE
:
109 /* Disable Clock PM */
112 return link
->clkpm_default
;
117 static void pcie_set_clkpm_nocheck(struct pcie_link_state
*link
, int enable
)
121 struct pci_dev
*child
;
122 struct pci_bus
*linkbus
= link
->pdev
->subordinate
;
124 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
125 pos
= pci_pcie_cap(child
);
128 pci_read_config_word(child
, pos
+ PCI_EXP_LNKCTL
, ®16
);
130 reg16
|= PCI_EXP_LNKCTL_CLKREQ_EN
;
132 reg16
&= ~PCI_EXP_LNKCTL_CLKREQ_EN
;
133 pci_write_config_word(child
, pos
+ PCI_EXP_LNKCTL
, reg16
);
135 link
->clkpm_enabled
= !!enable
;
138 static void pcie_set_clkpm(struct pcie_link_state
*link
, int enable
)
140 /* Don't enable Clock PM if the link is not Clock PM capable */
141 if (!link
->clkpm_capable
&& enable
)
143 /* Need nothing if the specified equals to current state */
144 if (link
->clkpm_enabled
== enable
)
146 pcie_set_clkpm_nocheck(link
, enable
);
149 static void pcie_clkpm_cap_init(struct pcie_link_state
*link
, int blacklist
)
151 int pos
, capable
= 1, enabled
= 1;
154 struct pci_dev
*child
;
155 struct pci_bus
*linkbus
= link
->pdev
->subordinate
;
157 /* All functions should have the same cap and state, take the worst */
158 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
159 pos
= pci_pcie_cap(child
);
162 pci_read_config_dword(child
, pos
+ PCI_EXP_LNKCAP
, ®32
);
163 if (!(reg32
& PCI_EXP_LNKCAP_CLKPM
)) {
168 pci_read_config_word(child
, pos
+ PCI_EXP_LNKCTL
, ®16
);
169 if (!(reg16
& PCI_EXP_LNKCTL_CLKREQ_EN
))
172 link
->clkpm_enabled
= enabled
;
173 link
->clkpm_default
= enabled
;
174 link
->clkpm_capable
= (blacklist
) ? 0 : capable
;
178 * pcie_aspm_configure_common_clock: check if the 2 ends of a link
179 * could use common clock. If they are, configure them to use the
180 * common clock. That will reduce the ASPM state exit latency.
182 static void pcie_aspm_configure_common_clock(struct pcie_link_state
*link
)
184 int ppos
, cpos
, same_clock
= 1;
185 u16 reg16
, parent_reg
, child_reg
[8];
186 unsigned long start_jiffies
;
187 struct pci_dev
*child
, *parent
= link
->pdev
;
188 struct pci_bus
*linkbus
= parent
->subordinate
;
190 * All functions of a slot should have the same Slot Clock
191 * Configuration, so just check one function
193 child
= list_entry(linkbus
->devices
.next
, struct pci_dev
, bus_list
);
194 BUG_ON(!pci_is_pcie(child
));
196 /* Check downstream component if bit Slot Clock Configuration is 1 */
197 cpos
= pci_pcie_cap(child
);
198 pci_read_config_word(child
, cpos
+ PCI_EXP_LNKSTA
, ®16
);
199 if (!(reg16
& PCI_EXP_LNKSTA_SLC
))
202 /* Check upstream component if bit Slot Clock Configuration is 1 */
203 ppos
= pci_pcie_cap(parent
);
204 pci_read_config_word(parent
, ppos
+ PCI_EXP_LNKSTA
, ®16
);
205 if (!(reg16
& PCI_EXP_LNKSTA_SLC
))
208 /* Configure downstream component, all functions */
209 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
210 cpos
= pci_pcie_cap(child
);
211 pci_read_config_word(child
, cpos
+ PCI_EXP_LNKCTL
, ®16
);
212 child_reg
[PCI_FUNC(child
->devfn
)] = reg16
;
214 reg16
|= PCI_EXP_LNKCTL_CCC
;
216 reg16
&= ~PCI_EXP_LNKCTL_CCC
;
217 pci_write_config_word(child
, cpos
+ PCI_EXP_LNKCTL
, reg16
);
220 /* Configure upstream component */
221 pci_read_config_word(parent
, ppos
+ PCI_EXP_LNKCTL
, ®16
);
224 reg16
|= PCI_EXP_LNKCTL_CCC
;
226 reg16
&= ~PCI_EXP_LNKCTL_CCC
;
227 pci_write_config_word(parent
, ppos
+ PCI_EXP_LNKCTL
, reg16
);
230 reg16
|= PCI_EXP_LNKCTL_RL
;
231 pci_write_config_word(parent
, ppos
+ PCI_EXP_LNKCTL
, reg16
);
233 /* Wait for link training end. Break out after waiting for timeout */
234 start_jiffies
= jiffies
;
236 pci_read_config_word(parent
, ppos
+ PCI_EXP_LNKSTA
, ®16
);
237 if (!(reg16
& PCI_EXP_LNKSTA_LT
))
239 if (time_after(jiffies
, start_jiffies
+ LINK_RETRAIN_TIMEOUT
))
243 if (!(reg16
& PCI_EXP_LNKSTA_LT
))
246 /* Training failed. Restore common clock configurations */
247 dev_printk(KERN_ERR
, &parent
->dev
,
248 "ASPM: Could not configure common clock\n");
249 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
250 cpos
= pci_pcie_cap(child
);
251 pci_write_config_word(child
, cpos
+ PCI_EXP_LNKCTL
,
252 child_reg
[PCI_FUNC(child
->devfn
)]);
254 pci_write_config_word(parent
, ppos
+ PCI_EXP_LNKCTL
, parent_reg
);
257 /* Convert L0s latency encoding to ns */
258 static u32
calc_l0s_latency(u32 encoding
)
261 return (5 * 1000); /* > 4us */
262 return (64 << encoding
);
265 /* Convert L0s acceptable latency encoding to ns */
266 static u32
calc_l0s_acceptable(u32 encoding
)
270 return (64 << encoding
);
273 /* Convert L1 latency encoding to ns */
274 static u32
calc_l1_latency(u32 encoding
)
277 return (65 * 1000); /* > 64us */
278 return (1000 << encoding
);
281 /* Convert L1 acceptable latency encoding to ns */
282 static u32
calc_l1_acceptable(u32 encoding
)
286 return (1000 << encoding
);
289 struct aspm_register_info
{
292 u32 latency_encoding_l0s
;
293 u32 latency_encoding_l1
;
296 static void pcie_get_aspm_reg(struct pci_dev
*pdev
,
297 struct aspm_register_info
*info
)
303 pos
= pci_pcie_cap(pdev
);
304 pci_read_config_dword(pdev
, pos
+ PCI_EXP_LNKCAP
, ®32
);
305 info
->support
= (reg32
& PCI_EXP_LNKCAP_ASPMS
) >> 10;
306 info
->latency_encoding_l0s
= (reg32
& PCI_EXP_LNKCAP_L0SEL
) >> 12;
307 info
->latency_encoding_l1
= (reg32
& PCI_EXP_LNKCAP_L1EL
) >> 15;
308 pci_read_config_word(pdev
, pos
+ PCI_EXP_LNKCTL
, ®16
);
309 info
->enabled
= reg16
& PCI_EXP_LNKCTL_ASPMC
;
312 static void pcie_aspm_check_latency(struct pci_dev
*endpoint
)
314 u32 latency
, l1_switch_latency
= 0;
315 struct aspm_latency
*acceptable
;
316 struct pcie_link_state
*link
;
318 /* Device not in D0 doesn't need latency check */
319 if ((endpoint
->current_state
!= PCI_D0
) &&
320 (endpoint
->current_state
!= PCI_UNKNOWN
))
323 link
= endpoint
->bus
->self
->link_state
;
324 acceptable
= &link
->acceptable
[PCI_FUNC(endpoint
->devfn
)];
327 /* Check upstream direction L0s latency */
328 if ((link
->aspm_capable
& ASPM_STATE_L0S_UP
) &&
329 (link
->latency_up
.l0s
> acceptable
->l0s
))
330 link
->aspm_capable
&= ~ASPM_STATE_L0S_UP
;
332 /* Check downstream direction L0s latency */
333 if ((link
->aspm_capable
& ASPM_STATE_L0S_DW
) &&
334 (link
->latency_dw
.l0s
> acceptable
->l0s
))
335 link
->aspm_capable
&= ~ASPM_STATE_L0S_DW
;
338 * Every switch on the path to root complex need 1
339 * more microsecond for L1. Spec doesn't mention L0s.
341 latency
= max_t(u32
, link
->latency_up
.l1
, link
->latency_dw
.l1
);
342 if ((link
->aspm_capable
& ASPM_STATE_L1
) &&
343 (latency
+ l1_switch_latency
> acceptable
->l1
))
344 link
->aspm_capable
&= ~ASPM_STATE_L1
;
345 l1_switch_latency
+= 1000;
351 static void pcie_aspm_cap_init(struct pcie_link_state
*link
, int blacklist
)
353 struct pci_dev
*child
, *parent
= link
->pdev
;
354 struct pci_bus
*linkbus
= parent
->subordinate
;
355 struct aspm_register_info upreg
, dwreg
;
358 /* Set enabled/disable so that we will disable ASPM later */
359 link
->aspm_enabled
= ASPM_STATE_ALL
;
360 link
->aspm_disable
= ASPM_STATE_ALL
;
364 /* Configure common clock before checking latencies */
365 pcie_aspm_configure_common_clock(link
);
367 /* Get upstream/downstream components' register state */
368 pcie_get_aspm_reg(parent
, &upreg
);
369 child
= list_entry(linkbus
->devices
.next
, struct pci_dev
, bus_list
);
370 pcie_get_aspm_reg(child
, &dwreg
);
375 * Note that we must not enable L0s in either direction on a
376 * given link unless components on both sides of the link each
379 if (dwreg
.support
& upreg
.support
& PCIE_LINK_STATE_L0S
)
380 link
->aspm_support
|= ASPM_STATE_L0S
;
381 if (dwreg
.enabled
& PCIE_LINK_STATE_L0S
)
382 link
->aspm_enabled
|= ASPM_STATE_L0S_UP
;
383 if (upreg
.enabled
& PCIE_LINK_STATE_L0S
)
384 link
->aspm_enabled
|= ASPM_STATE_L0S_DW
;
385 link
->latency_up
.l0s
= calc_l0s_latency(upreg
.latency_encoding_l0s
);
386 link
->latency_dw
.l0s
= calc_l0s_latency(dwreg
.latency_encoding_l0s
);
389 if (upreg
.support
& dwreg
.support
& PCIE_LINK_STATE_L1
)
390 link
->aspm_support
|= ASPM_STATE_L1
;
391 if (upreg
.enabled
& dwreg
.enabled
& PCIE_LINK_STATE_L1
)
392 link
->aspm_enabled
|= ASPM_STATE_L1
;
393 link
->latency_up
.l1
= calc_l1_latency(upreg
.latency_encoding_l1
);
394 link
->latency_dw
.l1
= calc_l1_latency(dwreg
.latency_encoding_l1
);
396 /* Save default state */
397 link
->aspm_default
= link
->aspm_enabled
;
399 /* Setup initial capable state. Will be updated later */
400 link
->aspm_capable
= link
->aspm_support
;
402 * If the downstream component has pci bridge function, don't
405 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
406 if (child
->pcie_type
== PCI_EXP_TYPE_PCI_BRIDGE
) {
407 link
->aspm_disable
= ASPM_STATE_ALL
;
412 /* Get and check endpoint acceptable latencies */
413 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
416 struct aspm_latency
*acceptable
=
417 &link
->acceptable
[PCI_FUNC(child
->devfn
)];
419 if (child
->pcie_type
!= PCI_EXP_TYPE_ENDPOINT
&&
420 child
->pcie_type
!= PCI_EXP_TYPE_LEG_END
)
423 pos
= pci_pcie_cap(child
);
424 pci_read_config_dword(child
, pos
+ PCI_EXP_DEVCAP
, ®32
);
425 /* Calculate endpoint L0s acceptable latency */
426 encoding
= (reg32
& PCI_EXP_DEVCAP_L0S
) >> 6;
427 acceptable
->l0s
= calc_l0s_acceptable(encoding
);
428 /* Calculate endpoint L1 acceptable latency */
429 encoding
= (reg32
& PCI_EXP_DEVCAP_L1
) >> 9;
430 acceptable
->l1
= calc_l1_acceptable(encoding
);
432 pcie_aspm_check_latency(child
);
436 static void pcie_config_aspm_dev(struct pci_dev
*pdev
, u32 val
)
439 int pos
= pci_pcie_cap(pdev
);
441 pci_read_config_word(pdev
, pos
+ PCI_EXP_LNKCTL
, ®16
);
444 pci_write_config_word(pdev
, pos
+ PCI_EXP_LNKCTL
, reg16
);
447 static void pcie_config_aspm_link(struct pcie_link_state
*link
, u32 state
)
449 u32 upstream
= 0, dwstream
= 0;
450 struct pci_dev
*child
, *parent
= link
->pdev
;
451 struct pci_bus
*linkbus
= parent
->subordinate
;
453 /* Nothing to do if the link is already in the requested state */
454 state
&= (link
->aspm_capable
& ~link
->aspm_disable
);
455 if (link
->aspm_enabled
== state
)
457 /* Convert ASPM state to upstream/downstream ASPM register state */
458 if (state
& ASPM_STATE_L0S_UP
)
459 dwstream
|= PCIE_LINK_STATE_L0S
;
460 if (state
& ASPM_STATE_L0S_DW
)
461 upstream
|= PCIE_LINK_STATE_L0S
;
462 if (state
& ASPM_STATE_L1
) {
463 upstream
|= PCIE_LINK_STATE_L1
;
464 dwstream
|= PCIE_LINK_STATE_L1
;
467 * Spec 2.0 suggests all functions should be configured the
468 * same setting for ASPM. Enabling ASPM L1 should be done in
469 * upstream component first and then downstream, and vice
470 * versa for disabling ASPM L1. Spec doesn't mention L0S.
472 if (state
& ASPM_STATE_L1
)
473 pcie_config_aspm_dev(parent
, upstream
);
474 list_for_each_entry(child
, &linkbus
->devices
, bus_list
)
475 pcie_config_aspm_dev(child
, dwstream
);
476 if (!(state
& ASPM_STATE_L1
))
477 pcie_config_aspm_dev(parent
, upstream
);
479 link
->aspm_enabled
= state
;
482 static void pcie_config_aspm_path(struct pcie_link_state
*link
)
485 pcie_config_aspm_link(link
, policy_to_aspm_state(link
));
490 static void free_link_state(struct pcie_link_state
*link
)
492 link
->pdev
->link_state
= NULL
;
496 static int pcie_aspm_sanity_check(struct pci_dev
*pdev
)
498 struct pci_dev
*child
;
502 if (aspm_clear_state
)
506 * Some functions in a slot might not all be PCIe functions,
507 * very strange. Disable ASPM for the whole slot
509 list_for_each_entry(child
, &pdev
->subordinate
->devices
, bus_list
) {
510 pos
= pci_pcie_cap(child
);
514 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
515 * RBER bit to determine if a function is 1.1 version device
517 pci_read_config_dword(child
, pos
+ PCI_EXP_DEVCAP
, ®32
);
518 if (!(reg32
& PCI_EXP_DEVCAP_RBER
) && !aspm_force
) {
519 dev_printk(KERN_INFO
, &child
->dev
, "disabling ASPM"
520 " on pre-1.1 PCIe device. You can enable it"
521 " with 'pcie_aspm=force'\n");
528 static struct pcie_link_state
*alloc_pcie_link_state(struct pci_dev
*pdev
)
530 struct pcie_link_state
*link
;
532 link
= kzalloc(sizeof(*link
), GFP_KERNEL
);
535 INIT_LIST_HEAD(&link
->sibling
);
536 INIT_LIST_HEAD(&link
->children
);
537 INIT_LIST_HEAD(&link
->link
);
539 if (pdev
->pcie_type
== PCI_EXP_TYPE_DOWNSTREAM
) {
540 struct pcie_link_state
*parent
;
541 parent
= pdev
->bus
->parent
->self
->link_state
;
546 link
->parent
= parent
;
547 list_add(&link
->link
, &parent
->children
);
549 /* Setup a pointer to the root port link */
553 link
->root
= link
->parent
->root
;
555 list_add(&link
->sibling
, &link_list
);
556 pdev
->link_state
= link
;
561 * pcie_aspm_init_link_state: Initiate PCI express link state.
562 * It is called after the pcie and its children devices are scaned.
563 * @pdev: the root port or switch downstream port
565 void pcie_aspm_init_link_state(struct pci_dev
*pdev
)
567 struct pcie_link_state
*link
;
568 int blacklist
= !!pcie_aspm_sanity_check(pdev
);
570 if (!pci_is_pcie(pdev
) || pdev
->link_state
)
572 if (pdev
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
&&
573 pdev
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
)
576 if (aspm_disabled
&& !aspm_clear_state
)
579 /* VIA has a strange chipset, root port is under a bridge */
580 if (pdev
->pcie_type
== PCI_EXP_TYPE_ROOT_PORT
&&
584 down_read(&pci_bus_sem
);
585 if (list_empty(&pdev
->subordinate
->devices
))
588 mutex_lock(&aspm_lock
);
589 link
= alloc_pcie_link_state(pdev
);
593 * Setup initial ASPM state. Note that we need to configure
594 * upstream links also because capable state of them can be
595 * update through pcie_aspm_cap_init().
597 pcie_aspm_cap_init(link
, blacklist
);
599 /* Setup initial Clock PM state */
600 pcie_clkpm_cap_init(link
, blacklist
);
603 * At this stage drivers haven't had an opportunity to change the
604 * link policy setting. Enabling ASPM on broken hardware can cripple
605 * it even before the driver has had a chance to disable ASPM, so
606 * default to a safe level right now. If we're enabling ASPM beyond
607 * the BIOS's expectation, we'll do so once pci_enable_device() is
610 if (aspm_policy
!= POLICY_POWERSAVE
) {
611 pcie_config_aspm_path(link
);
612 pcie_set_clkpm(link
, policy_to_clkpm_state(link
));
616 mutex_unlock(&aspm_lock
);
618 up_read(&pci_bus_sem
);
621 /* Recheck latencies and update aspm_capable for links under the root */
622 static void pcie_update_aspm_capable(struct pcie_link_state
*root
)
624 struct pcie_link_state
*link
;
625 BUG_ON(root
->parent
);
626 list_for_each_entry(link
, &link_list
, sibling
) {
627 if (link
->root
!= root
)
629 link
->aspm_capable
= link
->aspm_support
;
631 list_for_each_entry(link
, &link_list
, sibling
) {
632 struct pci_dev
*child
;
633 struct pci_bus
*linkbus
= link
->pdev
->subordinate
;
634 if (link
->root
!= root
)
636 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
637 if ((child
->pcie_type
!= PCI_EXP_TYPE_ENDPOINT
) &&
638 (child
->pcie_type
!= PCI_EXP_TYPE_LEG_END
))
640 pcie_aspm_check_latency(child
);
645 /* @pdev: the endpoint device */
646 void pcie_aspm_exit_link_state(struct pci_dev
*pdev
)
648 struct pci_dev
*parent
= pdev
->bus
->self
;
649 struct pcie_link_state
*link
, *root
, *parent_link
;
651 if ((aspm_disabled
&& !aspm_clear_state
) || !pci_is_pcie(pdev
) ||
652 !parent
|| !parent
->link_state
)
654 if ((parent
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
) &&
655 (parent
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
))
658 down_read(&pci_bus_sem
);
659 mutex_lock(&aspm_lock
);
661 * All PCIe functions are in one slot, remove one function will remove
662 * the whole slot, so just wait until we are the last function left.
664 if (!list_is_last(&pdev
->bus_list
, &parent
->subordinate
->devices
))
667 link
= parent
->link_state
;
669 parent_link
= link
->parent
;
671 /* All functions are removed, so just disable ASPM for the link */
672 pcie_config_aspm_link(link
, 0);
673 list_del(&link
->sibling
);
674 list_del(&link
->link
);
675 /* Clock PM is for endpoint device */
676 free_link_state(link
);
678 /* Recheck latencies and configure upstream links */
680 pcie_update_aspm_capable(root
);
681 pcie_config_aspm_path(parent_link
);
684 mutex_unlock(&aspm_lock
);
685 up_read(&pci_bus_sem
);
688 /* @pdev: the root port or switch downstream port */
689 void pcie_aspm_pm_state_change(struct pci_dev
*pdev
)
691 struct pcie_link_state
*link
= pdev
->link_state
;
693 if (aspm_disabled
|| !pci_is_pcie(pdev
) || !link
)
695 if ((pdev
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
) &&
696 (pdev
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
))
699 * Devices changed PM state, we should recheck if latency
700 * meets all functions' requirement
702 down_read(&pci_bus_sem
);
703 mutex_lock(&aspm_lock
);
704 pcie_update_aspm_capable(link
->root
);
705 pcie_config_aspm_path(link
);
706 mutex_unlock(&aspm_lock
);
707 up_read(&pci_bus_sem
);
711 * pci_disable_link_state - disable pci device's link state, so the link will
712 * never enter specific states
714 void pci_disable_link_state(struct pci_dev
*pdev
, int state
)
716 struct pci_dev
*parent
= pdev
->bus
->self
;
717 struct pcie_link_state
*link
;
719 if (aspm_disabled
|| !pci_is_pcie(pdev
))
721 if (pdev
->pcie_type
== PCI_EXP_TYPE_ROOT_PORT
||
722 pdev
->pcie_type
== PCI_EXP_TYPE_DOWNSTREAM
)
724 if (!parent
|| !parent
->link_state
)
727 down_read(&pci_bus_sem
);
728 mutex_lock(&aspm_lock
);
729 link
= parent
->link_state
;
730 if (state
& PCIE_LINK_STATE_L0S
)
731 link
->aspm_disable
|= ASPM_STATE_L0S
;
732 if (state
& PCIE_LINK_STATE_L1
)
733 link
->aspm_disable
|= ASPM_STATE_L1
;
734 pcie_config_aspm_link(link
, policy_to_aspm_state(link
));
736 if (state
& PCIE_LINK_STATE_CLKPM
) {
737 link
->clkpm_capable
= 0;
738 pcie_set_clkpm(link
, 0);
740 mutex_unlock(&aspm_lock
);
741 up_read(&pci_bus_sem
);
743 EXPORT_SYMBOL(pci_disable_link_state
);
745 static int pcie_aspm_set_policy(const char *val
, struct kernel_param
*kp
)
748 struct pcie_link_state
*link
;
750 for (i
= 0; i
< ARRAY_SIZE(policy_str
); i
++)
751 if (!strncmp(val
, policy_str
[i
], strlen(policy_str
[i
])))
753 if (i
>= ARRAY_SIZE(policy_str
))
755 if (i
== aspm_policy
)
758 down_read(&pci_bus_sem
);
759 mutex_lock(&aspm_lock
);
761 list_for_each_entry(link
, &link_list
, sibling
) {
762 pcie_config_aspm_link(link
, policy_to_aspm_state(link
));
763 pcie_set_clkpm(link
, policy_to_clkpm_state(link
));
765 mutex_unlock(&aspm_lock
);
766 up_read(&pci_bus_sem
);
770 static int pcie_aspm_get_policy(char *buffer
, struct kernel_param
*kp
)
773 for (i
= 0; i
< ARRAY_SIZE(policy_str
); i
++)
774 if (i
== aspm_policy
)
775 cnt
+= sprintf(buffer
+ cnt
, "[%s] ", policy_str
[i
]);
777 cnt
+= sprintf(buffer
+ cnt
, "%s ", policy_str
[i
]);
781 module_param_call(policy
, pcie_aspm_set_policy
, pcie_aspm_get_policy
,
784 #ifdef CONFIG_PCIEASPM_DEBUG
785 static ssize_t
link_state_show(struct device
*dev
,
786 struct device_attribute
*attr
,
789 struct pci_dev
*pci_device
= to_pci_dev(dev
);
790 struct pcie_link_state
*link_state
= pci_device
->link_state
;
792 return sprintf(buf
, "%d\n", link_state
->aspm_enabled
);
795 static ssize_t
link_state_store(struct device
*dev
,
796 struct device_attribute
*attr
,
800 struct pci_dev
*pdev
= to_pci_dev(dev
);
801 struct pcie_link_state
*link
, *root
= pdev
->link_state
->root
;
802 u32 val
= buf
[0] - '0', state
= 0;
804 if (n
< 1 || val
> 3)
807 /* Convert requested state to ASPM state */
808 if (val
& PCIE_LINK_STATE_L0S
)
809 state
|= ASPM_STATE_L0S
;
810 if (val
& PCIE_LINK_STATE_L1
)
811 state
|= ASPM_STATE_L1
;
813 down_read(&pci_bus_sem
);
814 mutex_lock(&aspm_lock
);
815 list_for_each_entry(link
, &link_list
, sibling
) {
816 if (link
->root
!= root
)
818 pcie_config_aspm_link(link
, state
);
820 mutex_unlock(&aspm_lock
);
821 up_read(&pci_bus_sem
);
825 static ssize_t
clk_ctl_show(struct device
*dev
,
826 struct device_attribute
*attr
,
829 struct pci_dev
*pci_device
= to_pci_dev(dev
);
830 struct pcie_link_state
*link_state
= pci_device
->link_state
;
832 return sprintf(buf
, "%d\n", link_state
->clkpm_enabled
);
835 static ssize_t
clk_ctl_store(struct device
*dev
,
836 struct device_attribute
*attr
,
840 struct pci_dev
*pdev
= to_pci_dev(dev
);
847 down_read(&pci_bus_sem
);
848 mutex_lock(&aspm_lock
);
849 pcie_set_clkpm_nocheck(pdev
->link_state
, !!state
);
850 mutex_unlock(&aspm_lock
);
851 up_read(&pci_bus_sem
);
856 static DEVICE_ATTR(link_state
, 0644, link_state_show
, link_state_store
);
857 static DEVICE_ATTR(clk_ctl
, 0644, clk_ctl_show
, clk_ctl_store
);
859 static char power_group
[] = "power";
860 void pcie_aspm_create_sysfs_dev_files(struct pci_dev
*pdev
)
862 struct pcie_link_state
*link_state
= pdev
->link_state
;
864 if (!pci_is_pcie(pdev
) ||
865 (pdev
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
&&
866 pdev
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
) || !link_state
)
869 if (link_state
->aspm_support
)
870 sysfs_add_file_to_group(&pdev
->dev
.kobj
,
871 &dev_attr_link_state
.attr
, power_group
);
872 if (link_state
->clkpm_capable
)
873 sysfs_add_file_to_group(&pdev
->dev
.kobj
,
874 &dev_attr_clk_ctl
.attr
, power_group
);
877 void pcie_aspm_remove_sysfs_dev_files(struct pci_dev
*pdev
)
879 struct pcie_link_state
*link_state
= pdev
->link_state
;
881 if (!pci_is_pcie(pdev
) ||
882 (pdev
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
&&
883 pdev
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
) || !link_state
)
886 if (link_state
->aspm_support
)
887 sysfs_remove_file_from_group(&pdev
->dev
.kobj
,
888 &dev_attr_link_state
.attr
, power_group
);
889 if (link_state
->clkpm_capable
)
890 sysfs_remove_file_from_group(&pdev
->dev
.kobj
,
891 &dev_attr_clk_ctl
.attr
, power_group
);
895 static int __init
pcie_aspm_disable(char *str
)
897 if (!strcmp(str
, "off")) {
899 printk(KERN_INFO
"PCIe ASPM is disabled\n");
900 } else if (!strcmp(str
, "force")) {
902 printk(KERN_INFO
"PCIe ASPM is forcedly enabled\n");
907 __setup("pcie_aspm=", pcie_aspm_disable
);
909 void pcie_clear_aspm(void)
912 aspm_clear_state
= 1;
915 void pcie_no_aspm(void)
922 * pcie_aspm_enabled - is PCIe ASPM enabled?
924 * Returns true if ASPM has not been disabled by the command-line option
927 int pcie_aspm_enabled(void)
929 return !aspm_disabled
;
931 EXPORT_SYMBOL(pcie_aspm_enabled
);
This page took 0.068301 seconds and 5 git commands to generate.