2 * Generic OPP Interface
4 * Copyright (C) 2009-2010 Texas Instruments Incorporated.
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
14 #include <linux/cpu.h>
15 #include <linux/kernel.h>
16 #include <linux/errno.h>
17 #include <linux/err.h>
18 #include <linux/slab.h>
19 #include <linux/device.h>
20 #include <linux/list.h>
21 #include <linux/rculist.h>
22 #include <linux/rcupdate.h>
23 #include <linux/pm_opp.h>
25 #include <linux/export.h>
28 * Internal data structure organization with the OPP layer library is as
31 * |- device 1 (represents voltage domain 1)
32 * | |- opp 1 (availability, freq, voltage)
36 * |- device 2 (represents the next voltage domain)
38 * `- device m (represents mth voltage domain)
39 * device 1, 2.. are represented by dev_opp structure while each opp
40 * is represented by the opp structure.
44 * struct dev_pm_opp - Generic OPP description structure
45 * @node: opp list node. The nodes are maintained throughout the lifetime
46 * of boot. It is expected only an optimal set of OPPs are
47 * added to the library by the SoC framework.
48 * RCU usage: opp list is traversed with RCU locks. node
49 * modification is possible realtime, hence the modifications
50 * are protected by the dev_opp_list_lock for integrity.
51 * IMPORTANT: the opp nodes should be maintained in increasing
53 * @dynamic: not-created from static DT entries.
54 * @available: true/false - marks if this OPP as available or not
55 * @turbo: true if turbo (boost) OPP
56 * @rate: Frequency in hertz
57 * @u_volt: Target voltage in microvolts corresponding to this OPP
58 * @u_volt_min: Minimum voltage in microvolts corresponding to this OPP
59 * @u_volt_max: Maximum voltage in microvolts corresponding to this OPP
60 * @u_amp: Maximum current drawn by the device in microamperes
61 * @clock_latency_ns: Latency (in nanoseconds) of switching to this OPP's
62 * frequency from any other OPP's frequency.
63 * @dev_opp: points back to the device_opp struct this opp belongs to
64 * @rcu_head: RCU callback head used for deferred freeing
65 * @np: OPP's device node.
67 * This structure stores the OPP information for a given device.
70 struct list_head node
;
78 unsigned long u_volt_min
;
79 unsigned long u_volt_max
;
81 unsigned long clock_latency_ns
;
83 struct device_opp
*dev_opp
;
84 struct rcu_head rcu_head
;
86 struct device_node
*np
;
90 * struct device_list_opp - devices managed by 'struct device_opp'
92 * @dev: device to which the struct object belongs
93 * @rcu_head: RCU callback head used for deferred freeing
95 * This is an internal data structure maintaining the list of devices that are
96 * managed by 'struct device_opp'.
98 struct device_list_opp
{
99 struct list_head node
;
100 const struct device
*dev
;
101 struct rcu_head rcu_head
;
105 * struct device_opp - Device opp structure
106 * @node: list node - contains the devices with OPPs that
107 * have been registered. Nodes once added are not modified in this
109 * RCU usage: nodes are not modified in the list of device_opp,
110 * however addition is possible and is secured by dev_opp_list_lock
111 * @srcu_head: notifier head to notify the OPP availability changes.
112 * @rcu_head: RCU callback head used for deferred freeing
113 * @dev_list: list of devices that share these OPPs
114 * @opp_list: list of opps
115 * @np: struct device_node pointer for opp's DT node.
116 * @shared_opp: OPP is shared between multiple devices.
118 * This is an internal data structure maintaining the link to opps attached to
119 * a device. This structure is not meant to be shared to users as it is
120 * meant for book keeping and private to OPP library.
122 * Because the opp structures can be used from both rcu and srcu readers, we
123 * need to wait for the grace period of both of them before freeing any
124 * resources. And so we have used kfree_rcu() from within call_srcu() handlers.
127 struct list_head node
;
129 struct srcu_notifier_head srcu_head
;
130 struct rcu_head rcu_head
;
131 struct list_head dev_list
;
132 struct list_head opp_list
;
134 struct device_node
*np
;
135 unsigned long clock_latency_ns_max
;
137 struct dev_pm_opp
*suspend_opp
;
141 * The root of the list of all devices. All device_opp structures branch off
142 * from here, with each device_opp containing the list of opp it supports in
143 * various states of availability.
145 static LIST_HEAD(dev_opp_list
);
146 /* Lock to allow exclusive modification to the device and opp lists */
147 static DEFINE_MUTEX(dev_opp_list_lock
);
149 #define opp_rcu_lockdep_assert() \
151 RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
152 !lockdep_is_held(&dev_opp_list_lock), \
153 "Missing rcu_read_lock() or " \
154 "dev_opp_list_lock protection"); \
157 static struct device_list_opp
*_find_list_dev(const struct device
*dev
,
158 struct device_opp
*dev_opp
)
160 struct device_list_opp
*list_dev
;
162 list_for_each_entry(list_dev
, &dev_opp
->dev_list
, node
)
163 if (list_dev
->dev
== dev
)
169 static struct device_opp
*_managed_opp(const struct device_node
*np
)
171 struct device_opp
*dev_opp
;
173 list_for_each_entry_rcu(dev_opp
, &dev_opp_list
, node
) {
174 if (dev_opp
->np
== np
) {
176 * Multiple devices can point to the same OPP table and
177 * so will have same node-pointer, np.
179 * But the OPPs will be considered as shared only if the
180 * OPP table contains a "opp-shared" property.
182 return dev_opp
->shared_opp
? dev_opp
: NULL
;
190 * _find_device_opp() - find device_opp struct using device pointer
191 * @dev: device pointer used to lookup device OPPs
193 * Search list of device OPPs for one containing matching device. Does a RCU
194 * reader operation to grab the pointer needed.
196 * Return: pointer to 'struct device_opp' if found, otherwise -ENODEV or
197 * -EINVAL based on type of error.
199 * Locking: This function must be called under rcu_read_lock(). device_opp
200 * is a RCU protected pointer. This means that device_opp is valid as long
201 * as we are under RCU lock.
203 static struct device_opp
*_find_device_opp(struct device
*dev
)
205 struct device_opp
*dev_opp
;
207 if (IS_ERR_OR_NULL(dev
)) {
208 pr_err("%s: Invalid parameters\n", __func__
);
209 return ERR_PTR(-EINVAL
);
212 list_for_each_entry_rcu(dev_opp
, &dev_opp_list
, node
)
213 if (_find_list_dev(dev
, dev_opp
))
216 return ERR_PTR(-ENODEV
);
220 * dev_pm_opp_get_voltage() - Gets the voltage corresponding to an available opp
221 * @opp: opp for which voltage has to be returned for
223 * Return: voltage in micro volt corresponding to the opp, else
226 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
227 * protected pointer. This means that opp which could have been fetched by
228 * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are
229 * under RCU lock. The pointer returned by the opp_find_freq family must be
230 * used in the same section as the usage of this function with the pointer
231 * prior to unlocking with rcu_read_unlock() to maintain the integrity of the
234 unsigned long dev_pm_opp_get_voltage(struct dev_pm_opp
*opp
)
236 struct dev_pm_opp
*tmp_opp
;
239 opp_rcu_lockdep_assert();
241 tmp_opp
= rcu_dereference(opp
);
242 if (IS_ERR_OR_NULL(tmp_opp
) || !tmp_opp
->available
)
243 pr_err("%s: Invalid parameters\n", __func__
);
249 EXPORT_SYMBOL_GPL(dev_pm_opp_get_voltage
);
252 * dev_pm_opp_get_freq() - Gets the frequency corresponding to an available opp
253 * @opp: opp for which frequency has to be returned for
255 * Return: frequency in hertz corresponding to the opp, else
258 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
259 * protected pointer. This means that opp which could have been fetched by
260 * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are
261 * under RCU lock. The pointer returned by the opp_find_freq family must be
262 * used in the same section as the usage of this function with the pointer
263 * prior to unlocking with rcu_read_unlock() to maintain the integrity of the
266 unsigned long dev_pm_opp_get_freq(struct dev_pm_opp
*opp
)
268 struct dev_pm_opp
*tmp_opp
;
271 opp_rcu_lockdep_assert();
273 tmp_opp
= rcu_dereference(opp
);
274 if (IS_ERR_OR_NULL(tmp_opp
) || !tmp_opp
->available
)
275 pr_err("%s: Invalid parameters\n", __func__
);
281 EXPORT_SYMBOL_GPL(dev_pm_opp_get_freq
);
284 * dev_pm_opp_is_turbo() - Returns if opp is turbo OPP or not
285 * @opp: opp for which turbo mode is being verified
287 * Turbo OPPs are not for normal use, and can be enabled (under certain
288 * conditions) for short duration of times to finish high throughput work
289 * quickly. Running on them for longer times may overheat the chip.
291 * Return: true if opp is turbo opp, else false.
293 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
294 * protected pointer. This means that opp which could have been fetched by
295 * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are
296 * under RCU lock. The pointer returned by the opp_find_freq family must be
297 * used in the same section as the usage of this function with the pointer
298 * prior to unlocking with rcu_read_unlock() to maintain the integrity of the
301 bool dev_pm_opp_is_turbo(struct dev_pm_opp
*opp
)
303 struct dev_pm_opp
*tmp_opp
;
305 opp_rcu_lockdep_assert();
307 tmp_opp
= rcu_dereference(opp
);
308 if (IS_ERR_OR_NULL(tmp_opp
) || !tmp_opp
->available
) {
309 pr_err("%s: Invalid parameters\n", __func__
);
313 return tmp_opp
->turbo
;
315 EXPORT_SYMBOL_GPL(dev_pm_opp_is_turbo
);
318 * dev_pm_opp_get_max_clock_latency() - Get max clock latency in nanoseconds
319 * @dev: device for which we do this operation
321 * Return: This function returns the max clock latency in nanoseconds.
323 * Locking: This function takes rcu_read_lock().
325 unsigned long dev_pm_opp_get_max_clock_latency(struct device
*dev
)
327 struct device_opp
*dev_opp
;
328 unsigned long clock_latency_ns
;
332 dev_opp
= _find_device_opp(dev
);
334 clock_latency_ns
= 0;
336 clock_latency_ns
= dev_opp
->clock_latency_ns_max
;
339 return clock_latency_ns
;
341 EXPORT_SYMBOL_GPL(dev_pm_opp_get_max_clock_latency
);
344 * dev_pm_opp_get_suspend_opp() - Get suspend opp
345 * @dev: device for which we do this operation
347 * Return: This function returns pointer to the suspend opp if it is
348 * defined and available, otherwise it returns NULL.
350 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
351 * protected pointer. The reason for the same is that the opp pointer which is
352 * returned will remain valid for use with opp_get_{voltage, freq} only while
353 * under the locked area. The pointer returned must be used prior to unlocking
354 * with rcu_read_unlock() to maintain the integrity of the pointer.
356 struct dev_pm_opp
*dev_pm_opp_get_suspend_opp(struct device
*dev
)
358 struct device_opp
*dev_opp
;
360 opp_rcu_lockdep_assert();
362 dev_opp
= _find_device_opp(dev
);
363 if (IS_ERR(dev_opp
) || !dev_opp
->suspend_opp
||
364 !dev_opp
->suspend_opp
->available
)
367 return dev_opp
->suspend_opp
;
369 EXPORT_SYMBOL_GPL(dev_pm_opp_get_suspend_opp
);
372 * dev_pm_opp_get_opp_count() - Get number of opps available in the opp list
373 * @dev: device for which we do this operation
375 * Return: This function returns the number of available opps if there are any,
376 * else returns 0 if none or the corresponding error value.
378 * Locking: This function takes rcu_read_lock().
380 int dev_pm_opp_get_opp_count(struct device
*dev
)
382 struct device_opp
*dev_opp
;
383 struct dev_pm_opp
*temp_opp
;
388 dev_opp
= _find_device_opp(dev
);
389 if (IS_ERR(dev_opp
)) {
390 count
= PTR_ERR(dev_opp
);
391 dev_err(dev
, "%s: device OPP not found (%d)\n",
396 list_for_each_entry_rcu(temp_opp
, &dev_opp
->opp_list
, node
) {
397 if (temp_opp
->available
)
405 EXPORT_SYMBOL_GPL(dev_pm_opp_get_opp_count
);
408 * dev_pm_opp_find_freq_exact() - search for an exact frequency
409 * @dev: device for which we do this operation
410 * @freq: frequency to search for
411 * @available: true/false - match for available opp
413 * Return: Searches for exact match in the opp list and returns pointer to the
414 * matching opp if found, else returns ERR_PTR in case of error and should
415 * be handled using IS_ERR. Error return values can be:
416 * EINVAL: for bad pointer
417 * ERANGE: no match found for search
418 * ENODEV: if device not found in list of registered devices
420 * Note: available is a modifier for the search. if available=true, then the
421 * match is for exact matching frequency and is available in the stored OPP
422 * table. if false, the match is for exact frequency which is not available.
424 * This provides a mechanism to enable an opp which is not available currently
425 * or the opposite as well.
427 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
428 * protected pointer. The reason for the same is that the opp pointer which is
429 * returned will remain valid for use with opp_get_{voltage, freq} only while
430 * under the locked area. The pointer returned must be used prior to unlocking
431 * with rcu_read_unlock() to maintain the integrity of the pointer.
433 struct dev_pm_opp
*dev_pm_opp_find_freq_exact(struct device
*dev
,
437 struct device_opp
*dev_opp
;
438 struct dev_pm_opp
*temp_opp
, *opp
= ERR_PTR(-ERANGE
);
440 opp_rcu_lockdep_assert();
442 dev_opp
= _find_device_opp(dev
);
443 if (IS_ERR(dev_opp
)) {
444 int r
= PTR_ERR(dev_opp
);
445 dev_err(dev
, "%s: device OPP not found (%d)\n", __func__
, r
);
449 list_for_each_entry_rcu(temp_opp
, &dev_opp
->opp_list
, node
) {
450 if (temp_opp
->available
== available
&&
451 temp_opp
->rate
== freq
) {
459 EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_exact
);
462 * dev_pm_opp_find_freq_ceil() - Search for an rounded ceil freq
463 * @dev: device for which we do this operation
464 * @freq: Start frequency
466 * Search for the matching ceil *available* OPP from a starting freq
469 * Return: matching *opp and refreshes *freq accordingly, else returns
470 * ERR_PTR in case of error and should be handled using IS_ERR. Error return
472 * EINVAL: for bad pointer
473 * ERANGE: no match found for search
474 * ENODEV: if device not found in list of registered devices
476 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
477 * protected pointer. The reason for the same is that the opp pointer which is
478 * returned will remain valid for use with opp_get_{voltage, freq} only while
479 * under the locked area. The pointer returned must be used prior to unlocking
480 * with rcu_read_unlock() to maintain the integrity of the pointer.
482 struct dev_pm_opp
*dev_pm_opp_find_freq_ceil(struct device
*dev
,
485 struct device_opp
*dev_opp
;
486 struct dev_pm_opp
*temp_opp
, *opp
= ERR_PTR(-ERANGE
);
488 opp_rcu_lockdep_assert();
491 dev_err(dev
, "%s: Invalid argument freq=%p\n", __func__
, freq
);
492 return ERR_PTR(-EINVAL
);
495 dev_opp
= _find_device_opp(dev
);
497 return ERR_CAST(dev_opp
);
499 list_for_each_entry_rcu(temp_opp
, &dev_opp
->opp_list
, node
) {
500 if (temp_opp
->available
&& temp_opp
->rate
>= *freq
) {
509 EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_ceil
);
512 * dev_pm_opp_find_freq_floor() - Search for a rounded floor freq
513 * @dev: device for which we do this operation
514 * @freq: Start frequency
516 * Search for the matching floor *available* OPP from a starting freq
519 * Return: matching *opp and refreshes *freq accordingly, else returns
520 * ERR_PTR in case of error and should be handled using IS_ERR. Error return
522 * EINVAL: for bad pointer
523 * ERANGE: no match found for search
524 * ENODEV: if device not found in list of registered devices
526 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
527 * protected pointer. The reason for the same is that the opp pointer which is
528 * returned will remain valid for use with opp_get_{voltage, freq} only while
529 * under the locked area. The pointer returned must be used prior to unlocking
530 * with rcu_read_unlock() to maintain the integrity of the pointer.
532 struct dev_pm_opp
*dev_pm_opp_find_freq_floor(struct device
*dev
,
535 struct device_opp
*dev_opp
;
536 struct dev_pm_opp
*temp_opp
, *opp
= ERR_PTR(-ERANGE
);
538 opp_rcu_lockdep_assert();
541 dev_err(dev
, "%s: Invalid argument freq=%p\n", __func__
, freq
);
542 return ERR_PTR(-EINVAL
);
545 dev_opp
= _find_device_opp(dev
);
547 return ERR_CAST(dev_opp
);
549 list_for_each_entry_rcu(temp_opp
, &dev_opp
->opp_list
, node
) {
550 if (temp_opp
->available
) {
551 /* go to the next node, before choosing prev */
552 if (temp_opp
->rate
> *freq
)
563 EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_floor
);
565 /* List-dev Helpers */
566 static void _kfree_list_dev_rcu(struct rcu_head
*head
)
568 struct device_list_opp
*list_dev
;
570 list_dev
= container_of(head
, struct device_list_opp
, rcu_head
);
571 kfree_rcu(list_dev
, rcu_head
);
574 static void _remove_list_dev(struct device_list_opp
*list_dev
,
575 struct device_opp
*dev_opp
)
577 list_del(&list_dev
->node
);
578 call_srcu(&dev_opp
->srcu_head
.srcu
, &list_dev
->rcu_head
,
579 _kfree_list_dev_rcu
);
582 static struct device_list_opp
*_add_list_dev(const struct device
*dev
,
583 struct device_opp
*dev_opp
)
585 struct device_list_opp
*list_dev
;
587 list_dev
= kzalloc(sizeof(*list_dev
), GFP_KERNEL
);
591 /* Initialize list-dev */
593 list_add_rcu(&list_dev
->node
, &dev_opp
->dev_list
);
599 * _add_device_opp() - Find device OPP table or allocate a new one
600 * @dev: device for which we do this operation
602 * It tries to find an existing table first, if it couldn't find one, it
603 * allocates a new OPP table and returns that.
605 * Return: valid device_opp pointer if success, else NULL.
607 static struct device_opp
*_add_device_opp(struct device
*dev
)
609 struct device_opp
*dev_opp
;
610 struct device_list_opp
*list_dev
;
612 /* Check for existing list for 'dev' first */
613 dev_opp
= _find_device_opp(dev
);
614 if (!IS_ERR(dev_opp
))
618 * Allocate a new device OPP table. In the infrequent case where a new
619 * device is needed to be added, we pay this penalty.
621 dev_opp
= kzalloc(sizeof(*dev_opp
), GFP_KERNEL
);
625 INIT_LIST_HEAD(&dev_opp
->dev_list
);
627 list_dev
= _add_list_dev(dev
, dev_opp
);
633 srcu_init_notifier_head(&dev_opp
->srcu_head
);
634 INIT_LIST_HEAD(&dev_opp
->opp_list
);
636 /* Secure the device list modification */
637 list_add_rcu(&dev_opp
->node
, &dev_opp_list
);
642 * _kfree_device_rcu() - Free device_opp RCU handler
645 static void _kfree_device_rcu(struct rcu_head
*head
)
647 struct device_opp
*device_opp
= container_of(head
, struct device_opp
, rcu_head
);
649 kfree_rcu(device_opp
, rcu_head
);
653 * _remove_device_opp() - Removes a device OPP table
654 * @dev_opp: device OPP table to be removed.
656 * Removes/frees device OPP table it it doesn't contain any OPPs.
658 static void _remove_device_opp(struct device_opp
*dev_opp
)
660 struct device_list_opp
*list_dev
;
662 if (!list_empty(&dev_opp
->opp_list
))
665 list_dev
= list_first_entry(&dev_opp
->dev_list
, struct device_list_opp
,
668 _remove_list_dev(list_dev
, dev_opp
);
670 /* dev_list must be empty now */
671 WARN_ON(!list_empty(&dev_opp
->dev_list
));
673 list_del_rcu(&dev_opp
->node
);
674 call_srcu(&dev_opp
->srcu_head
.srcu
, &dev_opp
->rcu_head
,
679 * _kfree_opp_rcu() - Free OPP RCU handler
682 static void _kfree_opp_rcu(struct rcu_head
*head
)
684 struct dev_pm_opp
*opp
= container_of(head
, struct dev_pm_opp
, rcu_head
);
686 kfree_rcu(opp
, rcu_head
);
690 * _opp_remove() - Remove an OPP from a table definition
691 * @dev_opp: points back to the device_opp struct this opp belongs to
692 * @opp: pointer to the OPP to remove
693 * @notify: OPP_EVENT_REMOVE notification should be sent or not
695 * This function removes an opp definition from the opp list.
697 * Locking: The internal device_opp and opp structures are RCU protected.
698 * It is assumed that the caller holds required mutex for an RCU updater
701 static void _opp_remove(struct device_opp
*dev_opp
,
702 struct dev_pm_opp
*opp
, bool notify
)
705 * Notify the changes in the availability of the operable
706 * frequency/voltage list.
709 srcu_notifier_call_chain(&dev_opp
->srcu_head
, OPP_EVENT_REMOVE
, opp
);
710 list_del_rcu(&opp
->node
);
711 call_srcu(&dev_opp
->srcu_head
.srcu
, &opp
->rcu_head
, _kfree_opp_rcu
);
713 _remove_device_opp(dev_opp
);
717 * dev_pm_opp_remove() - Remove an OPP from OPP list
718 * @dev: device for which we do this operation
719 * @freq: OPP to remove with matching 'freq'
721 * This function removes an opp from the opp list.
723 * Locking: The internal device_opp and opp structures are RCU protected.
724 * Hence this function internally uses RCU updater strategy with mutex locks
725 * to keep the integrity of the internal data structures. Callers should ensure
726 * that this function is *NOT* called under RCU protection or in contexts where
727 * mutex cannot be locked.
729 void dev_pm_opp_remove(struct device
*dev
, unsigned long freq
)
731 struct dev_pm_opp
*opp
;
732 struct device_opp
*dev_opp
;
735 /* Hold our list modification lock here */
736 mutex_lock(&dev_opp_list_lock
);
738 dev_opp
= _find_device_opp(dev
);
742 list_for_each_entry(opp
, &dev_opp
->opp_list
, node
) {
743 if (opp
->rate
== freq
) {
750 dev_warn(dev
, "%s: Couldn't find OPP with freq: %lu\n",
755 _opp_remove(dev_opp
, opp
, true);
757 mutex_unlock(&dev_opp_list_lock
);
759 EXPORT_SYMBOL_GPL(dev_pm_opp_remove
);
761 static struct dev_pm_opp
*_allocate_opp(struct device
*dev
,
762 struct device_opp
**dev_opp
)
764 struct dev_pm_opp
*opp
;
766 /* allocate new OPP node */
767 opp
= kzalloc(sizeof(*opp
), GFP_KERNEL
);
771 INIT_LIST_HEAD(&opp
->node
);
773 *dev_opp
= _add_device_opp(dev
);
782 static int _opp_add(struct device
*dev
, struct dev_pm_opp
*new_opp
,
783 struct device_opp
*dev_opp
)
785 struct dev_pm_opp
*opp
;
786 struct list_head
*head
= &dev_opp
->opp_list
;
789 * Insert new OPP in order of increasing frequency and discard if
792 * Need to use &dev_opp->opp_list in the condition part of the 'for'
793 * loop, don't replace it with head otherwise it will become an infinite
796 list_for_each_entry_rcu(opp
, &dev_opp
->opp_list
, node
) {
797 if (new_opp
->rate
> opp
->rate
) {
802 if (new_opp
->rate
< opp
->rate
)
806 dev_warn(dev
, "%s: duplicate OPPs detected. Existing: freq: %lu, volt: %lu, enabled: %d. New: freq: %lu, volt: %lu, enabled: %d\n",
807 __func__
, opp
->rate
, opp
->u_volt
, opp
->available
,
808 new_opp
->rate
, new_opp
->u_volt
, new_opp
->available
);
810 return opp
->available
&& new_opp
->u_volt
== opp
->u_volt
?
814 new_opp
->dev_opp
= dev_opp
;
815 list_add_rcu(&new_opp
->node
, head
);
821 * _opp_add_dynamic() - Allocate a dynamic OPP.
822 * @dev: device for which we do this operation
823 * @freq: Frequency in Hz for this OPP
824 * @u_volt: Voltage in uVolts for this OPP
825 * @dynamic: Dynamically added OPPs.
827 * This function adds an opp definition to the opp list and returns status.
828 * The opp is made available by default and it can be controlled using
829 * dev_pm_opp_enable/disable functions and may be removed by dev_pm_opp_remove.
831 * NOTE: "dynamic" parameter impacts OPPs added by the of_init_opp_table and
832 * freed by of_free_opp_table.
834 * Locking: The internal device_opp and opp structures are RCU protected.
835 * Hence this function internally uses RCU updater strategy with mutex locks
836 * to keep the integrity of the internal data structures. Callers should ensure
837 * that this function is *NOT* called under RCU protection or in contexts where
838 * mutex cannot be locked.
842 * Duplicate OPPs (both freq and volt are same) and opp->available
843 * -EEXIST Freq are same and volt are different OR
844 * Duplicate OPPs (both freq and volt are same) and !opp->available
845 * -ENOMEM Memory allocation failure
847 static int _opp_add_dynamic(struct device
*dev
, unsigned long freq
,
848 long u_volt
, bool dynamic
)
850 struct device_opp
*dev_opp
;
851 struct dev_pm_opp
*new_opp
;
854 /* Hold our list modification lock here */
855 mutex_lock(&dev_opp_list_lock
);
857 new_opp
= _allocate_opp(dev
, &dev_opp
);
863 /* populate the opp table */
864 new_opp
->rate
= freq
;
865 new_opp
->u_volt
= u_volt
;
866 new_opp
->available
= true;
867 new_opp
->dynamic
= dynamic
;
869 ret
= _opp_add(dev
, new_opp
, dev_opp
);
873 mutex_unlock(&dev_opp_list_lock
);
876 * Notify the changes in the availability of the operable
877 * frequency/voltage list.
879 srcu_notifier_call_chain(&dev_opp
->srcu_head
, OPP_EVENT_ADD
, new_opp
);
883 _opp_remove(dev_opp
, new_opp
, false);
885 mutex_unlock(&dev_opp_list_lock
);
889 /* TODO: Support multiple regulators */
890 static int opp_get_microvolt(struct dev_pm_opp
*opp
, struct device
*dev
)
892 u32 microvolt
[3] = {0};
895 count
= of_property_count_u32_elems(opp
->np
, "opp-microvolt");
899 /* There can be one or three elements here */
900 if (count
!= 1 && count
!= 3) {
901 dev_err(dev
, "%s: Invalid number of elements in opp-microvolt property (%d)\n",
906 ret
= of_property_read_u32_array(opp
->np
, "opp-microvolt", microvolt
,
909 dev_err(dev
, "%s: error parsing opp-microvolt: %d\n", __func__
,
914 opp
->u_volt
= microvolt
[0];
915 opp
->u_volt_min
= microvolt
[1];
916 opp
->u_volt_max
= microvolt
[2];
922 * _opp_add_static_v2() - Allocate static OPPs (As per 'v2' DT bindings)
923 * @dev: device for which we do this operation
926 * This function adds an opp definition to the opp list and returns status. The
927 * opp can be controlled using dev_pm_opp_enable/disable functions and may be
928 * removed by dev_pm_opp_remove.
930 * Locking: The internal device_opp and opp structures are RCU protected.
931 * Hence this function internally uses RCU updater strategy with mutex locks
932 * to keep the integrity of the internal data structures. Callers should ensure
933 * that this function is *NOT* called under RCU protection or in contexts where
934 * mutex cannot be locked.
938 * Duplicate OPPs (both freq and volt are same) and opp->available
939 * -EEXIST Freq are same and volt are different OR
940 * Duplicate OPPs (both freq and volt are same) and !opp->available
941 * -ENOMEM Memory allocation failure
942 * -EINVAL Failed parsing the OPP node
944 static int _opp_add_static_v2(struct device
*dev
, struct device_node
*np
)
946 struct device_opp
*dev_opp
;
947 struct dev_pm_opp
*new_opp
;
952 /* Hold our list modification lock here */
953 mutex_lock(&dev_opp_list_lock
);
955 new_opp
= _allocate_opp(dev
, &dev_opp
);
961 ret
= of_property_read_u64(np
, "opp-hz", &rate
);
963 dev_err(dev
, "%s: opp-hz not found\n", __func__
);
968 * Rate is defined as an unsigned long in clk API, and so casting
969 * explicitly to its type. Must be fixed once rate is 64 bit
970 * guaranteed in clk API.
972 new_opp
->rate
= (unsigned long)rate
;
973 new_opp
->turbo
= of_property_read_bool(np
, "turbo-mode");
976 new_opp
->dynamic
= false;
977 new_opp
->available
= true;
979 if (!of_property_read_u32(np
, "clock-latency-ns", &val
))
980 new_opp
->clock_latency_ns
= val
;
982 ret
= opp_get_microvolt(new_opp
, dev
);
986 if (!of_property_read_u32(new_opp
->np
, "opp-microamp", &val
))
987 new_opp
->u_amp
= val
;
989 ret
= _opp_add(dev
, new_opp
, dev_opp
);
993 /* OPP to select on device suspend */
994 if (of_property_read_bool(np
, "opp-suspend")) {
995 if (dev_opp
->suspend_opp
)
996 dev_warn(dev
, "%s: Multiple suspend OPPs found (%lu %lu)\n",
997 __func__
, dev_opp
->suspend_opp
->rate
,
1000 dev_opp
->suspend_opp
= new_opp
;
1003 if (new_opp
->clock_latency_ns
> dev_opp
->clock_latency_ns_max
)
1004 dev_opp
->clock_latency_ns_max
= new_opp
->clock_latency_ns
;
1006 mutex_unlock(&dev_opp_list_lock
);
1008 pr_debug("%s: turbo:%d rate:%lu uv:%lu uvmin:%lu uvmax:%lu latency:%lu\n",
1009 __func__
, new_opp
->turbo
, new_opp
->rate
, new_opp
->u_volt
,
1010 new_opp
->u_volt_min
, new_opp
->u_volt_max
,
1011 new_opp
->clock_latency_ns
);
1014 * Notify the changes in the availability of the operable
1015 * frequency/voltage list.
1017 srcu_notifier_call_chain(&dev_opp
->srcu_head
, OPP_EVENT_ADD
, new_opp
);
1021 _opp_remove(dev_opp
, new_opp
, false);
1023 mutex_unlock(&dev_opp_list_lock
);
1028 * dev_pm_opp_add() - Add an OPP table from a table definitions
1029 * @dev: device for which we do this operation
1030 * @freq: Frequency in Hz for this OPP
1031 * @u_volt: Voltage in uVolts for this OPP
1033 * This function adds an opp definition to the opp list and returns status.
1034 * The opp is made available by default and it can be controlled using
1035 * dev_pm_opp_enable/disable functions.
1037 * Locking: The internal device_opp and opp structures are RCU protected.
1038 * Hence this function internally uses RCU updater strategy with mutex locks
1039 * to keep the integrity of the internal data structures. Callers should ensure
1040 * that this function is *NOT* called under RCU protection or in contexts where
1041 * mutex cannot be locked.
1045 * Duplicate OPPs (both freq and volt are same) and opp->available
1046 * -EEXIST Freq are same and volt are different OR
1047 * Duplicate OPPs (both freq and volt are same) and !opp->available
1048 * -ENOMEM Memory allocation failure
1050 int dev_pm_opp_add(struct device
*dev
, unsigned long freq
, unsigned long u_volt
)
1052 return _opp_add_dynamic(dev
, freq
, u_volt
, true);
1054 EXPORT_SYMBOL_GPL(dev_pm_opp_add
);
1057 * _opp_set_availability() - helper to set the availability of an opp
1058 * @dev: device for which we do this operation
1059 * @freq: OPP frequency to modify availability
1060 * @availability_req: availability status requested for this opp
1062 * Set the availability of an OPP with an RCU operation, opp_{enable,disable}
1063 * share a common logic which is isolated here.
1065 * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
1066 * copy operation, returns 0 if no modifcation was done OR modification was
1069 * Locking: The internal device_opp and opp structures are RCU protected.
1070 * Hence this function internally uses RCU updater strategy with mutex locks to
1071 * keep the integrity of the internal data structures. Callers should ensure
1072 * that this function is *NOT* called under RCU protection or in contexts where
1073 * mutex locking or synchronize_rcu() blocking calls cannot be used.
1075 static int _opp_set_availability(struct device
*dev
, unsigned long freq
,
1076 bool availability_req
)
1078 struct device_opp
*dev_opp
;
1079 struct dev_pm_opp
*new_opp
, *tmp_opp
, *opp
= ERR_PTR(-ENODEV
);
1082 /* keep the node allocated */
1083 new_opp
= kmalloc(sizeof(*new_opp
), GFP_KERNEL
);
1087 mutex_lock(&dev_opp_list_lock
);
1089 /* Find the device_opp */
1090 dev_opp
= _find_device_opp(dev
);
1091 if (IS_ERR(dev_opp
)) {
1092 r
= PTR_ERR(dev_opp
);
1093 dev_warn(dev
, "%s: Device OPP not found (%d)\n", __func__
, r
);
1097 /* Do we have the frequency? */
1098 list_for_each_entry(tmp_opp
, &dev_opp
->opp_list
, node
) {
1099 if (tmp_opp
->rate
== freq
) {
1109 /* Is update really needed? */
1110 if (opp
->available
== availability_req
)
1112 /* copy the old data over */
1115 /* plug in new node */
1116 new_opp
->available
= availability_req
;
1118 list_replace_rcu(&opp
->node
, &new_opp
->node
);
1119 mutex_unlock(&dev_opp_list_lock
);
1120 call_srcu(&dev_opp
->srcu_head
.srcu
, &opp
->rcu_head
, _kfree_opp_rcu
);
1122 /* Notify the change of the OPP availability */
1123 if (availability_req
)
1124 srcu_notifier_call_chain(&dev_opp
->srcu_head
, OPP_EVENT_ENABLE
,
1127 srcu_notifier_call_chain(&dev_opp
->srcu_head
, OPP_EVENT_DISABLE
,
1133 mutex_unlock(&dev_opp_list_lock
);
1139 * dev_pm_opp_enable() - Enable a specific OPP
1140 * @dev: device for which we do this operation
1141 * @freq: OPP frequency to enable
1143 * Enables a provided opp. If the operation is valid, this returns 0, else the
1144 * corresponding error value. It is meant to be used for users an OPP available
1145 * after being temporarily made unavailable with dev_pm_opp_disable.
1147 * Locking: The internal device_opp and opp structures are RCU protected.
1148 * Hence this function indirectly uses RCU and mutex locks to keep the
1149 * integrity of the internal data structures. Callers should ensure that
1150 * this function is *NOT* called under RCU protection or in contexts where
1151 * mutex locking or synchronize_rcu() blocking calls cannot be used.
1153 * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
1154 * copy operation, returns 0 if no modifcation was done OR modification was
1157 int dev_pm_opp_enable(struct device
*dev
, unsigned long freq
)
1159 return _opp_set_availability(dev
, freq
, true);
1161 EXPORT_SYMBOL_GPL(dev_pm_opp_enable
);
1164 * dev_pm_opp_disable() - Disable a specific OPP
1165 * @dev: device for which we do this operation
1166 * @freq: OPP frequency to disable
1168 * Disables a provided opp. If the operation is valid, this returns
1169 * 0, else the corresponding error value. It is meant to be a temporary
1170 * control by users to make this OPP not available until the circumstances are
1171 * right to make it available again (with a call to dev_pm_opp_enable).
1173 * Locking: The internal device_opp and opp structures are RCU protected.
1174 * Hence this function indirectly uses RCU and mutex locks to keep the
1175 * integrity of the internal data structures. Callers should ensure that
1176 * this function is *NOT* called under RCU protection or in contexts where
1177 * mutex locking or synchronize_rcu() blocking calls cannot be used.
1179 * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
1180 * copy operation, returns 0 if no modifcation was done OR modification was
1183 int dev_pm_opp_disable(struct device
*dev
, unsigned long freq
)
1185 return _opp_set_availability(dev
, freq
, false);
1187 EXPORT_SYMBOL_GPL(dev_pm_opp_disable
);
1190 * dev_pm_opp_get_notifier() - find notifier_head of the device with opp
1191 * @dev: device pointer used to lookup device OPPs.
1193 * Return: pointer to notifier head if found, otherwise -ENODEV or
1194 * -EINVAL based on type of error casted as pointer. value must be checked
1195 * with IS_ERR to determine valid pointer or error result.
1197 * Locking: This function must be called under rcu_read_lock(). dev_opp is a RCU
1198 * protected pointer. The reason for the same is that the opp pointer which is
1199 * returned will remain valid for use with opp_get_{voltage, freq} only while
1200 * under the locked area. The pointer returned must be used prior to unlocking
1201 * with rcu_read_unlock() to maintain the integrity of the pointer.
1203 struct srcu_notifier_head
*dev_pm_opp_get_notifier(struct device
*dev
)
1205 struct device_opp
*dev_opp
= _find_device_opp(dev
);
1207 if (IS_ERR(dev_opp
))
1208 return ERR_CAST(dev_opp
); /* matching type */
1210 return &dev_opp
->srcu_head
;
1212 EXPORT_SYMBOL_GPL(dev_pm_opp_get_notifier
);
1216 * of_free_opp_table() - Free OPP table entries created from static DT entries
1217 * @dev: device pointer used to lookup device OPPs.
1219 * Free OPPs created using static entries present in DT.
1221 * Locking: The internal device_opp and opp structures are RCU protected.
1222 * Hence this function indirectly uses RCU updater strategy with mutex locks
1223 * to keep the integrity of the internal data structures. Callers should ensure
1224 * that this function is *NOT* called under RCU protection or in contexts where
1225 * mutex cannot be locked.
1227 void of_free_opp_table(struct device
*dev
)
1229 struct device_opp
*dev_opp
;
1230 struct dev_pm_opp
*opp
, *tmp
;
1232 /* Hold our list modification lock here */
1233 mutex_lock(&dev_opp_list_lock
);
1235 /* Check for existing list for 'dev' */
1236 dev_opp
= _find_device_opp(dev
);
1237 if (IS_ERR(dev_opp
)) {
1238 int error
= PTR_ERR(dev_opp
);
1240 if (error
!= -ENODEV
)
1241 WARN(1, "%s: dev_opp: %d\n",
1242 IS_ERR_OR_NULL(dev
) ?
1243 "Invalid device" : dev_name(dev
),
1248 /* Find if dev_opp manages a single device */
1249 if (list_is_singular(&dev_opp
->dev_list
)) {
1250 /* Free static OPPs */
1251 list_for_each_entry_safe(opp
, tmp
, &dev_opp
->opp_list
, node
) {
1253 _opp_remove(dev_opp
, opp
, true);
1256 _remove_list_dev(_find_list_dev(dev
, dev_opp
), dev_opp
);
1260 mutex_unlock(&dev_opp_list_lock
);
1262 EXPORT_SYMBOL_GPL(of_free_opp_table
);
1264 void of_cpumask_free_opp_table(cpumask_var_t cpumask
)
1266 struct device
*cpu_dev
;
1269 WARN_ON(cpumask_empty(cpumask
));
1271 for_each_cpu(cpu
, cpumask
) {
1272 cpu_dev
= get_cpu_device(cpu
);
1274 pr_err("%s: failed to get cpu%d device\n", __func__
,
1279 of_free_opp_table(cpu_dev
);
1282 EXPORT_SYMBOL_GPL(of_cpumask_free_opp_table
);
1284 /* Returns opp descriptor node from its phandle. Caller must do of_node_put() */
1285 static struct device_node
*
1286 _of_get_opp_desc_node_from_prop(struct device
*dev
, const struct property
*prop
)
1288 struct device_node
*opp_np
;
1290 opp_np
= of_find_node_by_phandle(be32_to_cpup(prop
->value
));
1292 dev_err(dev
, "%s: Prop: %s contains invalid opp desc phandle\n",
1293 __func__
, prop
->name
);
1294 return ERR_PTR(-EINVAL
);
1300 /* Returns opp descriptor node for a device. Caller must do of_node_put() */
1301 static struct device_node
*_of_get_opp_desc_node(struct device
*dev
)
1303 const struct property
*prop
;
1305 prop
= of_find_property(dev
->of_node
, "operating-points-v2", NULL
);
1307 return ERR_PTR(-ENODEV
);
1309 return ERR_PTR(-ENODATA
);
1312 * TODO: Support for multiple OPP tables.
1314 * There should be only ONE phandle present in "operating-points-v2"
1317 if (prop
->length
!= sizeof(__be32
)) {
1318 dev_err(dev
, "%s: Invalid opp desc phandle\n", __func__
);
1319 return ERR_PTR(-EINVAL
);
1322 return _of_get_opp_desc_node_from_prop(dev
, prop
);
1325 /* Initializes OPP tables based on new bindings */
1326 static int _of_init_opp_table_v2(struct device
*dev
,
1327 const struct property
*prop
)
1329 struct device_node
*opp_np
, *np
;
1330 struct device_opp
*dev_opp
;
1331 int ret
= 0, count
= 0;
1337 opp_np
= _of_get_opp_desc_node_from_prop(dev
, prop
);
1339 return PTR_ERR(opp_np
);
1341 dev_opp
= _managed_opp(opp_np
);
1343 /* OPPs are already managed */
1344 if (!_add_list_dev(dev
, dev_opp
))
1349 /* We have opp-list node now, iterate over it and add OPPs */
1350 for_each_available_child_of_node(opp_np
, np
) {
1353 ret
= _opp_add_static_v2(dev
, np
);
1355 dev_err(dev
, "%s: Failed to add OPP, %d\n", __func__
,
1361 /* There should be one of more OPP defined */
1362 if (WARN_ON(!count
)) {
1367 dev_opp
= _find_device_opp(dev
);
1368 if (WARN_ON(IS_ERR(dev_opp
))) {
1369 ret
= PTR_ERR(dev_opp
);
1373 dev_opp
->np
= opp_np
;
1374 dev_opp
->shared_opp
= of_property_read_bool(opp_np
, "opp-shared");
1376 of_node_put(opp_np
);
1380 of_free_opp_table(dev
);
1382 of_node_put(opp_np
);
1387 /* Initializes OPP tables based on old-deprecated bindings */
1388 static int _of_init_opp_table_v1(struct device
*dev
)
1390 const struct property
*prop
;
1394 prop
= of_find_property(dev
->of_node
, "operating-points", NULL
);
1401 * Each OPP is a set of tuples consisting of frequency and
1402 * voltage like <freq-kHz vol-uV>.
1404 nr
= prop
->length
/ sizeof(u32
);
1406 dev_err(dev
, "%s: Invalid OPP list\n", __func__
);
1412 unsigned long freq
= be32_to_cpup(val
++) * 1000;
1413 unsigned long volt
= be32_to_cpup(val
++);
1415 if (_opp_add_dynamic(dev
, freq
, volt
, false))
1416 dev_warn(dev
, "%s: Failed to add OPP %ld\n",
1425 * of_init_opp_table() - Initialize opp table from device tree
1426 * @dev: device pointer used to lookup device OPPs.
1428 * Register the initial OPP table with the OPP library for given device.
1430 * Locking: The internal device_opp and opp structures are RCU protected.
1431 * Hence this function indirectly uses RCU updater strategy with mutex locks
1432 * to keep the integrity of the internal data structures. Callers should ensure
1433 * that this function is *NOT* called under RCU protection or in contexts where
1434 * mutex cannot be locked.
1438 * Duplicate OPPs (both freq and volt are same) and opp->available
1439 * -EEXIST Freq are same and volt are different OR
1440 * Duplicate OPPs (both freq and volt are same) and !opp->available
1441 * -ENOMEM Memory allocation failure
1442 * -ENODEV when 'operating-points' property is not found or is invalid data
1444 * -ENODATA when empty 'operating-points' property is found
1445 * -EINVAL when invalid entries are found in opp-v2 table
1447 int of_init_opp_table(struct device
*dev
)
1449 const struct property
*prop
;
1452 * OPPs have two version of bindings now. The older one is deprecated,
1453 * try for the new binding first.
1455 prop
= of_find_property(dev
->of_node
, "operating-points-v2", NULL
);
1458 * Try old-deprecated bindings for backward compatibility with
1461 return _of_init_opp_table_v1(dev
);
1464 return _of_init_opp_table_v2(dev
, prop
);
1466 EXPORT_SYMBOL_GPL(of_init_opp_table
);
1468 int of_cpumask_init_opp_table(cpumask_var_t cpumask
)
1470 struct device
*cpu_dev
;
1473 WARN_ON(cpumask_empty(cpumask
));
1475 for_each_cpu(cpu
, cpumask
) {
1476 cpu_dev
= get_cpu_device(cpu
);
1478 pr_err("%s: failed to get cpu%d device\n", __func__
,
1483 ret
= of_init_opp_table(cpu_dev
);
1485 pr_err("%s: couldn't find opp table for cpu:%d, %d\n",
1486 __func__
, cpu
, ret
);
1488 /* Free all other OPPs */
1489 of_cpumask_free_opp_table(cpumask
);
1496 EXPORT_SYMBOL_GPL(of_cpumask_init_opp_table
);
1498 /* Required only for V1 bindings, as v2 can manage it from DT itself */
1499 int set_cpus_sharing_opps(struct device
*cpu_dev
, cpumask_var_t cpumask
)
1501 struct device_list_opp
*list_dev
;
1502 struct device_opp
*dev_opp
;
1508 dev_opp
= _find_device_opp(cpu_dev
);
1509 if (IS_ERR(dev_opp
)) {
1511 goto out_rcu_read_unlock
;
1514 for_each_cpu(cpu
, cpumask
) {
1515 if (cpu
== cpu_dev
->id
)
1518 dev
= get_cpu_device(cpu
);
1520 dev_err(cpu_dev
, "%s: failed to get cpu%d device\n",
1525 list_dev
= _add_list_dev(dev
, dev_opp
);
1527 dev_err(dev
, "%s: failed to add list-dev for cpu%d device\n",
1532 out_rcu_read_unlock
:
1537 EXPORT_SYMBOL_GPL(set_cpus_sharing_opps
);
1540 * Works only for OPP v2 bindings.
1542 * cpumask should be already set to mask of cpu_dev->id.
1543 * Returns -ENOENT if operating-points-v2 bindings aren't supported.
1545 int of_get_cpus_sharing_opps(struct device
*cpu_dev
, cpumask_var_t cpumask
)
1547 struct device_node
*np
, *tmp_np
;
1548 struct device
*tcpu_dev
;
1551 /* Get OPP descriptor node */
1552 np
= _of_get_opp_desc_node(cpu_dev
);
1554 dev_dbg(cpu_dev
, "%s: Couldn't find opp node: %ld\n", __func__
,
1559 /* OPPs are shared ? */
1560 if (!of_property_read_bool(np
, "opp-shared"))
1563 for_each_possible_cpu(cpu
) {
1564 if (cpu
== cpu_dev
->id
)
1567 tcpu_dev
= get_cpu_device(cpu
);
1569 dev_err(cpu_dev
, "%s: failed to get cpu%d device\n",
1575 /* Get OPP descriptor node */
1576 tmp_np
= _of_get_opp_desc_node(tcpu_dev
);
1577 if (IS_ERR(tmp_np
)) {
1578 dev_err(tcpu_dev
, "%s: Couldn't find opp node: %ld\n",
1579 __func__
, PTR_ERR(tmp_np
));
1580 ret
= PTR_ERR(tmp_np
);
1584 /* CPUs are sharing opp node */
1586 cpumask_set_cpu(cpu
, cpumask
);
1588 of_node_put(tmp_np
);
1595 EXPORT_SYMBOL_GPL(of_get_cpus_sharing_opps
);