2 * Generic OPP Interface
4 * Copyright (C) 2009-2010 Texas Instruments Incorporated.
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
14 #include <linux/cpu.h>
15 #include <linux/kernel.h>
16 #include <linux/errno.h>
17 #include <linux/err.h>
18 #include <linux/slab.h>
19 #include <linux/device.h>
20 #include <linux/list.h>
21 #include <linux/rculist.h>
22 #include <linux/rcupdate.h>
23 #include <linux/pm_opp.h>
25 #include <linux/export.h>
28 * Internal data structure organization with the OPP layer library is as
31 * |- device 1 (represents voltage domain 1)
32 * | |- opp 1 (availability, freq, voltage)
36 * |- device 2 (represents the next voltage domain)
38 * `- device m (represents mth voltage domain)
39 * device 1, 2.. are represented by dev_opp structure while each opp
40 * is represented by the opp structure.
44 * struct dev_pm_opp - Generic OPP description structure
45 * @node: opp list node. The nodes are maintained throughout the lifetime
46 * of boot. It is expected only an optimal set of OPPs are
47 * added to the library by the SoC framework.
48 * RCU usage: opp list is traversed with RCU locks. node
49 * modification is possible realtime, hence the modifications
50 * are protected by the dev_opp_list_lock for integrity.
51 * IMPORTANT: the opp nodes should be maintained in increasing
53 * @dynamic: not-created from static DT entries.
54 * @available: true/false - marks if this OPP as available or not
55 * @turbo: true if turbo (boost) OPP
56 * @rate: Frequency in hertz
57 * @u_volt: Target voltage in microvolts corresponding to this OPP
58 * @u_volt_min: Minimum voltage in microvolts corresponding to this OPP
59 * @u_volt_max: Maximum voltage in microvolts corresponding to this OPP
60 * @u_amp: Maximum current drawn by the device in microamperes
61 * @clock_latency_ns: Latency (in nanoseconds) of switching to this OPP's
62 * frequency from any other OPP's frequency.
63 * @dev_opp: points back to the device_opp struct this opp belongs to
64 * @rcu_head: RCU callback head used for deferred freeing
65 * @np: OPP's device node.
67 * This structure stores the OPP information for a given device.
70 struct list_head node
;
78 unsigned long u_volt_min
;
79 unsigned long u_volt_max
;
81 unsigned long clock_latency_ns
;
83 struct device_opp
*dev_opp
;
84 struct rcu_head rcu_head
;
86 struct device_node
*np
;
90 * struct device_list_opp - devices managed by 'struct device_opp'
92 * @dev: device to which the struct object belongs
93 * @rcu_head: RCU callback head used for deferred freeing
95 * This is an internal data structure maintaining the list of devices that are
96 * managed by 'struct device_opp'.
98 struct device_list_opp
{
99 struct list_head node
;
100 const struct device
*dev
;
101 struct rcu_head rcu_head
;
105 * struct device_opp - Device opp structure
106 * @node: list node - contains the devices with OPPs that
107 * have been registered. Nodes once added are not modified in this
109 * RCU usage: nodes are not modified in the list of device_opp,
110 * however addition is possible and is secured by dev_opp_list_lock
111 * @srcu_head: notifier head to notify the OPP availability changes.
112 * @rcu_head: RCU callback head used for deferred freeing
113 * @dev_list: list of devices that share these OPPs
114 * @opp_list: list of opps
115 * @np: struct device_node pointer for opp's DT node.
116 * @shared_opp: OPP is shared between multiple devices.
118 * This is an internal data structure maintaining the link to opps attached to
119 * a device. This structure is not meant to be shared to users as it is
120 * meant for book keeping and private to OPP library.
122 * Because the opp structures can be used from both rcu and srcu readers, we
123 * need to wait for the grace period of both of them before freeing any
124 * resources. And so we have used kfree_rcu() from within call_srcu() handlers.
127 struct list_head node
;
129 struct srcu_notifier_head srcu_head
;
130 struct rcu_head rcu_head
;
131 struct list_head dev_list
;
132 struct list_head opp_list
;
134 struct device_node
*np
;
135 unsigned long clock_latency_ns_max
;
137 struct dev_pm_opp
*suspend_opp
;
141 * The root of the list of all devices. All device_opp structures branch off
142 * from here, with each device_opp containing the list of opp it supports in
143 * various states of availability.
145 static LIST_HEAD(dev_opp_list
);
146 /* Lock to allow exclusive modification to the device and opp lists */
147 static DEFINE_MUTEX(dev_opp_list_lock
);
149 #define opp_rcu_lockdep_assert() \
151 RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
152 !lockdep_is_held(&dev_opp_list_lock), \
153 "Missing rcu_read_lock() or " \
154 "dev_opp_list_lock protection"); \
157 static struct device_list_opp
*_find_list_dev(const struct device
*dev
,
158 struct device_opp
*dev_opp
)
160 struct device_list_opp
*list_dev
;
162 list_for_each_entry(list_dev
, &dev_opp
->dev_list
, node
)
163 if (list_dev
->dev
== dev
)
169 static struct device_opp
*_managed_opp(const struct device_node
*np
)
171 struct device_opp
*dev_opp
;
173 list_for_each_entry_rcu(dev_opp
, &dev_opp_list
, node
) {
174 if (dev_opp
->np
== np
) {
176 * Multiple devices can point to the same OPP table and
177 * so will have same node-pointer, np.
179 * But the OPPs will be considered as shared only if the
180 * OPP table contains a "opp-shared" property.
182 return dev_opp
->shared_opp
? dev_opp
: NULL
;
190 * _find_device_opp() - find device_opp struct using device pointer
191 * @dev: device pointer used to lookup device OPPs
193 * Search list of device OPPs for one containing matching device. Does a RCU
194 * reader operation to grab the pointer needed.
196 * Return: pointer to 'struct device_opp' if found, otherwise -ENODEV or
197 * -EINVAL based on type of error.
199 * Locking: This function must be called under rcu_read_lock(). device_opp
200 * is a RCU protected pointer. This means that device_opp is valid as long
201 * as we are under RCU lock.
203 static struct device_opp
*_find_device_opp(struct device
*dev
)
205 struct device_opp
*dev_opp
;
207 if (IS_ERR_OR_NULL(dev
)) {
208 pr_err("%s: Invalid parameters\n", __func__
);
209 return ERR_PTR(-EINVAL
);
212 list_for_each_entry_rcu(dev_opp
, &dev_opp_list
, node
)
213 if (_find_list_dev(dev
, dev_opp
))
216 return ERR_PTR(-ENODEV
);
220 * dev_pm_opp_get_voltage() - Gets the voltage corresponding to an available opp
221 * @opp: opp for which voltage has to be returned for
223 * Return: voltage in micro volt corresponding to the opp, else
226 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
227 * protected pointer. This means that opp which could have been fetched by
228 * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are
229 * under RCU lock. The pointer returned by the opp_find_freq family must be
230 * used in the same section as the usage of this function with the pointer
231 * prior to unlocking with rcu_read_unlock() to maintain the integrity of the
234 unsigned long dev_pm_opp_get_voltage(struct dev_pm_opp
*opp
)
236 struct dev_pm_opp
*tmp_opp
;
239 opp_rcu_lockdep_assert();
241 tmp_opp
= rcu_dereference(opp
);
242 if (IS_ERR_OR_NULL(tmp_opp
) || !tmp_opp
->available
)
243 pr_err("%s: Invalid parameters\n", __func__
);
249 EXPORT_SYMBOL_GPL(dev_pm_opp_get_voltage
);
252 * dev_pm_opp_get_freq() - Gets the frequency corresponding to an available opp
253 * @opp: opp for which frequency has to be returned for
255 * Return: frequency in hertz corresponding to the opp, else
258 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
259 * protected pointer. This means that opp which could have been fetched by
260 * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are
261 * under RCU lock. The pointer returned by the opp_find_freq family must be
262 * used in the same section as the usage of this function with the pointer
263 * prior to unlocking with rcu_read_unlock() to maintain the integrity of the
266 unsigned long dev_pm_opp_get_freq(struct dev_pm_opp
*opp
)
268 struct dev_pm_opp
*tmp_opp
;
271 opp_rcu_lockdep_assert();
273 tmp_opp
= rcu_dereference(opp
);
274 if (IS_ERR_OR_NULL(tmp_opp
) || !tmp_opp
->available
)
275 pr_err("%s: Invalid parameters\n", __func__
);
281 EXPORT_SYMBOL_GPL(dev_pm_opp_get_freq
);
284 * dev_pm_opp_is_turbo() - Returns if opp is turbo OPP or not
285 * @opp: opp for which turbo mode is being verified
287 * Turbo OPPs are not for normal use, and can be enabled (under certain
288 * conditions) for short duration of times to finish high throughput work
289 * quickly. Running on them for longer times may overheat the chip.
291 * Return: true if opp is turbo opp, else false.
293 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
294 * protected pointer. This means that opp which could have been fetched by
295 * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are
296 * under RCU lock. The pointer returned by the opp_find_freq family must be
297 * used in the same section as the usage of this function with the pointer
298 * prior to unlocking with rcu_read_unlock() to maintain the integrity of the
301 bool dev_pm_opp_is_turbo(struct dev_pm_opp
*opp
)
303 struct dev_pm_opp
*tmp_opp
;
305 opp_rcu_lockdep_assert();
307 tmp_opp
= rcu_dereference(opp
);
308 if (IS_ERR_OR_NULL(tmp_opp
) || !tmp_opp
->available
) {
309 pr_err("%s: Invalid parameters\n", __func__
);
313 return tmp_opp
->turbo
;
315 EXPORT_SYMBOL_GPL(dev_pm_opp_is_turbo
);
318 * dev_pm_opp_get_max_clock_latency() - Get max clock latency in nanoseconds
319 * @dev: device for which we do this operation
321 * Return: This function returns the max clock latency in nanoseconds.
323 * Locking: This function takes rcu_read_lock().
325 unsigned long dev_pm_opp_get_max_clock_latency(struct device
*dev
)
327 struct device_opp
*dev_opp
;
328 unsigned long clock_latency_ns
;
332 dev_opp
= _find_device_opp(dev
);
334 clock_latency_ns
= 0;
336 clock_latency_ns
= dev_opp
->clock_latency_ns_max
;
339 return clock_latency_ns
;
341 EXPORT_SYMBOL_GPL(dev_pm_opp_get_max_clock_latency
);
344 * dev_pm_opp_get_opp_count() - Get number of opps available in the opp list
345 * @dev: device for which we do this operation
347 * Return: This function returns the number of available opps if there are any,
348 * else returns 0 if none or the corresponding error value.
350 * Locking: This function takes rcu_read_lock().
352 int dev_pm_opp_get_opp_count(struct device
*dev
)
354 struct device_opp
*dev_opp
;
355 struct dev_pm_opp
*temp_opp
;
360 dev_opp
= _find_device_opp(dev
);
361 if (IS_ERR(dev_opp
)) {
362 count
= PTR_ERR(dev_opp
);
363 dev_err(dev
, "%s: device OPP not found (%d)\n",
368 list_for_each_entry_rcu(temp_opp
, &dev_opp
->opp_list
, node
) {
369 if (temp_opp
->available
)
377 EXPORT_SYMBOL_GPL(dev_pm_opp_get_opp_count
);
380 * dev_pm_opp_find_freq_exact() - search for an exact frequency
381 * @dev: device for which we do this operation
382 * @freq: frequency to search for
383 * @available: true/false - match for available opp
385 * Return: Searches for exact match in the opp list and returns pointer to the
386 * matching opp if found, else returns ERR_PTR in case of error and should
387 * be handled using IS_ERR. Error return values can be:
388 * EINVAL: for bad pointer
389 * ERANGE: no match found for search
390 * ENODEV: if device not found in list of registered devices
392 * Note: available is a modifier for the search. if available=true, then the
393 * match is for exact matching frequency and is available in the stored OPP
394 * table. if false, the match is for exact frequency which is not available.
396 * This provides a mechanism to enable an opp which is not available currently
397 * or the opposite as well.
399 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
400 * protected pointer. The reason for the same is that the opp pointer which is
401 * returned will remain valid for use with opp_get_{voltage, freq} only while
402 * under the locked area. The pointer returned must be used prior to unlocking
403 * with rcu_read_unlock() to maintain the integrity of the pointer.
405 struct dev_pm_opp
*dev_pm_opp_find_freq_exact(struct device
*dev
,
409 struct device_opp
*dev_opp
;
410 struct dev_pm_opp
*temp_opp
, *opp
= ERR_PTR(-ERANGE
);
412 opp_rcu_lockdep_assert();
414 dev_opp
= _find_device_opp(dev
);
415 if (IS_ERR(dev_opp
)) {
416 int r
= PTR_ERR(dev_opp
);
417 dev_err(dev
, "%s: device OPP not found (%d)\n", __func__
, r
);
421 list_for_each_entry_rcu(temp_opp
, &dev_opp
->opp_list
, node
) {
422 if (temp_opp
->available
== available
&&
423 temp_opp
->rate
== freq
) {
431 EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_exact
);
434 * dev_pm_opp_find_freq_ceil() - Search for an rounded ceil freq
435 * @dev: device for which we do this operation
436 * @freq: Start frequency
438 * Search for the matching ceil *available* OPP from a starting freq
441 * Return: matching *opp and refreshes *freq accordingly, else returns
442 * ERR_PTR in case of error and should be handled using IS_ERR. Error return
444 * EINVAL: for bad pointer
445 * ERANGE: no match found for search
446 * ENODEV: if device not found in list of registered devices
448 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
449 * protected pointer. The reason for the same is that the opp pointer which is
450 * returned will remain valid for use with opp_get_{voltage, freq} only while
451 * under the locked area. The pointer returned must be used prior to unlocking
452 * with rcu_read_unlock() to maintain the integrity of the pointer.
454 struct dev_pm_opp
*dev_pm_opp_find_freq_ceil(struct device
*dev
,
457 struct device_opp
*dev_opp
;
458 struct dev_pm_opp
*temp_opp
, *opp
= ERR_PTR(-ERANGE
);
460 opp_rcu_lockdep_assert();
463 dev_err(dev
, "%s: Invalid argument freq=%p\n", __func__
, freq
);
464 return ERR_PTR(-EINVAL
);
467 dev_opp
= _find_device_opp(dev
);
469 return ERR_CAST(dev_opp
);
471 list_for_each_entry_rcu(temp_opp
, &dev_opp
->opp_list
, node
) {
472 if (temp_opp
->available
&& temp_opp
->rate
>= *freq
) {
481 EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_ceil
);
484 * dev_pm_opp_find_freq_floor() - Search for a rounded floor freq
485 * @dev: device for which we do this operation
486 * @freq: Start frequency
488 * Search for the matching floor *available* OPP from a starting freq
491 * Return: matching *opp and refreshes *freq accordingly, else returns
492 * ERR_PTR in case of error and should be handled using IS_ERR. Error return
494 * EINVAL: for bad pointer
495 * ERANGE: no match found for search
496 * ENODEV: if device not found in list of registered devices
498 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
499 * protected pointer. The reason for the same is that the opp pointer which is
500 * returned will remain valid for use with opp_get_{voltage, freq} only while
501 * under the locked area. The pointer returned must be used prior to unlocking
502 * with rcu_read_unlock() to maintain the integrity of the pointer.
504 struct dev_pm_opp
*dev_pm_opp_find_freq_floor(struct device
*dev
,
507 struct device_opp
*dev_opp
;
508 struct dev_pm_opp
*temp_opp
, *opp
= ERR_PTR(-ERANGE
);
510 opp_rcu_lockdep_assert();
513 dev_err(dev
, "%s: Invalid argument freq=%p\n", __func__
, freq
);
514 return ERR_PTR(-EINVAL
);
517 dev_opp
= _find_device_opp(dev
);
519 return ERR_CAST(dev_opp
);
521 list_for_each_entry_rcu(temp_opp
, &dev_opp
->opp_list
, node
) {
522 if (temp_opp
->available
) {
523 /* go to the next node, before choosing prev */
524 if (temp_opp
->rate
> *freq
)
535 EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_floor
);
537 /* List-dev Helpers */
538 static void _kfree_list_dev_rcu(struct rcu_head
*head
)
540 struct device_list_opp
*list_dev
;
542 list_dev
= container_of(head
, struct device_list_opp
, rcu_head
);
543 kfree_rcu(list_dev
, rcu_head
);
546 static void _remove_list_dev(struct device_list_opp
*list_dev
,
547 struct device_opp
*dev_opp
)
549 list_del(&list_dev
->node
);
550 call_srcu(&dev_opp
->srcu_head
.srcu
, &list_dev
->rcu_head
,
551 _kfree_list_dev_rcu
);
554 static struct device_list_opp
*_add_list_dev(const struct device
*dev
,
555 struct device_opp
*dev_opp
)
557 struct device_list_opp
*list_dev
;
559 list_dev
= kzalloc(sizeof(*list_dev
), GFP_KERNEL
);
563 /* Initialize list-dev */
565 list_add_rcu(&list_dev
->node
, &dev_opp
->dev_list
);
571 * _add_device_opp() - Find device OPP table or allocate a new one
572 * @dev: device for which we do this operation
574 * It tries to find an existing table first, if it couldn't find one, it
575 * allocates a new OPP table and returns that.
577 * Return: valid device_opp pointer if success, else NULL.
579 static struct device_opp
*_add_device_opp(struct device
*dev
)
581 struct device_opp
*dev_opp
;
582 struct device_list_opp
*list_dev
;
584 /* Check for existing list for 'dev' first */
585 dev_opp
= _find_device_opp(dev
);
586 if (!IS_ERR(dev_opp
))
590 * Allocate a new device OPP table. In the infrequent case where a new
591 * device is needed to be added, we pay this penalty.
593 dev_opp
= kzalloc(sizeof(*dev_opp
), GFP_KERNEL
);
597 INIT_LIST_HEAD(&dev_opp
->dev_list
);
599 list_dev
= _add_list_dev(dev
, dev_opp
);
605 srcu_init_notifier_head(&dev_opp
->srcu_head
);
606 INIT_LIST_HEAD(&dev_opp
->opp_list
);
608 /* Secure the device list modification */
609 list_add_rcu(&dev_opp
->node
, &dev_opp_list
);
614 * _kfree_device_rcu() - Free device_opp RCU handler
617 static void _kfree_device_rcu(struct rcu_head
*head
)
619 struct device_opp
*device_opp
= container_of(head
, struct device_opp
, rcu_head
);
621 kfree_rcu(device_opp
, rcu_head
);
625 * _remove_device_opp() - Removes a device OPP table
626 * @dev_opp: device OPP table to be removed.
628 * Removes/frees device OPP table it it doesn't contain any OPPs.
630 static void _remove_device_opp(struct device_opp
*dev_opp
)
632 struct device_list_opp
*list_dev
;
634 if (!list_empty(&dev_opp
->opp_list
))
637 list_dev
= list_first_entry(&dev_opp
->dev_list
, struct device_list_opp
,
640 _remove_list_dev(list_dev
, dev_opp
);
642 /* dev_list must be empty now */
643 WARN_ON(!list_empty(&dev_opp
->dev_list
));
645 list_del_rcu(&dev_opp
->node
);
646 call_srcu(&dev_opp
->srcu_head
.srcu
, &dev_opp
->rcu_head
,
651 * _kfree_opp_rcu() - Free OPP RCU handler
654 static void _kfree_opp_rcu(struct rcu_head
*head
)
656 struct dev_pm_opp
*opp
= container_of(head
, struct dev_pm_opp
, rcu_head
);
658 kfree_rcu(opp
, rcu_head
);
662 * _opp_remove() - Remove an OPP from a table definition
663 * @dev_opp: points back to the device_opp struct this opp belongs to
664 * @opp: pointer to the OPP to remove
665 * @notify: OPP_EVENT_REMOVE notification should be sent or not
667 * This function removes an opp definition from the opp list.
669 * Locking: The internal device_opp and opp structures are RCU protected.
670 * It is assumed that the caller holds required mutex for an RCU updater
673 static void _opp_remove(struct device_opp
*dev_opp
,
674 struct dev_pm_opp
*opp
, bool notify
)
677 * Notify the changes in the availability of the operable
678 * frequency/voltage list.
681 srcu_notifier_call_chain(&dev_opp
->srcu_head
, OPP_EVENT_REMOVE
, opp
);
682 list_del_rcu(&opp
->node
);
683 call_srcu(&dev_opp
->srcu_head
.srcu
, &opp
->rcu_head
, _kfree_opp_rcu
);
685 _remove_device_opp(dev_opp
);
689 * dev_pm_opp_remove() - Remove an OPP from OPP list
690 * @dev: device for which we do this operation
691 * @freq: OPP to remove with matching 'freq'
693 * This function removes an opp from the opp list.
695 * Locking: The internal device_opp and opp structures are RCU protected.
696 * Hence this function internally uses RCU updater strategy with mutex locks
697 * to keep the integrity of the internal data structures. Callers should ensure
698 * that this function is *NOT* called under RCU protection or in contexts where
699 * mutex cannot be locked.
701 void dev_pm_opp_remove(struct device
*dev
, unsigned long freq
)
703 struct dev_pm_opp
*opp
;
704 struct device_opp
*dev_opp
;
707 /* Hold our list modification lock here */
708 mutex_lock(&dev_opp_list_lock
);
710 dev_opp
= _find_device_opp(dev
);
714 list_for_each_entry(opp
, &dev_opp
->opp_list
, node
) {
715 if (opp
->rate
== freq
) {
722 dev_warn(dev
, "%s: Couldn't find OPP with freq: %lu\n",
727 _opp_remove(dev_opp
, opp
, true);
729 mutex_unlock(&dev_opp_list_lock
);
731 EXPORT_SYMBOL_GPL(dev_pm_opp_remove
);
733 static struct dev_pm_opp
*_allocate_opp(struct device
*dev
,
734 struct device_opp
**dev_opp
)
736 struct dev_pm_opp
*opp
;
738 /* allocate new OPP node */
739 opp
= kzalloc(sizeof(*opp
), GFP_KERNEL
);
743 INIT_LIST_HEAD(&opp
->node
);
745 *dev_opp
= _add_device_opp(dev
);
754 static int _opp_add(struct device
*dev
, struct dev_pm_opp
*new_opp
,
755 struct device_opp
*dev_opp
)
757 struct dev_pm_opp
*opp
;
758 struct list_head
*head
= &dev_opp
->opp_list
;
761 * Insert new OPP in order of increasing frequency and discard if
764 * Need to use &dev_opp->opp_list in the condition part of the 'for'
765 * loop, don't replace it with head otherwise it will become an infinite
768 list_for_each_entry_rcu(opp
, &dev_opp
->opp_list
, node
) {
769 if (new_opp
->rate
> opp
->rate
) {
774 if (new_opp
->rate
< opp
->rate
)
778 dev_warn(dev
, "%s: duplicate OPPs detected. Existing: freq: %lu, volt: %lu, enabled: %d. New: freq: %lu, volt: %lu, enabled: %d\n",
779 __func__
, opp
->rate
, opp
->u_volt
, opp
->available
,
780 new_opp
->rate
, new_opp
->u_volt
, new_opp
->available
);
782 return opp
->available
&& new_opp
->u_volt
== opp
->u_volt
?
786 new_opp
->dev_opp
= dev_opp
;
787 list_add_rcu(&new_opp
->node
, head
);
793 * _opp_add_dynamic() - Allocate a dynamic OPP.
794 * @dev: device for which we do this operation
795 * @freq: Frequency in Hz for this OPP
796 * @u_volt: Voltage in uVolts for this OPP
797 * @dynamic: Dynamically added OPPs.
799 * This function adds an opp definition to the opp list and returns status.
800 * The opp is made available by default and it can be controlled using
801 * dev_pm_opp_enable/disable functions and may be removed by dev_pm_opp_remove.
803 * NOTE: "dynamic" parameter impacts OPPs added by the of_init_opp_table and
804 * freed by of_free_opp_table.
806 * Locking: The internal device_opp and opp structures are RCU protected.
807 * Hence this function internally uses RCU updater strategy with mutex locks
808 * to keep the integrity of the internal data structures. Callers should ensure
809 * that this function is *NOT* called under RCU protection or in contexts where
810 * mutex cannot be locked.
814 * Duplicate OPPs (both freq and volt are same) and opp->available
815 * -EEXIST Freq are same and volt are different OR
816 * Duplicate OPPs (both freq and volt are same) and !opp->available
817 * -ENOMEM Memory allocation failure
819 static int _opp_add_dynamic(struct device
*dev
, unsigned long freq
,
820 long u_volt
, bool dynamic
)
822 struct device_opp
*dev_opp
;
823 struct dev_pm_opp
*new_opp
;
826 /* Hold our list modification lock here */
827 mutex_lock(&dev_opp_list_lock
);
829 new_opp
= _allocate_opp(dev
, &dev_opp
);
835 /* populate the opp table */
836 new_opp
->rate
= freq
;
837 new_opp
->u_volt
= u_volt
;
838 new_opp
->available
= true;
839 new_opp
->dynamic
= dynamic
;
841 ret
= _opp_add(dev
, new_opp
, dev_opp
);
845 mutex_unlock(&dev_opp_list_lock
);
848 * Notify the changes in the availability of the operable
849 * frequency/voltage list.
851 srcu_notifier_call_chain(&dev_opp
->srcu_head
, OPP_EVENT_ADD
, new_opp
);
855 _opp_remove(dev_opp
, new_opp
, false);
857 mutex_unlock(&dev_opp_list_lock
);
861 /* TODO: Support multiple regulators */
862 static int opp_get_microvolt(struct dev_pm_opp
*opp
, struct device
*dev
)
864 u32 microvolt
[3] = {0};
867 count
= of_property_count_u32_elems(opp
->np
, "opp-microvolt");
871 /* There can be one or three elements here */
872 if (count
!= 1 && count
!= 3) {
873 dev_err(dev
, "%s: Invalid number of elements in opp-microvolt property (%d)\n",
878 ret
= of_property_read_u32_array(opp
->np
, "opp-microvolt", microvolt
,
881 dev_err(dev
, "%s: error parsing opp-microvolt: %d\n", __func__
,
886 opp
->u_volt
= microvolt
[0];
887 opp
->u_volt_min
= microvolt
[1];
888 opp
->u_volt_max
= microvolt
[2];
894 * _opp_add_static_v2() - Allocate static OPPs (As per 'v2' DT bindings)
895 * @dev: device for which we do this operation
898 * This function adds an opp definition to the opp list and returns status. The
899 * opp can be controlled using dev_pm_opp_enable/disable functions and may be
900 * removed by dev_pm_opp_remove.
902 * Locking: The internal device_opp and opp structures are RCU protected.
903 * Hence this function internally uses RCU updater strategy with mutex locks
904 * to keep the integrity of the internal data structures. Callers should ensure
905 * that this function is *NOT* called under RCU protection or in contexts where
906 * mutex cannot be locked.
910 * Duplicate OPPs (both freq and volt are same) and opp->available
911 * -EEXIST Freq are same and volt are different OR
912 * Duplicate OPPs (both freq and volt are same) and !opp->available
913 * -ENOMEM Memory allocation failure
914 * -EINVAL Failed parsing the OPP node
916 static int _opp_add_static_v2(struct device
*dev
, struct device_node
*np
)
918 struct device_opp
*dev_opp
;
919 struct dev_pm_opp
*new_opp
;
924 /* Hold our list modification lock here */
925 mutex_lock(&dev_opp_list_lock
);
927 new_opp
= _allocate_opp(dev
, &dev_opp
);
933 ret
= of_property_read_u64(np
, "opp-hz", &rate
);
935 dev_err(dev
, "%s: opp-hz not found\n", __func__
);
940 * Rate is defined as an unsigned long in clk API, and so casting
941 * explicitly to its type. Must be fixed once rate is 64 bit
942 * guaranteed in clk API.
944 new_opp
->rate
= (unsigned long)rate
;
945 new_opp
->turbo
= of_property_read_bool(np
, "turbo-mode");
948 new_opp
->dynamic
= false;
949 new_opp
->available
= true;
951 if (!of_property_read_u32(np
, "clock-latency-ns", &val
))
952 new_opp
->clock_latency_ns
= val
;
954 ret
= opp_get_microvolt(new_opp
, dev
);
958 if (!of_property_read_u32(new_opp
->np
, "opp-microamp", &val
))
959 new_opp
->u_amp
= val
;
961 ret
= _opp_add(dev
, new_opp
, dev_opp
);
965 /* OPP to select on device suspend */
966 if (of_property_read_bool(np
, "opp-suspend")) {
967 if (dev_opp
->suspend_opp
)
968 dev_warn(dev
, "%s: Multiple suspend OPPs found (%lu %lu)\n",
969 __func__
, dev_opp
->suspend_opp
->rate
,
972 dev_opp
->suspend_opp
= new_opp
;
975 if (new_opp
->clock_latency_ns
> dev_opp
->clock_latency_ns_max
)
976 dev_opp
->clock_latency_ns_max
= new_opp
->clock_latency_ns
;
978 mutex_unlock(&dev_opp_list_lock
);
980 pr_debug("%s: turbo:%d rate:%lu uv:%lu uvmin:%lu uvmax:%lu latency:%lu\n",
981 __func__
, new_opp
->turbo
, new_opp
->rate
, new_opp
->u_volt
,
982 new_opp
->u_volt_min
, new_opp
->u_volt_max
,
983 new_opp
->clock_latency_ns
);
986 * Notify the changes in the availability of the operable
987 * frequency/voltage list.
989 srcu_notifier_call_chain(&dev_opp
->srcu_head
, OPP_EVENT_ADD
, new_opp
);
993 _opp_remove(dev_opp
, new_opp
, false);
995 mutex_unlock(&dev_opp_list_lock
);
1000 * dev_pm_opp_add() - Add an OPP table from a table definitions
1001 * @dev: device for which we do this operation
1002 * @freq: Frequency in Hz for this OPP
1003 * @u_volt: Voltage in uVolts for this OPP
1005 * This function adds an opp definition to the opp list and returns status.
1006 * The opp is made available by default and it can be controlled using
1007 * dev_pm_opp_enable/disable functions.
1009 * Locking: The internal device_opp and opp structures are RCU protected.
1010 * Hence this function internally uses RCU updater strategy with mutex locks
1011 * to keep the integrity of the internal data structures. Callers should ensure
1012 * that this function is *NOT* called under RCU protection or in contexts where
1013 * mutex cannot be locked.
1017 * Duplicate OPPs (both freq and volt are same) and opp->available
1018 * -EEXIST Freq are same and volt are different OR
1019 * Duplicate OPPs (both freq and volt are same) and !opp->available
1020 * -ENOMEM Memory allocation failure
1022 int dev_pm_opp_add(struct device
*dev
, unsigned long freq
, unsigned long u_volt
)
1024 return _opp_add_dynamic(dev
, freq
, u_volt
, true);
1026 EXPORT_SYMBOL_GPL(dev_pm_opp_add
);
1029 * _opp_set_availability() - helper to set the availability of an opp
1030 * @dev: device for which we do this operation
1031 * @freq: OPP frequency to modify availability
1032 * @availability_req: availability status requested for this opp
1034 * Set the availability of an OPP with an RCU operation, opp_{enable,disable}
1035 * share a common logic which is isolated here.
1037 * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
1038 * copy operation, returns 0 if no modifcation was done OR modification was
1041 * Locking: The internal device_opp and opp structures are RCU protected.
1042 * Hence this function internally uses RCU updater strategy with mutex locks to
1043 * keep the integrity of the internal data structures. Callers should ensure
1044 * that this function is *NOT* called under RCU protection or in contexts where
1045 * mutex locking or synchronize_rcu() blocking calls cannot be used.
1047 static int _opp_set_availability(struct device
*dev
, unsigned long freq
,
1048 bool availability_req
)
1050 struct device_opp
*dev_opp
;
1051 struct dev_pm_opp
*new_opp
, *tmp_opp
, *opp
= ERR_PTR(-ENODEV
);
1054 /* keep the node allocated */
1055 new_opp
= kmalloc(sizeof(*new_opp
), GFP_KERNEL
);
1059 mutex_lock(&dev_opp_list_lock
);
1061 /* Find the device_opp */
1062 dev_opp
= _find_device_opp(dev
);
1063 if (IS_ERR(dev_opp
)) {
1064 r
= PTR_ERR(dev_opp
);
1065 dev_warn(dev
, "%s: Device OPP not found (%d)\n", __func__
, r
);
1069 /* Do we have the frequency? */
1070 list_for_each_entry(tmp_opp
, &dev_opp
->opp_list
, node
) {
1071 if (tmp_opp
->rate
== freq
) {
1081 /* Is update really needed? */
1082 if (opp
->available
== availability_req
)
1084 /* copy the old data over */
1087 /* plug in new node */
1088 new_opp
->available
= availability_req
;
1090 list_replace_rcu(&opp
->node
, &new_opp
->node
);
1091 mutex_unlock(&dev_opp_list_lock
);
1092 call_srcu(&dev_opp
->srcu_head
.srcu
, &opp
->rcu_head
, _kfree_opp_rcu
);
1094 /* Notify the change of the OPP availability */
1095 if (availability_req
)
1096 srcu_notifier_call_chain(&dev_opp
->srcu_head
, OPP_EVENT_ENABLE
,
1099 srcu_notifier_call_chain(&dev_opp
->srcu_head
, OPP_EVENT_DISABLE
,
1105 mutex_unlock(&dev_opp_list_lock
);
1111 * dev_pm_opp_enable() - Enable a specific OPP
1112 * @dev: device for which we do this operation
1113 * @freq: OPP frequency to enable
1115 * Enables a provided opp. If the operation is valid, this returns 0, else the
1116 * corresponding error value. It is meant to be used for users an OPP available
1117 * after being temporarily made unavailable with dev_pm_opp_disable.
1119 * Locking: The internal device_opp and opp structures are RCU protected.
1120 * Hence this function indirectly uses RCU and mutex locks to keep the
1121 * integrity of the internal data structures. Callers should ensure that
1122 * this function is *NOT* called under RCU protection or in contexts where
1123 * mutex locking or synchronize_rcu() blocking calls cannot be used.
1125 * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
1126 * copy operation, returns 0 if no modifcation was done OR modification was
1129 int dev_pm_opp_enable(struct device
*dev
, unsigned long freq
)
1131 return _opp_set_availability(dev
, freq
, true);
1133 EXPORT_SYMBOL_GPL(dev_pm_opp_enable
);
1136 * dev_pm_opp_disable() - Disable a specific OPP
1137 * @dev: device for which we do this operation
1138 * @freq: OPP frequency to disable
1140 * Disables a provided opp. If the operation is valid, this returns
1141 * 0, else the corresponding error value. It is meant to be a temporary
1142 * control by users to make this OPP not available until the circumstances are
1143 * right to make it available again (with a call to dev_pm_opp_enable).
1145 * Locking: The internal device_opp and opp structures are RCU protected.
1146 * Hence this function indirectly uses RCU and mutex locks to keep the
1147 * integrity of the internal data structures. Callers should ensure that
1148 * this function is *NOT* called under RCU protection or in contexts where
1149 * mutex locking or synchronize_rcu() blocking calls cannot be used.
1151 * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
1152 * copy operation, returns 0 if no modifcation was done OR modification was
1155 int dev_pm_opp_disable(struct device
*dev
, unsigned long freq
)
1157 return _opp_set_availability(dev
, freq
, false);
1159 EXPORT_SYMBOL_GPL(dev_pm_opp_disable
);
1162 * dev_pm_opp_get_notifier() - find notifier_head of the device with opp
1163 * @dev: device pointer used to lookup device OPPs.
1165 * Return: pointer to notifier head if found, otherwise -ENODEV or
1166 * -EINVAL based on type of error casted as pointer. value must be checked
1167 * with IS_ERR to determine valid pointer or error result.
1169 * Locking: This function must be called under rcu_read_lock(). dev_opp is a RCU
1170 * protected pointer. The reason for the same is that the opp pointer which is
1171 * returned will remain valid for use with opp_get_{voltage, freq} only while
1172 * under the locked area. The pointer returned must be used prior to unlocking
1173 * with rcu_read_unlock() to maintain the integrity of the pointer.
1175 struct srcu_notifier_head
*dev_pm_opp_get_notifier(struct device
*dev
)
1177 struct device_opp
*dev_opp
= _find_device_opp(dev
);
1179 if (IS_ERR(dev_opp
))
1180 return ERR_CAST(dev_opp
); /* matching type */
1182 return &dev_opp
->srcu_head
;
1184 EXPORT_SYMBOL_GPL(dev_pm_opp_get_notifier
);
1188 * of_free_opp_table() - Free OPP table entries created from static DT entries
1189 * @dev: device pointer used to lookup device OPPs.
1191 * Free OPPs created using static entries present in DT.
1193 * Locking: The internal device_opp and opp structures are RCU protected.
1194 * Hence this function indirectly uses RCU updater strategy with mutex locks
1195 * to keep the integrity of the internal data structures. Callers should ensure
1196 * that this function is *NOT* called under RCU protection or in contexts where
1197 * mutex cannot be locked.
1199 void of_free_opp_table(struct device
*dev
)
1201 struct device_opp
*dev_opp
;
1202 struct dev_pm_opp
*opp
, *tmp
;
1204 /* Hold our list modification lock here */
1205 mutex_lock(&dev_opp_list_lock
);
1207 /* Check for existing list for 'dev' */
1208 dev_opp
= _find_device_opp(dev
);
1209 if (IS_ERR(dev_opp
)) {
1210 int error
= PTR_ERR(dev_opp
);
1212 if (error
!= -ENODEV
)
1213 WARN(1, "%s: dev_opp: %d\n",
1214 IS_ERR_OR_NULL(dev
) ?
1215 "Invalid device" : dev_name(dev
),
1220 /* Find if dev_opp manages a single device */
1221 if (list_is_singular(&dev_opp
->dev_list
)) {
1222 /* Free static OPPs */
1223 list_for_each_entry_safe(opp
, tmp
, &dev_opp
->opp_list
, node
) {
1225 _opp_remove(dev_opp
, opp
, true);
1228 _remove_list_dev(_find_list_dev(dev
, dev_opp
), dev_opp
);
1232 mutex_unlock(&dev_opp_list_lock
);
1234 EXPORT_SYMBOL_GPL(of_free_opp_table
);
1236 void of_cpumask_free_opp_table(cpumask_var_t cpumask
)
1238 struct device
*cpu_dev
;
1241 WARN_ON(cpumask_empty(cpumask
));
1243 for_each_cpu(cpu
, cpumask
) {
1244 cpu_dev
= get_cpu_device(cpu
);
1246 pr_err("%s: failed to get cpu%d device\n", __func__
,
1251 of_free_opp_table(cpu_dev
);
1254 EXPORT_SYMBOL_GPL(of_cpumask_free_opp_table
);
1256 /* Returns opp descriptor node from its phandle. Caller must do of_node_put() */
1257 static struct device_node
*
1258 _of_get_opp_desc_node_from_prop(struct device
*dev
, const struct property
*prop
)
1260 struct device_node
*opp_np
;
1262 opp_np
= of_find_node_by_phandle(be32_to_cpup(prop
->value
));
1264 dev_err(dev
, "%s: Prop: %s contains invalid opp desc phandle\n",
1265 __func__
, prop
->name
);
1266 return ERR_PTR(-EINVAL
);
1272 /* Returns opp descriptor node for a device. Caller must do of_node_put() */
1273 static struct device_node
*_of_get_opp_desc_node(struct device
*dev
)
1275 const struct property
*prop
;
1277 prop
= of_find_property(dev
->of_node
, "operating-points-v2", NULL
);
1279 return ERR_PTR(-ENODEV
);
1281 return ERR_PTR(-ENODATA
);
1284 * TODO: Support for multiple OPP tables.
1286 * There should be only ONE phandle present in "operating-points-v2"
1289 if (prop
->length
!= sizeof(__be32
)) {
1290 dev_err(dev
, "%s: Invalid opp desc phandle\n", __func__
);
1291 return ERR_PTR(-EINVAL
);
1294 return _of_get_opp_desc_node_from_prop(dev
, prop
);
1297 /* Initializes OPP tables based on new bindings */
1298 static int _of_init_opp_table_v2(struct device
*dev
,
1299 const struct property
*prop
)
1301 struct device_node
*opp_np
, *np
;
1302 struct device_opp
*dev_opp
;
1303 int ret
= 0, count
= 0;
1309 opp_np
= _of_get_opp_desc_node_from_prop(dev
, prop
);
1311 return PTR_ERR(opp_np
);
1313 dev_opp
= _managed_opp(opp_np
);
1315 /* OPPs are already managed */
1316 if (!_add_list_dev(dev
, dev_opp
))
1321 /* We have opp-list node now, iterate over it and add OPPs */
1322 for_each_available_child_of_node(opp_np
, np
) {
1325 ret
= _opp_add_static_v2(dev
, np
);
1327 dev_err(dev
, "%s: Failed to add OPP, %d\n", __func__
,
1333 /* There should be one of more OPP defined */
1334 if (WARN_ON(!count
)) {
1339 dev_opp
= _find_device_opp(dev
);
1340 if (WARN_ON(IS_ERR(dev_opp
))) {
1341 ret
= PTR_ERR(dev_opp
);
1345 dev_opp
->np
= opp_np
;
1346 dev_opp
->shared_opp
= of_property_read_bool(opp_np
, "opp-shared");
1348 of_node_put(opp_np
);
1352 of_free_opp_table(dev
);
1354 of_node_put(opp_np
);
1359 /* Initializes OPP tables based on old-deprecated bindings */
1360 static int _of_init_opp_table_v1(struct device
*dev
)
1362 const struct property
*prop
;
1366 prop
= of_find_property(dev
->of_node
, "operating-points", NULL
);
1373 * Each OPP is a set of tuples consisting of frequency and
1374 * voltage like <freq-kHz vol-uV>.
1376 nr
= prop
->length
/ sizeof(u32
);
1378 dev_err(dev
, "%s: Invalid OPP list\n", __func__
);
1384 unsigned long freq
= be32_to_cpup(val
++) * 1000;
1385 unsigned long volt
= be32_to_cpup(val
++);
1387 if (_opp_add_dynamic(dev
, freq
, volt
, false))
1388 dev_warn(dev
, "%s: Failed to add OPP %ld\n",
1397 * of_init_opp_table() - Initialize opp table from device tree
1398 * @dev: device pointer used to lookup device OPPs.
1400 * Register the initial OPP table with the OPP library for given device.
1402 * Locking: The internal device_opp and opp structures are RCU protected.
1403 * Hence this function indirectly uses RCU updater strategy with mutex locks
1404 * to keep the integrity of the internal data structures. Callers should ensure
1405 * that this function is *NOT* called under RCU protection or in contexts where
1406 * mutex cannot be locked.
1410 * Duplicate OPPs (both freq and volt are same) and opp->available
1411 * -EEXIST Freq are same and volt are different OR
1412 * Duplicate OPPs (both freq and volt are same) and !opp->available
1413 * -ENOMEM Memory allocation failure
1414 * -ENODEV when 'operating-points' property is not found or is invalid data
1416 * -ENODATA when empty 'operating-points' property is found
1417 * -EINVAL when invalid entries are found in opp-v2 table
1419 int of_init_opp_table(struct device
*dev
)
1421 const struct property
*prop
;
1424 * OPPs have two version of bindings now. The older one is deprecated,
1425 * try for the new binding first.
1427 prop
= of_find_property(dev
->of_node
, "operating-points-v2", NULL
);
1430 * Try old-deprecated bindings for backward compatibility with
1433 return _of_init_opp_table_v1(dev
);
1436 return _of_init_opp_table_v2(dev
, prop
);
1438 EXPORT_SYMBOL_GPL(of_init_opp_table
);
1440 int of_cpumask_init_opp_table(cpumask_var_t cpumask
)
1442 struct device
*cpu_dev
;
1445 WARN_ON(cpumask_empty(cpumask
));
1447 for_each_cpu(cpu
, cpumask
) {
1448 cpu_dev
= get_cpu_device(cpu
);
1450 pr_err("%s: failed to get cpu%d device\n", __func__
,
1455 ret
= of_init_opp_table(cpu_dev
);
1457 pr_err("%s: couldn't find opp table for cpu:%d, %d\n",
1458 __func__
, cpu
, ret
);
1460 /* Free all other OPPs */
1461 of_cpumask_free_opp_table(cpumask
);
1468 EXPORT_SYMBOL_GPL(of_cpumask_init_opp_table
);
1470 /* Required only for V1 bindings, as v2 can manage it from DT itself */
1471 int set_cpus_sharing_opps(struct device
*cpu_dev
, cpumask_var_t cpumask
)
1473 struct device_list_opp
*list_dev
;
1474 struct device_opp
*dev_opp
;
1480 dev_opp
= _find_device_opp(cpu_dev
);
1481 if (IS_ERR(dev_opp
)) {
1483 goto out_rcu_read_unlock
;
1486 for_each_cpu(cpu
, cpumask
) {
1487 if (cpu
== cpu_dev
->id
)
1490 dev
= get_cpu_device(cpu
);
1492 dev_err(cpu_dev
, "%s: failed to get cpu%d device\n",
1497 list_dev
= _add_list_dev(dev
, dev_opp
);
1499 dev_err(dev
, "%s: failed to add list-dev for cpu%d device\n",
1504 out_rcu_read_unlock
:
1509 EXPORT_SYMBOL_GPL(set_cpus_sharing_opps
);
1512 * Works only for OPP v2 bindings.
1514 * cpumask should be already set to mask of cpu_dev->id.
1515 * Returns -ENOENT if operating-points-v2 bindings aren't supported.
1517 int of_get_cpus_sharing_opps(struct device
*cpu_dev
, cpumask_var_t cpumask
)
1519 struct device_node
*np
, *tmp_np
;
1520 struct device
*tcpu_dev
;
1523 /* Get OPP descriptor node */
1524 np
= _of_get_opp_desc_node(cpu_dev
);
1526 dev_dbg(cpu_dev
, "%s: Couldn't find opp node: %ld\n", __func__
,
1531 /* OPPs are shared ? */
1532 if (!of_property_read_bool(np
, "opp-shared"))
1535 for_each_possible_cpu(cpu
) {
1536 if (cpu
== cpu_dev
->id
)
1539 tcpu_dev
= get_cpu_device(cpu
);
1541 dev_err(cpu_dev
, "%s: failed to get cpu%d device\n",
1547 /* Get OPP descriptor node */
1548 tmp_np
= _of_get_opp_desc_node(tcpu_dev
);
1549 if (IS_ERR(tmp_np
)) {
1550 dev_err(tcpu_dev
, "%s: Couldn't find opp node: %ld\n",
1551 __func__
, PTR_ERR(tmp_np
));
1552 ret
= PTR_ERR(tmp_np
);
1556 /* CPUs are sharing opp node */
1558 cpumask_set_cpu(cpu
, cpumask
);
1560 of_node_put(tmp_np
);
1567 EXPORT_SYMBOL_GPL(of_get_cpus_sharing_opps
);