2 * arch/arm/common/bL_switcher.c -- big.LITTLE cluster switcher core driver
4 * Created by: Nicolas Pitre, March 2012
5 * Copyright: (C) 2012-2013 Linaro Limited
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
12 #include <linux/init.h>
13 #include <linux/kernel.h>
14 #include <linux/module.h>
15 #include <linux/sched.h>
16 #include <linux/interrupt.h>
17 #include <linux/cpu_pm.h>
18 #include <linux/cpu.h>
19 #include <linux/cpumask.h>
20 #include <linux/kthread.h>
21 #include <linux/wait.h>
22 #include <linux/clockchips.h>
23 #include <linux/hrtimer.h>
24 #include <linux/tick.h>
26 #include <linux/mutex.h>
27 #include <linux/string.h>
28 #include <linux/sysfs.h>
29 #include <linux/irqchip/arm-gic.h>
30 #include <linux/moduleparam.h>
32 #include <asm/smp_plat.h>
33 #include <asm/suspend.h>
35 #include <asm/bL_switcher.h>
39 * Use our own MPIDR accessors as the generic ones in asm/cputype.h have
40 * __attribute_const__ and we don't want the compiler to assume any
41 * constness here as the value _does_ change along some code paths.
44 static int read_mpidr(void)
47 asm volatile ("mrc p15, 0, %0, c0, c0, 5" : "=r" (id
));
48 return id
& MPIDR_HWID_BITMASK
;
52 * bL switcher core code.
55 static void bL_do_switch(void *_unused
)
57 unsigned ib_mpidr
, ib_cpu
, ib_cluster
;
59 pr_debug("%s\n", __func__
);
61 ib_mpidr
= cpu_logical_map(smp_processor_id());
62 ib_cpu
= MPIDR_AFFINITY_LEVEL(ib_mpidr
, 0);
63 ib_cluster
= MPIDR_AFFINITY_LEVEL(ib_mpidr
, 1);
66 * Our state has been saved at this point. Let's release our
69 mcpm_set_entry_vector(ib_cpu
, ib_cluster
, cpu_resume
);
73 * From this point, we must assume that our counterpart CPU might
74 * have taken over in its parallel world already, as if execution
75 * just returned from cpu_suspend(). It is therefore important to
76 * be very careful not to make any change the other guy is not
77 * expecting. This is why we need stack isolation.
79 * Fancy under cover tasks could be performed here. For now
83 /* Let's put ourself down. */
84 mcpm_cpu_power_down();
86 /* should never get here */
91 * Stack isolation. To ensure 'current' remains valid, we just use another
92 * piece of our thread's stack space which should be fairly lightly used.
93 * The selected area starts just above the thread_info structure located
94 * at the very bottom of the stack, aligned to a cache line, and indexed
95 * with the cluster number.
97 #define STACK_SIZE 512
98 extern void call_with_stack(void (*fn
)(void *), void *arg
, void *sp
);
99 static int bL_switchpoint(unsigned long _arg
)
101 unsigned int mpidr
= read_mpidr();
102 unsigned int clusterid
= MPIDR_AFFINITY_LEVEL(mpidr
, 1);
103 void *stack
= current_thread_info() + 1;
104 stack
= PTR_ALIGN(stack
, L1_CACHE_BYTES
);
105 stack
+= clusterid
* STACK_SIZE
+ STACK_SIZE
;
106 call_with_stack(bL_do_switch
, (void *)_arg
, stack
);
111 * Generic switcher interface
114 static unsigned int bL_gic_id
[MAX_CPUS_PER_CLUSTER
][MAX_NR_CLUSTERS
];
115 static int bL_switcher_cpu_pairing
[NR_CPUS
];
118 * bL_switch_to - Switch to a specific cluster for the current CPU
119 * @new_cluster_id: the ID of the cluster to switch to.
121 * This function must be called on the CPU to be switched.
122 * Returns 0 on success, else a negative status code.
124 static int bL_switch_to(unsigned int new_cluster_id
)
126 unsigned int mpidr
, this_cpu
, that_cpu
;
127 unsigned int ob_mpidr
, ob_cpu
, ob_cluster
, ib_mpidr
, ib_cpu
, ib_cluster
;
128 struct tick_device
*tdev
;
129 enum clock_event_mode tdev_mode
;
132 this_cpu
= smp_processor_id();
133 ob_mpidr
= read_mpidr();
134 ob_cpu
= MPIDR_AFFINITY_LEVEL(ob_mpidr
, 0);
135 ob_cluster
= MPIDR_AFFINITY_LEVEL(ob_mpidr
, 1);
136 BUG_ON(cpu_logical_map(this_cpu
) != ob_mpidr
);
138 if (new_cluster_id
== ob_cluster
)
141 that_cpu
= bL_switcher_cpu_pairing
[this_cpu
];
142 ib_mpidr
= cpu_logical_map(that_cpu
);
143 ib_cpu
= MPIDR_AFFINITY_LEVEL(ib_mpidr
, 0);
144 ib_cluster
= MPIDR_AFFINITY_LEVEL(ib_mpidr
, 1);
146 pr_debug("before switch: CPU %d MPIDR %#x -> %#x\n",
147 this_cpu
, ob_mpidr
, ib_mpidr
);
149 /* Close the gate for our entry vectors */
150 mcpm_set_entry_vector(ob_cpu
, ob_cluster
, NULL
);
151 mcpm_set_entry_vector(ib_cpu
, ib_cluster
, NULL
);
154 * Let's wake up the inbound CPU now in case it requires some delay
155 * to come online, but leave it gated in our entry vector code.
157 ret
= mcpm_cpu_power_up(ib_cpu
, ib_cluster
);
159 pr_err("%s: mcpm_cpu_power_up() returned %d\n", __func__
, ret
);
164 * From this point we are entering the switch critical zone
165 * and can't take any interrupts anymore.
170 /* redirect GIC's SGIs to our counterpart */
171 gic_migrate_target(bL_gic_id
[ib_cpu
][ib_cluster
]);
174 * Raise a SGI on the inbound CPU to make sure it doesn't stall
175 * in a possible WFI, such as in mcpm_power_down().
177 arch_send_wakeup_ipi_mask(cpumask_of(this_cpu
));
179 tdev
= tick_get_device(this_cpu
);
180 if (tdev
&& !cpumask_equal(tdev
->evtdev
->cpumask
, cpumask_of(this_cpu
)))
183 tdev_mode
= tdev
->evtdev
->mode
;
184 clockevents_set_mode(tdev
->evtdev
, CLOCK_EVT_MODE_SHUTDOWN
);
187 ret
= cpu_pm_enter();
189 /* we can not tolerate errors at this point */
191 panic("%s: cpu_pm_enter() returned %d\n", __func__
, ret
);
193 /* Swap the physical CPUs in the logical map for this logical CPU. */
194 cpu_logical_map(this_cpu
) = ib_mpidr
;
195 cpu_logical_map(that_cpu
) = ob_mpidr
;
197 /* Let's do the actual CPU switch. */
198 ret
= cpu_suspend(0, bL_switchpoint
);
200 panic("%s: cpu_suspend() returned %d\n", __func__
, ret
);
202 /* We are executing on the inbound CPU at this point */
203 mpidr
= read_mpidr();
204 pr_debug("after switch: CPU %d MPIDR %#x\n", this_cpu
, mpidr
);
205 BUG_ON(mpidr
!= ib_mpidr
);
207 mcpm_cpu_powered_up();
212 clockevents_set_mode(tdev
->evtdev
, tdev_mode
);
213 clockevents_program_event(tdev
->evtdev
,
214 tdev
->evtdev
->next_event
, 1);
221 pr_err("%s exiting with error %d\n", __func__
, ret
);
226 struct task_struct
*task
;
227 wait_queue_head_t wq
;
229 struct completion started
;
232 static struct bL_thread bL_threads
[NR_CPUS
];
234 static int bL_switcher_thread(void *arg
)
236 struct bL_thread
*t
= arg
;
237 struct sched_param param
= { .sched_priority
= 1 };
240 sched_setscheduler_nocheck(current
, SCHED_FIFO
, ¶m
);
241 complete(&t
->started
);
244 if (signal_pending(current
))
245 flush_signals(current
);
246 wait_event_interruptible(t
->wq
,
247 t
->wanted_cluster
!= -1 ||
248 kthread_should_stop());
249 cluster
= xchg(&t
->wanted_cluster
, -1);
251 bL_switch_to(cluster
);
252 } while (!kthread_should_stop());
257 static struct task_struct
*bL_switcher_thread_create(int cpu
, void *arg
)
259 struct task_struct
*task
;
261 task
= kthread_create_on_node(bL_switcher_thread
, arg
,
262 cpu_to_node(cpu
), "kswitcher_%d", cpu
);
264 kthread_bind(task
, cpu
);
265 wake_up_process(task
);
267 pr_err("%s failed for CPU %d\n", __func__
, cpu
);
272 * bL_switch_request - Switch to a specific cluster for the given CPU
274 * @cpu: the CPU to switch
275 * @new_cluster_id: the ID of the cluster to switch to.
277 * This function causes a cluster switch on the given CPU by waking up
278 * the appropriate switcher thread. This function may or may not return
279 * before the switch has occurred.
281 int bL_switch_request(unsigned int cpu
, unsigned int new_cluster_id
)
285 if (cpu
>= ARRAY_SIZE(bL_threads
)) {
286 pr_err("%s: cpu %d out of bounds\n", __func__
, cpu
);
290 t
= &bL_threads
[cpu
];
292 return PTR_ERR(t
->task
);
296 t
->wanted_cluster
= new_cluster_id
;
300 EXPORT_SYMBOL_GPL(bL_switch_request
);
303 * Activation and configuration code.
306 static DEFINE_MUTEX(bL_switcher_activation_lock
);
307 static unsigned int bL_switcher_active
;
308 static unsigned int bL_switcher_cpu_original_cluster
[NR_CPUS
];
309 static cpumask_t bL_switcher_removed_logical_cpus
;
311 static void bL_switcher_restore_cpus(void)
315 for_each_cpu(i
, &bL_switcher_removed_logical_cpus
)
319 static int bL_switcher_halve_cpus(void)
321 int i
, j
, cluster_0
, gic_id
, ret
;
322 unsigned int cpu
, cluster
, mask
;
323 cpumask_t available_cpus
;
325 /* First pass to validate what we have */
327 for_each_online_cpu(i
) {
328 cpu
= MPIDR_AFFINITY_LEVEL(cpu_logical_map(i
), 0);
329 cluster
= MPIDR_AFFINITY_LEVEL(cpu_logical_map(i
), 1);
331 pr_err("%s: only dual cluster systems are supported\n", __func__
);
334 if (WARN_ON(cpu
>= MAX_CPUS_PER_CLUSTER
))
336 mask
|= (1 << cluster
);
339 pr_err("%s: no CPU pairing possible\n", __func__
);
344 * Now let's do the pairing. We match each CPU with another CPU
345 * from a different cluster. To get a uniform scheduling behavior
346 * without fiddling with CPU topology and compute capacity data,
347 * we'll use logical CPUs initially belonging to the same cluster.
349 memset(bL_switcher_cpu_pairing
, -1, sizeof(bL_switcher_cpu_pairing
));
350 cpumask_copy(&available_cpus
, cpu_online_mask
);
352 for_each_cpu(i
, &available_cpus
) {
354 cluster
= MPIDR_AFFINITY_LEVEL(cpu_logical_map(i
), 1);
357 if (cluster
!= cluster_0
)
359 cpumask_clear_cpu(i
, &available_cpus
);
360 for_each_cpu(j
, &available_cpus
) {
361 cluster
= MPIDR_AFFINITY_LEVEL(cpu_logical_map(j
), 1);
363 * Let's remember the last match to create "odd"
364 * pairings on purpose in order for other code not
365 * to assume any relation between physical and
366 * logical CPU numbers.
368 if (cluster
!= cluster_0
)
372 bL_switcher_cpu_pairing
[i
] = match
;
373 cpumask_clear_cpu(match
, &available_cpus
);
374 pr_info("CPU%d paired with CPU%d\n", i
, match
);
379 * Now we disable the unwanted CPUs i.e. everything that has no
380 * pairing information (that includes the pairing counterparts).
382 cpumask_clear(&bL_switcher_removed_logical_cpus
);
383 for_each_online_cpu(i
) {
384 cpu
= MPIDR_AFFINITY_LEVEL(cpu_logical_map(i
), 0);
385 cluster
= MPIDR_AFFINITY_LEVEL(cpu_logical_map(i
), 1);
387 /* Let's take note of the GIC ID for this CPU */
388 gic_id
= gic_get_cpu_id(i
);
390 pr_err("%s: bad GIC ID for CPU %d\n", __func__
, i
);
391 bL_switcher_restore_cpus();
394 bL_gic_id
[cpu
][cluster
] = gic_id
;
395 pr_info("GIC ID for CPU %u cluster %u is %u\n",
396 cpu
, cluster
, gic_id
);
398 if (bL_switcher_cpu_pairing
[i
] != -1) {
399 bL_switcher_cpu_original_cluster
[i
] = cluster
;
405 bL_switcher_restore_cpus();
408 cpumask_set_cpu(i
, &bL_switcher_removed_logical_cpus
);
414 static int bL_switcher_enable(void)
418 mutex_lock(&bL_switcher_activation_lock
);
419 cpu_hotplug_driver_lock();
420 if (bL_switcher_active
) {
421 cpu_hotplug_driver_unlock();
422 mutex_unlock(&bL_switcher_activation_lock
);
426 pr_info("big.LITTLE switcher initializing\n");
428 ret
= bL_switcher_halve_cpus();
430 cpu_hotplug_driver_unlock();
431 mutex_unlock(&bL_switcher_activation_lock
);
435 for_each_online_cpu(cpu
) {
436 struct bL_thread
*t
= &bL_threads
[cpu
];
437 init_waitqueue_head(&t
->wq
);
438 init_completion(&t
->started
);
439 t
->wanted_cluster
= -1;
440 t
->task
= bL_switcher_thread_create(cpu
, t
);
443 bL_switcher_active
= 1;
444 pr_info("big.LITTLE switcher initialized\n");
446 cpu_hotplug_driver_unlock();
447 mutex_unlock(&bL_switcher_activation_lock
);
453 static void bL_switcher_disable(void)
455 unsigned int cpu
, cluster
;
457 struct task_struct
*task
;
459 mutex_lock(&bL_switcher_activation_lock
);
460 cpu_hotplug_driver_lock();
461 if (!bL_switcher_active
) {
462 cpu_hotplug_driver_unlock();
463 mutex_unlock(&bL_switcher_activation_lock
);
466 bL_switcher_active
= 0;
469 * To deactivate the switcher, we must shut down the switcher
470 * threads to prevent any other requests from being accepted.
471 * Then, if the final cluster for given logical CPU is not the
472 * same as the original one, we'll recreate a switcher thread
473 * just for the purpose of switching the CPU back without any
474 * possibility for interference from external requests.
476 for_each_online_cpu(cpu
) {
477 t
= &bL_threads
[cpu
];
480 if (!task
|| IS_ERR(task
))
483 /* no more switch may happen on this CPU at this point */
484 cluster
= MPIDR_AFFINITY_LEVEL(cpu_logical_map(cpu
), 1);
485 if (cluster
== bL_switcher_cpu_original_cluster
[cpu
])
487 init_completion(&t
->started
);
488 t
->wanted_cluster
= bL_switcher_cpu_original_cluster
[cpu
];
489 task
= bL_switcher_thread_create(cpu
, t
);
491 wait_for_completion(&t
->started
);
493 cluster
= MPIDR_AFFINITY_LEVEL(cpu_logical_map(cpu
), 1);
494 if (cluster
== bL_switcher_cpu_original_cluster
[cpu
])
497 /* If execution gets here, we're in trouble. */
498 pr_crit("%s: unable to restore original cluster for CPU %d\n",
500 pr_crit("%s: CPU %d can't be restored\n",
501 __func__
, bL_switcher_cpu_pairing
[cpu
]);
502 cpumask_clear_cpu(bL_switcher_cpu_pairing
[cpu
],
503 &bL_switcher_removed_logical_cpus
);
506 bL_switcher_restore_cpus();
507 cpu_hotplug_driver_unlock();
508 mutex_unlock(&bL_switcher_activation_lock
);
511 static ssize_t
bL_switcher_active_show(struct kobject
*kobj
,
512 struct kobj_attribute
*attr
, char *buf
)
514 return sprintf(buf
, "%u\n", bL_switcher_active
);
517 static ssize_t
bL_switcher_active_store(struct kobject
*kobj
,
518 struct kobj_attribute
*attr
, const char *buf
, size_t count
)
524 bL_switcher_disable();
528 ret
= bL_switcher_enable();
534 return (ret
>= 0) ? count
: ret
;
537 static struct kobj_attribute bL_switcher_active_attr
=
538 __ATTR(active
, 0644, bL_switcher_active_show
, bL_switcher_active_store
);
540 static struct attribute
*bL_switcher_attrs
[] = {
541 &bL_switcher_active_attr
.attr
,
545 static struct attribute_group bL_switcher_attr_group
= {
546 .attrs
= bL_switcher_attrs
,
549 static struct kobject
*bL_switcher_kobj
;
551 static int __init
bL_switcher_sysfs_init(void)
555 bL_switcher_kobj
= kobject_create_and_add("bL_switcher", kernel_kobj
);
556 if (!bL_switcher_kobj
)
558 ret
= sysfs_create_group(bL_switcher_kobj
, &bL_switcher_attr_group
);
560 kobject_put(bL_switcher_kobj
);
564 #endif /* CONFIG_SYSFS */
566 bool bL_switcher_get_enabled(void)
568 mutex_lock(&bL_switcher_activation_lock
);
570 return bL_switcher_active
;
572 EXPORT_SYMBOL_GPL(bL_switcher_get_enabled
);
574 void bL_switcher_put_enabled(void)
576 mutex_unlock(&bL_switcher_activation_lock
);
578 EXPORT_SYMBOL_GPL(bL_switcher_put_enabled
);
581 * Veto any CPU hotplug operation on those CPUs we've removed
582 * while the switcher is active.
583 * We're just not ready to deal with that given the trickery involved.
585 static int bL_switcher_hotplug_callback(struct notifier_block
*nfb
,
586 unsigned long action
, void *hcpu
)
588 if (bL_switcher_active
) {
589 int pairing
= bL_switcher_cpu_pairing
[(unsigned long)hcpu
];
590 switch (action
& 0xf) {
592 case CPU_DOWN_PREPARE
:
600 static bool no_bL_switcher
;
601 core_param(no_bL_switcher
, no_bL_switcher
, bool, 0644);
603 static int __init
bL_switcher_init(void)
607 if (MAX_NR_CLUSTERS
!= 2) {
608 pr_err("%s: only dual cluster systems are supported\n", __func__
);
612 cpu_notifier(bL_switcher_hotplug_callback
, 0);
614 if (!no_bL_switcher
) {
615 ret
= bL_switcher_enable();
621 ret
= bL_switcher_sysfs_init();
623 pr_err("%s: unable to create sysfs entry\n", __func__
);
629 late_initcall(bL_switcher_init
);