[deliverable/linux.git] / kernel / time / tick-broadcast.c

/*
 * linux/kernel/time/tick-broadcast.c
 *
 * This file contains functions which emulate a local clock-event
 * device via a broadcast event source.
 *
 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
 * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
 *
 * This code is licenced under the GPL version 2. For details see
 * kernel-base/COPYING.
 */
#include <linux/cpu.h>
#include <linux/err.h>
#include <linux/hrtimer.h>
#include <linux/interrupt.h>
#include <linux/percpu.h>
#include <linux/profile.h>
#include <linux/sched.h>

#include "tick-internal.h"

/*
 * Broadcast support for broken x86 hardware, where the local apic
 * timer stops in C3 state.
 */

static struct tick_device tick_broadcast_device;
/* FIXME: Use cpumask_var_t. */
static DECLARE_BITMAP(tick_broadcast_mask, NR_CPUS);
static DECLARE_BITMAP(tmpmask, NR_CPUS);
static DEFINE_RAW_SPINLOCK(tick_broadcast_lock);
static int tick_broadcast_force;

#ifdef CONFIG_TICK_ONESHOT
static void tick_broadcast_clear_oneshot(int cpu);
#else
static inline void tick_broadcast_clear_oneshot(int cpu) { }
#endif

/*
 * Debugging: see timer_list.c
 */
struct tick_device *tick_get_broadcast_device(void)
{
	return &tick_broadcast_device;
}

struct cpumask *tick_get_broadcast_mask(void)
{
	return to_cpumask(tick_broadcast_mask);
}

/*
 * Start the device in periodic mode
 */
static void tick_broadcast_start_periodic(struct clock_event_device *bc)
{
	if (bc)
		tick_setup_periodic(bc, 1);
}

/*
 * Check, if the device can be utilized as broadcast device:
 */
int tick_check_broadcast_device(struct clock_event_device *dev)
{
	if ((tick_broadcast_device.evtdev &&
	     tick_broadcast_device.evtdev->rating >= dev->rating) ||
	     (dev->features & CLOCK_EVT_FEAT_C3STOP))
		return 0;

	clockevents_exchange_device(tick_broadcast_device.evtdev, dev);
	tick_broadcast_device.evtdev = dev;
	if (!cpumask_empty(tick_get_broadcast_mask()))
		tick_broadcast_start_periodic(dev);
	return 1;
}

/*
 * Check, if the device is the broadcast device
 */
int tick_is_broadcast_device(struct clock_event_device *dev)
{
	return (dev && tick_broadcast_device.evtdev == dev);
}

/*
 * Check, if the device is disfunctional and a place holder, which
 * needs to be handled by the broadcast device.
 */
int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
{
	unsigned long flags;
	int ret = 0;

	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);

	/*
	 * Devices might be registered with both periodic and oneshot
	 * mode disabled. This signals, that the device needs to be
	 * operated from the broadcast device and is a placeholder for
	 * the cpu local device.
	 */
	if (!tick_device_is_functional(dev)) {
		dev->event_handler = tick_handle_periodic;
		cpumask_set_cpu(cpu, tick_get_broadcast_mask());
		tick_broadcast_start_periodic(tick_broadcast_device.evtdev);
		ret = 1;
	} else {
		/*
		 * When the new device is not affected by the stop
		 * feature and the cpu is marked in the broadcast mask
		 * then clear the broadcast bit.
		 */
		if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) {
			int cpu = smp_processor_id();

			cpumask_clear_cpu(cpu, tick_get_broadcast_mask());
			tick_broadcast_clear_oneshot(cpu);
		}
	}
	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
	return ret;
}

/*
 * Broadcast the event to the cpus, which are set in the mask (mangled).
 */
static void tick_do_broadcast(struct cpumask *mask)
{
	int cpu = smp_processor_id();
	struct tick_device *td;

	/*
	 * Check, if the current cpu is in the mask
	 */
	if (cpumask_test_cpu(cpu, mask)) {
		cpumask_clear_cpu(cpu, mask);
		td = &per_cpu(tick_cpu_device, cpu);
		td->evtdev->event_handler(td->evtdev);
	}

	if (!cpumask_empty(mask)) {
		/*
		 * It might be necessary to actually check whether the devices
		 * have different broadcast functions. For now, just use the
		 * one of the first device. This works as long as we have this
		 * misfeature only on x86 (lapic)
		 */
		td = &per_cpu(tick_cpu_device, cpumask_first(mask));
		td->evtdev->broadcast(mask);
	}
}

/*
 * Periodic broadcast:
 * - invoke the broadcast handlers
 */
static void tick_do_periodic_broadcast(void)
{
	raw_spin_lock(&tick_broadcast_lock);

	cpumask_and(to_cpumask(tmpmask),
		    cpu_online_mask, tick_get_broadcast_mask());
	tick_do_broadcast(to_cpumask(tmpmask));

	raw_spin_unlock(&tick_broadcast_lock);
}

/*
 * Event handler for periodic broadcast ticks
 */
static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
{
	ktime_t next;

	tick_do_periodic_broadcast();

	/*
	 * The device is in periodic mode. No reprogramming necessary:
	 */
	if (dev->mode == CLOCK_EVT_MODE_PERIODIC)
		return;

	/*
	 * Setup the next period for devices, which do not have
	 * periodic mode. We read dev->next_event first and add to it
	 * when the event already expired. clockevents_program_event()
	 * sets dev->next_event only when the event is really
	 * programmed to the device.
	 */
	for (next = dev->next_event; ;) {
		next = ktime_add(next, tick_period);

		if (!clockevents_program_event(dev, next, false))
			return;
		tick_do_periodic_broadcast();
	}
}

/*
 * Powerstate information: The system enters/leaves a state, where
 * affected devices might stop
 */
static void tick_do_broadcast_on_off(unsigned long *reason)
{
	struct clock_event_device *bc, *dev;
	struct tick_device *td;
	unsigned long flags;
	int cpu, bc_stopped;

	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);

	cpu = smp_processor_id();
	td = &per_cpu(tick_cpu_device, cpu);
	dev = td->evtdev;
	bc = tick_broadcast_device.evtdev;

	/*
	 * Is the device not affected by the powerstate ?
	 */
	if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP))
		goto out;

	if (!tick_device_is_functional(dev))
		goto out;

	bc_stopped = cpumask_empty(tick_get_broadcast_mask());

	switch (*reason) {
	case CLOCK_EVT_NOTIFY_BROADCAST_ON:
	case CLOCK_EVT_NOTIFY_BROADCAST_FORCE:
		if (!cpumask_test_cpu(cpu, tick_get_broadcast_mask())) {
			cpumask_set_cpu(cpu, tick_get_broadcast_mask());
			if (tick_broadcast_device.mode ==
			    TICKDEV_MODE_PERIODIC)
				clockevents_shutdown(dev);
		}
		if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE)
			tick_broadcast_force = 1;
		break;
	case CLOCK_EVT_NOTIFY_BROADCAST_OFF:
		if (!tick_broadcast_force &&
		    cpumask_test_cpu(cpu, tick_get_broadcast_mask())) {
			cpumask_clear_cpu(cpu, tick_get_broadcast_mask());
			if (tick_broadcast_device.mode ==
			    TICKDEV_MODE_PERIODIC)
				tick_setup_periodic(dev, 0);
		}
		break;
	}

	if (cpumask_empty(tick_get_broadcast_mask())) {
		if (!bc_stopped)
			clockevents_shutdown(bc);
	} else if (bc_stopped) {
		if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
			tick_broadcast_start_periodic(bc);
		else
			tick_broadcast_setup_oneshot(bc);
	}
out:
	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
}

/*
 * Powerstate information: The system enters/leaves a state, where
 * affected devices might stop.
 */
void tick_broadcast_on_off(unsigned long reason, int *oncpu)
{
	if (!cpumask_test_cpu(*oncpu, cpu_online_mask))
		printk(KERN_ERR "tick-broadcast: ignoring broadcast for "
		       "offline CPU #%d\n", *oncpu);
	else
		tick_do_broadcast_on_off(&reason);
}

/*
 * Set the periodic handler depending on broadcast on/off
 */
void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
{
	if (!broadcast)
		dev->event_handler = tick_handle_periodic;
	else
		dev->event_handler = tick_handle_periodic_broadcast;
}

/*
 * Remove a CPU from broadcasting
 */
void tick_shutdown_broadcast(unsigned int *cpup)
{
	struct clock_event_device *bc;
	unsigned long flags;
	unsigned int cpu = *cpup;

	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);

	bc = tick_broadcast_device.evtdev;
	cpumask_clear_cpu(cpu, tick_get_broadcast_mask());

	if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
		if (bc && cpumask_empty(tick_get_broadcast_mask()))
			clockevents_shutdown(bc);
	}

	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
}

void tick_suspend_broadcast(void)
{
	struct clock_event_device *bc;
	unsigned long flags;

	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);

	bc = tick_broadcast_device.evtdev;
	if (bc)
		clockevents_shutdown(bc);

	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
}

int tick_resume_broadcast(void)
{
	struct clock_event_device *bc;
	unsigned long flags;
	int broadcast = 0;

	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);

	bc = tick_broadcast_device.evtdev;

	if (bc) {
		clockevents_set_mode(bc, CLOCK_EVT_MODE_RESUME);

		switch (tick_broadcast_device.mode) {
		case TICKDEV_MODE_PERIODIC:
			if (!cpumask_empty(tick_get_broadcast_mask()))
				tick_broadcast_start_periodic(bc);
			broadcast = cpumask_test_cpu(smp_processor_id(),
						     tick_get_broadcast_mask());
			break;
		case TICKDEV_MODE_ONESHOT:
			broadcast = tick_resume_broadcast_oneshot(bc);
			break;
		}
	}
	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);

	return broadcast;
}


#ifdef CONFIG_TICK_ONESHOT

/* FIXME: use cpumask_var_t. */
static DECLARE_BITMAP(tick_broadcast_oneshot_mask, NR_CPUS);

/*
 * Exposed for debugging: see timer_list.c
 */
struct cpumask *tick_get_broadcast_oneshot_mask(void)
{
	return to_cpumask(tick_broadcast_oneshot_mask);
}

static int tick_broadcast_set_event(ktime_t expires, int force)
{
	struct clock_event_device *bc = tick_broadcast_device.evtdev;

	if (bc->mode != CLOCK_EVT_MODE_ONESHOT)
		clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);

	return clockevents_program_event(bc, expires, force);
}

int tick_resume_broadcast_oneshot(struct clock_event_device *bc)
{
	clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
	return 0;
}

/*
 * Called from irq_enter() when idle was interrupted to reenable the
 * per cpu device.
 */
void tick_check_oneshot_broadcast(int cpu)
{
	if (cpumask_test_cpu(cpu, to_cpumask(tick_broadcast_oneshot_mask))) {
		struct tick_device *td = &per_cpu(tick_cpu_device, cpu);

		clockevents_set_mode(td->evtdev, CLOCK_EVT_MODE_ONESHOT);
	}
}

/*
 * Handle oneshot mode broadcasting
 */
static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
{
	struct tick_device *td;
	ktime_t now, next_event;
	int cpu;

	raw_spin_lock(&tick_broadcast_lock);
again:
	dev->next_event.tv64 = KTIME_MAX;
	next_event.tv64 = KTIME_MAX;
	cpumask_clear(to_cpumask(tmpmask));
	now = ktime_get();
	/* Find all expired events */
	for_each_cpu(cpu, tick_get_broadcast_oneshot_mask()) {
		td = &per_cpu(tick_cpu_device, cpu);
		if (td->evtdev->next_event.tv64 <= now.tv64)
			cpumask_set_cpu(cpu, to_cpumask(tmpmask));
		else if (td->evtdev->next_event.tv64 < next_event.tv64)
			next_event.tv64 = td->evtdev->next_event.tv64;
	}

	/*
	 * Wakeup the cpus which have an expired event.
	 */
	tick_do_broadcast(to_cpumask(tmpmask));

	/*
	 * Two reasons for reprogram:
	 *
	 * - The global event did not expire any CPU local
	 * events. This happens in dyntick mode, as the maximum PIT
	 * delta is quite small.
	 *
	 * - There are pending events on sleeping CPUs which were not
	 * in the event mask
	 */
	if (next_event.tv64 != KTIME_MAX) {
		/*
		 * Rearm the broadcast device. If event expired,
		 * repeat the above
		 */
		if (tick_broadcast_set_event(next_event, 0))
			goto again;
	}
	raw_spin_unlock(&tick_broadcast_lock);
}

/*
 * Powerstate information: The system enters/leaves a state, where
 * affected devices might stop
 */
void tick_broadcast_oneshot_control(unsigned long reason)
{
	struct clock_event_device *bc, *dev;
	struct tick_device *td;
	unsigned long flags;
	int cpu;

	/*
	 * Periodic mode does not care about the enter/exit of power
	 * states
	 */
	if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
		return;

	/*
	 * We are called with preemtion disabled from the depth of the
	 * idle code, so we can't be moved away.
	 */
	cpu = smp_processor_id();
	td = &per_cpu(tick_cpu_device, cpu);
	dev = td->evtdev;

	if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
		return;

	bc = tick_broadcast_device.evtdev;

	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
	if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) {
		if (!cpumask_test_cpu(cpu, tick_get_broadcast_oneshot_mask())) {
			cpumask_set_cpu(cpu, tick_get_broadcast_oneshot_mask());
			clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN);
			if (dev->next_event.tv64 < bc->next_event.tv64)
				tick_broadcast_set_event(dev->next_event, 1);
		}
	} else {
		if (cpumask_test_cpu(cpu, tick_get_broadcast_oneshot_mask())) {
			cpumask_clear_cpu(cpu,
					  tick_get_broadcast_oneshot_mask());
			clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT);
			if (dev->next_event.tv64 != KTIME_MAX)
				tick_program_event(dev->next_event, 1);
		}
	}
	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
}

/*
 * Reset the one shot broadcast for a cpu
 *
 * Called with tick_broadcast_lock held
 */
static void tick_broadcast_clear_oneshot(int cpu)
{
	cpumask_clear_cpu(cpu, tick_get_broadcast_oneshot_mask());
}

static void tick_broadcast_init_next_event(struct cpumask *mask,
					   ktime_t expires)
{
	struct tick_device *td;
	int cpu;

	for_each_cpu(cpu, mask) {
		td = &per_cpu(tick_cpu_device, cpu);
		if (td->evtdev)
			td->evtdev->next_event = expires;
	}
}

/**
 * tick_broadcast_setup_oneshot - setup the broadcast device
 */
void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
{
	int cpu = smp_processor_id();

	/* Set it up only once ! */
	if (bc->event_handler != tick_handle_oneshot_broadcast) {
		int was_periodic = bc->mode == CLOCK_EVT_MODE_PERIODIC;

		bc->event_handler = tick_handle_oneshot_broadcast;

		/* Take the do_timer update */
		tick_do_timer_cpu = cpu;

		/*
		 * We must be careful here. There might be other CPUs
		 * waiting for periodic broadcast. We need to set the
		 * oneshot_mask bits for those and program the
		 * broadcast device to fire.
		 */
		cpumask_copy(to_cpumask(tmpmask), tick_get_broadcast_mask());
		cpumask_clear_cpu(cpu, to_cpumask(tmpmask));
		cpumask_or(tick_get_broadcast_oneshot_mask(),
			   tick_get_broadcast_oneshot_mask(),
			   to_cpumask(tmpmask));

		if (was_periodic && !cpumask_empty(to_cpumask(tmpmask))) {
			clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
			tick_broadcast_init_next_event(to_cpumask(tmpmask),
						       tick_next_period);
			tick_broadcast_set_event(tick_next_period, 1);
		} else
			bc->next_event.tv64 = KTIME_MAX;
	} else {
		/*
		 * The first cpu which switches to oneshot mode sets
		 * the bit for all other cpus which are in the general
		 * (periodic) broadcast mask. So the bit is set and
		 * would prevent the first broadcast enter after this
		 * to program the bc device.
		 */
		tick_broadcast_clear_oneshot(cpu);
	}
}

/*
 * Select oneshot operating mode for the broadcast device
 */
void tick_broadcast_switch_to_oneshot(void)
{
	struct clock_event_device *bc;
	unsigned long flags;

	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);

	tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
	bc = tick_broadcast_device.evtdev;
	if (bc)
		tick_broadcast_setup_oneshot(bc);

	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
}


/*
 * Remove a dead CPU from broadcasting
 */
void tick_shutdown_broadcast_oneshot(unsigned int *cpup)
{
	unsigned long flags;
	unsigned int cpu = *cpup;

	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);

	/*
	 * Clear the broadcast mask flag for the dead cpu, but do not
	 * stop the broadcast device!
	 */
	cpumask_clear_cpu(cpu, tick_get_broadcast_oneshot_mask());

	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
}

/*
 * Check, whether the broadcast device is in one shot mode
 */
int tick_broadcast_oneshot_active(void)
{
	return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT;
}

/*
 * Check whether the broadcast device supports oneshot.
 */
bool tick_broadcast_oneshot_available(void)
{
	struct clock_event_device *bc = tick_broadcast_device.evtdev;

	return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false;
}

#endif
Commit	Line	Data
f8381cba TG	1	/*
	2	* linux/kernel/time/tick-broadcast.c
	3	*
	4	* This file contains functions which emulate a local clock-event
	5	* device via a broadcast event source.
	6	*
	7	* Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
	8	* Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
	9	* Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
	10	*
	11	* This code is licenced under the GPL version 2. For details see
	12	* kernel-base/COPYING.
	13	*/
	14	#include <linux/cpu.h>
	15	#include <linux/err.h>
	16	#include <linux/hrtimer.h>
d7b90689	17	#include <linux/interrupt.h>
f8381cba TG	18	#include <linux/percpu.h>
	19	#include <linux/profile.h>
	20	#include <linux/sched.h>
f8381cba TG	21
	22	#include "tick-internal.h"
	23
	24	/*
	25	* Broadcast support for broken x86 hardware, where the local apic
	26	* timer stops in C3 state.
	27	*/
	28
a52f5c56	29	static struct tick_device tick_broadcast_device;
6b954823 RR	30	/* FIXME: Use cpumask_var_t. */
	31	static DECLARE_BITMAP(tick_broadcast_mask, NR_CPUS);
	32	static DECLARE_BITMAP(tmpmask, NR_CPUS);
b5f91da0	33	static DEFINE_RAW_SPINLOCK(tick_broadcast_lock);
aa276e1c	34	static int tick_broadcast_force;
f8381cba	35
5590a536 TG	36	#ifdef CONFIG_TICK_ONESHOT
	37	static void tick_broadcast_clear_oneshot(int cpu);
	38	#else
	39	static inline void tick_broadcast_clear_oneshot(int cpu) { }
	40	#endif
	41
289f480a IM	42	/*
	43	* Debugging: see timer_list.c
	44	*/
	45	struct tick_device *tick_get_broadcast_device(void)
	46	{
	47	return &tick_broadcast_device;
	48	}
	49
6b954823	50	struct cpumask *tick_get_broadcast_mask(void)
289f480a	51	{
6b954823	52	return to_cpumask(tick_broadcast_mask);
289f480a IM	53	}
289f480a IM	54
f8381cba TG	55	/*
	56	* Start the device in periodic mode
	57	*/
	58	static void tick_broadcast_start_periodic(struct clock_event_device *bc)
	59	{
18de5bc4	60	if (bc)
f8381cba TG	61	tick_setup_periodic(bc, 1);
	62	}
	63
	64	/*
	65	* Check, if the device can be utilized as broadcast device:
	66	*/
	67	int tick_check_broadcast_device(struct clock_event_device *dev)
	68	{
4a93232d VP	69	if ((tick_broadcast_device.evtdev &&
	70	tick_broadcast_device.evtdev->rating >= dev->rating) \|\|
	71	(dev->features & CLOCK_EVT_FEAT_C3STOP))
f8381cba TG	72	return 0;
f8381cba TG	73
c1be8430	74	clockevents_exchange_device(tick_broadcast_device.evtdev, dev);
f8381cba	75	tick_broadcast_device.evtdev = dev;
6b954823	76	if (!cpumask_empty(tick_get_broadcast_mask()))
f8381cba TG	77	tick_broadcast_start_periodic(dev);
	78	return 1;
	79	}
	80
	81	/*
	82	* Check, if the device is the broadcast device
	83	*/
	84	int tick_is_broadcast_device(struct clock_event_device *dev)
	85	{
	86	return (dev && tick_broadcast_device.evtdev == dev);
	87	}
	88
	89	/*
	90	* Check, if the device is disfunctional and a place holder, which
	91	* needs to be handled by the broadcast device.
	92	*/
	93	int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
	94	{
	95	unsigned long flags;
	96	int ret = 0;
	97
b5f91da0	98	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
f8381cba TG	99
	100	/*
	101	* Devices might be registered with both periodic and oneshot
	102	* mode disabled. This signals, that the device needs to be
	103	* operated from the broadcast device and is a placeholder for
	104	* the cpu local device.
	105	*/
	106	if (!tick_device_is_functional(dev)) {
	107	dev->event_handler = tick_handle_periodic;
6b954823	108	cpumask_set_cpu(cpu, tick_get_broadcast_mask());
f8381cba TG	109	tick_broadcast_start_periodic(tick_broadcast_device.evtdev);
f8381cba TG	110	ret = 1;
5590a536 TG	111	} else {
	112	/*
	113	* When the new device is not affected by the stop
	114	* feature and the cpu is marked in the broadcast mask
	115	* then clear the broadcast bit.
	116	*/
	117	if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) {
	118	int cpu = smp_processor_id();
f8381cba	119
6b954823	120	cpumask_clear_cpu(cpu, tick_get_broadcast_mask());
5590a536 TG	121	tick_broadcast_clear_oneshot(cpu);
	122	}
	123	}
b5f91da0	124	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
f8381cba TG	125	return ret;
	126	}
	127
	128	/*
6b954823	129	* Broadcast the event to the cpus, which are set in the mask (mangled).
f8381cba	130	*/
6b954823	131	static void tick_do_broadcast(struct cpumask *mask)
f8381cba	132	{
186e3cb8	133	int cpu = smp_processor_id();
f8381cba TG	134	struct tick_device *td;
	135
	136	/*
	137	* Check, if the current cpu is in the mask
	138	*/
6b954823 RR	139	if (cpumask_test_cpu(cpu, mask)) {
6b954823 RR	140	cpumask_clear_cpu(cpu, mask);
f8381cba TG	141	td = &per_cpu(tick_cpu_device, cpu);
f8381cba TG	142	td->evtdev->event_handler(td->evtdev);
f8381cba TG	143	}
f8381cba TG	144
6b954823	145	if (!cpumask_empty(mask)) {
f8381cba TG	146	/*
	147	* It might be necessary to actually check whether the devices
	148	* have different broadcast functions. For now, just use the
	149	* one of the first device. This works as long as we have this
	150	* misfeature only on x86 (lapic)
	151	*/
6b954823 RR	152	td = &per_cpu(tick_cpu_device, cpumask_first(mask));
6b954823 RR	153	td->evtdev->broadcast(mask);
f8381cba	154	}
f8381cba TG	155	}
	156
	157	/*
	158	* Periodic broadcast:
	159	* - invoke the broadcast handlers
	160	*/
	161	static void tick_do_periodic_broadcast(void)
	162	{
b5f91da0	163	raw_spin_lock(&tick_broadcast_lock);
f8381cba	164
6b954823 RR	165	cpumask_and(to_cpumask(tmpmask),
	166	cpu_online_mask, tick_get_broadcast_mask());
	167	tick_do_broadcast(to_cpumask(tmpmask));
f8381cba	168
b5f91da0	169	raw_spin_unlock(&tick_broadcast_lock);
f8381cba TG	170	}
	171
	172	/*
	173	* Event handler for periodic broadcast ticks
	174	*/
	175	static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
	176	{
d4496b39 TG	177	ktime_t next;
d4496b39 TG	178
f8381cba TG	179	tick_do_periodic_broadcast();
	180
	181	/*
	182	* The device is in periodic mode. No reprogramming necessary:
	183	*/
	184	if (dev->mode == CLOCK_EVT_MODE_PERIODIC)
	185	return;
	186
	187	/*
	188	* Setup the next period for devices, which do not have
d4496b39	189	* periodic mode. We read dev->next_event first and add to it
698f9315	190	* when the event already expired. clockevents_program_event()
d4496b39 TG	191	* sets dev->next_event only when the event is really
d4496b39 TG	192	* programmed to the device.
f8381cba	193	*/
d4496b39 TG	194	for (next = dev->next_event; ;) {
d4496b39 TG	195	next = ktime_add(next, tick_period);
f8381cba	196
d1748302	197	if (!clockevents_program_event(dev, next, false))
f8381cba TG	198	return;
	199	tick_do_periodic_broadcast();
	200	}
	201	}
	202
	203	/*
	204	* Powerstate information: The system enters/leaves a state, where
	205	* affected devices might stop
	206	*/
f833bab8	207	static void tick_do_broadcast_on_off(unsigned long *reason)
f8381cba TG	208	{
	209	struct clock_event_device bc, dev;
	210	struct tick_device *td;
f833bab8	211	unsigned long flags;
9c17bcda	212	int cpu, bc_stopped;
f8381cba	213
b5f91da0	214	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
f8381cba TG	215
	216	cpu = smp_processor_id();
	217	td = &per_cpu(tick_cpu_device, cpu);
	218	dev = td->evtdev;
	219	bc = tick_broadcast_device.evtdev;
	220
	221	/*
1595f452	222	* Is the device not affected by the powerstate ?
f8381cba	223	*/
1595f452	224	if (!dev \|\| !(dev->features & CLOCK_EVT_FEAT_C3STOP))
f8381cba TG	225	goto out;
f8381cba TG	226
3dfbc884 TG	227	if (!tick_device_is_functional(dev))
3dfbc884 TG	228	goto out;
1595f452	229
6b954823	230	bc_stopped = cpumask_empty(tick_get_broadcast_mask());
9c17bcda	231
1595f452 TG	232	switch (*reason) {
	233	case CLOCK_EVT_NOTIFY_BROADCAST_ON:
	234	case CLOCK_EVT_NOTIFY_BROADCAST_FORCE:
6b954823 RR	235	if (!cpumask_test_cpu(cpu, tick_get_broadcast_mask())) {
6b954823 RR	236	cpumask_set_cpu(cpu, tick_get_broadcast_mask());
07454bff TG	237	if (tick_broadcast_device.mode ==
07454bff TG	238	TICKDEV_MODE_PERIODIC)
2344abbc	239	clockevents_shutdown(dev);
f8381cba	240	}
3dfbc884	241	if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE)
aa276e1c	242	tick_broadcast_force = 1;
1595f452 TG	243	break;
1595f452 TG	244	case CLOCK_EVT_NOTIFY_BROADCAST_OFF:
aa276e1c	245	if (!tick_broadcast_force &&
6b954823 RR	246	cpumask_test_cpu(cpu, tick_get_broadcast_mask())) {
6b954823 RR	247	cpumask_clear_cpu(cpu, tick_get_broadcast_mask());
07454bff TG	248	if (tick_broadcast_device.mode ==
07454bff TG	249	TICKDEV_MODE_PERIODIC)
f8381cba TG	250	tick_setup_periodic(dev, 0);
f8381cba TG	251	}
1595f452	252	break;
f8381cba TG	253	}
f8381cba TG	254
6b954823	255	if (cpumask_empty(tick_get_broadcast_mask())) {
9c17bcda	256	if (!bc_stopped)
2344abbc	257	clockevents_shutdown(bc);
9c17bcda	258	} else if (bc_stopped) {
f8381cba TG	259	if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
f8381cba TG	260	tick_broadcast_start_periodic(bc);
79bf2bb3 TG	261	else
79bf2bb3 TG	262	tick_broadcast_setup_oneshot(bc);
f8381cba TG	263	}
f8381cba TG	264	out:
b5f91da0	265	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
f8381cba TG	266	}
	267
	268	/*
	269	* Powerstate information: The system enters/leaves a state, where
	270	* affected devices might stop.
	271	*/
	272	void tick_broadcast_on_off(unsigned long reason, int *oncpu)
	273	{
6b954823	274	if (!cpumask_test_cpu(*oncpu, cpu_online_mask))
833df317	275	printk(KERN_ERR "tick-broadcast: ignoring broadcast for "
72fcde96	276	"offline CPU #%d\n", *oncpu);
bf020cb7	277	else
f833bab8	278	tick_do_broadcast_on_off(&reason);
f8381cba TG	279	}
	280
	281	/*
	282	* Set the periodic handler depending on broadcast on/off
	283	*/
	284	void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
	285	{
	286	if (!broadcast)
	287	dev->event_handler = tick_handle_periodic;
	288	else
	289	dev->event_handler = tick_handle_periodic_broadcast;
	290	}
	291
	292	/*
	293	* Remove a CPU from broadcasting
	294	*/
	295	void tick_shutdown_broadcast(unsigned int *cpup)
	296	{
	297	struct clock_event_device *bc;
	298	unsigned long flags;
	299	unsigned int cpu = *cpup;
	300
b5f91da0	301	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
f8381cba TG	302
f8381cba TG	303	bc = tick_broadcast_device.evtdev;
6b954823	304	cpumask_clear_cpu(cpu, tick_get_broadcast_mask());
f8381cba TG	305
f8381cba TG	306	if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
6b954823	307	if (bc && cpumask_empty(tick_get_broadcast_mask()))
2344abbc	308	clockevents_shutdown(bc);
f8381cba TG	309	}
f8381cba TG	310
b5f91da0	311	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
f8381cba	312	}
79bf2bb3	313
6321dd60 TG	314	void tick_suspend_broadcast(void)
	315	{
	316	struct clock_event_device *bc;
	317	unsigned long flags;
	318
b5f91da0	319	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
6321dd60 TG	320
6321dd60 TG	321	bc = tick_broadcast_device.evtdev;
18de5bc4	322	if (bc)
2344abbc	323	clockevents_shutdown(bc);
6321dd60	324
b5f91da0	325	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
6321dd60 TG	326	}
	327
	328	int tick_resume_broadcast(void)
	329	{
	330	struct clock_event_device *bc;
	331	unsigned long flags;
	332	int broadcast = 0;
	333
b5f91da0	334	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
6321dd60 TG	335
6321dd60 TG	336	bc = tick_broadcast_device.evtdev;
6321dd60	337
cd05a1f8	338	if (bc) {
18de5bc4 TG	339	clockevents_set_mode(bc, CLOCK_EVT_MODE_RESUME);
18de5bc4 TG	340
cd05a1f8 TG	341	switch (tick_broadcast_device.mode) {
cd05a1f8 TG	342	case TICKDEV_MODE_PERIODIC:
6b954823	343	if (!cpumask_empty(tick_get_broadcast_mask()))
cd05a1f8	344	tick_broadcast_start_periodic(bc);
6b954823 RR	345	broadcast = cpumask_test_cpu(smp_processor_id(),
6b954823 RR	346	tick_get_broadcast_mask());
cd05a1f8 TG	347	break;
	348	case TICKDEV_MODE_ONESHOT:
	349	broadcast = tick_resume_broadcast_oneshot(bc);
	350	break;
	351	}
6321dd60	352	}
b5f91da0	353	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
6321dd60 TG	354
	355	return broadcast;
	356	}
	357
	358
79bf2bb3 TG	359	#ifdef CONFIG_TICK_ONESHOT
79bf2bb3 TG	360
6b954823 RR	361	/* FIXME: use cpumask_var_t. */
6b954823 RR	362	static DECLARE_BITMAP(tick_broadcast_oneshot_mask, NR_CPUS);
79bf2bb3	363
289f480a	364	/*
6b954823	365	* Exposed for debugging: see timer_list.c
289f480a	366	*/
6b954823	367	struct cpumask *tick_get_broadcast_oneshot_mask(void)
289f480a	368	{
6b954823	369	return to_cpumask(tick_broadcast_oneshot_mask);
289f480a IM	370	}
289f480a IM	371
79bf2bb3 TG	372	static int tick_broadcast_set_event(ktime_t expires, int force)
	373	{
	374	struct clock_event_device *bc = tick_broadcast_device.evtdev;
1fb9b7d2	375
b9a6a235 TG	376	if (bc->mode != CLOCK_EVT_MODE_ONESHOT)
	377	clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
	378
d1748302	379	return clockevents_program_event(bc, expires, force);
79bf2bb3 TG	380	}
79bf2bb3 TG	381
cd05a1f8 TG	382	int tick_resume_broadcast_oneshot(struct clock_event_device *bc)
	383	{
	384	clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
b7e113dc	385	return 0;
cd05a1f8 TG	386	}
cd05a1f8 TG	387
fb02fbc1 TG	388	/*
	389	* Called from irq_enter() when idle was interrupted to reenable the
	390	* per cpu device.
	391	*/
	392	void tick_check_oneshot_broadcast(int cpu)
	393	{
6b954823	394	if (cpumask_test_cpu(cpu, to_cpumask(tick_broadcast_oneshot_mask))) {
fb02fbc1 TG	395	struct tick_device *td = &per_cpu(tick_cpu_device, cpu);
	396
	397	clockevents_set_mode(td->evtdev, CLOCK_EVT_MODE_ONESHOT);
	398	}
	399	}
	400
79bf2bb3 TG	401	/*
	402	* Handle oneshot mode broadcasting
	403	*/
	404	static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
	405	{
	406	struct tick_device *td;
cdc6f27d	407	ktime_t now, next_event;
79bf2bb3 TG	408	int cpu;
79bf2bb3 TG	409
b5f91da0	410	raw_spin_lock(&tick_broadcast_lock);
79bf2bb3 TG	411	again:
79bf2bb3 TG	412	dev->next_event.tv64 = KTIME_MAX;
cdc6f27d	413	next_event.tv64 = KTIME_MAX;
6b954823	414	cpumask_clear(to_cpumask(tmpmask));
79bf2bb3 TG	415	now = ktime_get();
79bf2bb3 TG	416	/* Find all expired events */
6b954823	417	for_each_cpu(cpu, tick_get_broadcast_oneshot_mask()) {
79bf2bb3 TG	418	td = &per_cpu(tick_cpu_device, cpu);
79bf2bb3 TG	419	if (td->evtdev->next_event.tv64 <= now.tv64)
6b954823	420	cpumask_set_cpu(cpu, to_cpumask(tmpmask));
cdc6f27d TG	421	else if (td->evtdev->next_event.tv64 < next_event.tv64)
cdc6f27d TG	422	next_event.tv64 = td->evtdev->next_event.tv64;
79bf2bb3 TG	423	}
	424
	425	/*
cdc6f27d TG	426	* Wakeup the cpus which have an expired event.
cdc6f27d TG	427	*/
6b954823	428	tick_do_broadcast(to_cpumask(tmpmask));
cdc6f27d TG	429
	430	/*
	431	* Two reasons for reprogram:
	432	*
	433	* - The global event did not expire any CPU local
	434	* events. This happens in dyntick mode, as the maximum PIT
	435	* delta is quite small.
	436	*
	437	* - There are pending events on sleeping CPUs which were not
	438	* in the event mask
79bf2bb3	439	*/
cdc6f27d	440	if (next_event.tv64 != KTIME_MAX) {
79bf2bb3	441	/*
cdc6f27d TG	442	* Rearm the broadcast device. If event expired,
cdc6f27d TG	443	* repeat the above
79bf2bb3	444	*/
cdc6f27d	445	if (tick_broadcast_set_event(next_event, 0))
79bf2bb3 TG	446	goto again;
79bf2bb3 TG	447	}
b5f91da0	448	raw_spin_unlock(&tick_broadcast_lock);
79bf2bb3 TG	449	}
	450
	451	/*
	452	* Powerstate information: The system enters/leaves a state, where
	453	* affected devices might stop
	454	*/
	455	void tick_broadcast_oneshot_control(unsigned long reason)
	456	{
	457	struct clock_event_device bc, dev;
	458	struct tick_device *td;
	459	unsigned long flags;
	460	int cpu;
	461
79bf2bb3 TG	462	/*
	463	* Periodic mode does not care about the enter/exit of power
	464	* states
	465	*/
	466	if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
7372b0b1	467	return;
79bf2bb3	468
7372b0b1 AK	469	/*
	470	* We are called with preemtion disabled from the depth of the
	471	* idle code, so we can't be moved away.
	472	*/
79bf2bb3 TG	473	cpu = smp_processor_id();
	474	td = &per_cpu(tick_cpu_device, cpu);
	475	dev = td->evtdev;
	476
	477	if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
7372b0b1 AK	478	return;
	479
	480	bc = tick_broadcast_device.evtdev;
79bf2bb3	481
7372b0b1	482	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
79bf2bb3	483	if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) {
6b954823 RR	484	if (!cpumask_test_cpu(cpu, tick_get_broadcast_oneshot_mask())) {
6b954823 RR	485	cpumask_set_cpu(cpu, tick_get_broadcast_oneshot_mask());
79bf2bb3 TG	486	clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN);
	487	if (dev->next_event.tv64 < bc->next_event.tv64)
	488	tick_broadcast_set_event(dev->next_event, 1);
	489	}
	490	} else {
6b954823 RR	491	if (cpumask_test_cpu(cpu, tick_get_broadcast_oneshot_mask())) {
	492	cpumask_clear_cpu(cpu,
	493	tick_get_broadcast_oneshot_mask());
79bf2bb3 TG	494	clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT);
	495	if (dev->next_event.tv64 != KTIME_MAX)
	496	tick_program_event(dev->next_event, 1);
	497	}
	498	}
b5f91da0	499	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
79bf2bb3 TG	500	}
79bf2bb3 TG	501
5590a536 TG	502	/*
	503	* Reset the one shot broadcast for a cpu
	504	*
	505	* Called with tick_broadcast_lock held
	506	*/
	507	static void tick_broadcast_clear_oneshot(int cpu)
	508	{
6b954823	509	cpumask_clear_cpu(cpu, tick_get_broadcast_oneshot_mask());
5590a536 TG	510	}
5590a536 TG	511
6b954823 RR	512	static void tick_broadcast_init_next_event(struct cpumask *mask,
6b954823 RR	513	ktime_t expires)
7300711e TG	514	{
	515	struct tick_device *td;
	516	int cpu;
	517
5db0e1e9	518	for_each_cpu(cpu, mask) {
7300711e TG	519	td = &per_cpu(tick_cpu_device, cpu);
	520	if (td->evtdev)
	521	td->evtdev->next_event = expires;
	522	}
	523	}
	524
79bf2bb3	525	/**
8dce39c2	526	* tick_broadcast_setup_oneshot - setup the broadcast device
79bf2bb3 TG	527	*/
	528	void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
	529	{
07f4beb0 TG	530	int cpu = smp_processor_id();
07f4beb0 TG	531
9c17bcda TG	532	/* Set it up only once ! */
9c17bcda TG	533	if (bc->event_handler != tick_handle_oneshot_broadcast) {
7300711e	534	int was_periodic = bc->mode == CLOCK_EVT_MODE_PERIODIC;
7300711e	535
9c17bcda	536	bc->event_handler = tick_handle_oneshot_broadcast;
7300711e TG	537
	538	/* Take the do_timer update */
	539	tick_do_timer_cpu = cpu;
	540
	541	/*
	542	* We must be careful here. There might be other CPUs
	543	* waiting for periodic broadcast. We need to set the
	544	* oneshot_mask bits for those and program the
	545	* broadcast device to fire.
	546	*/
6b954823 RR	547	cpumask_copy(to_cpumask(tmpmask), tick_get_broadcast_mask());
	548	cpumask_clear_cpu(cpu, to_cpumask(tmpmask));
	549	cpumask_or(tick_get_broadcast_oneshot_mask(),
	550	tick_get_broadcast_oneshot_mask(),
	551	to_cpumask(tmpmask));
	552
	553	if (was_periodic && !cpumask_empty(to_cpumask(tmpmask))) {
b435092f	554	clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
6b954823 RR	555	tick_broadcast_init_next_event(to_cpumask(tmpmask),
6b954823 RR	556	tick_next_period);
7300711e TG	557	tick_broadcast_set_event(tick_next_period, 1);
	558	} else
	559	bc->next_event.tv64 = KTIME_MAX;
07f4beb0 TG	560	} else {
	561	/*
	562	* The first cpu which switches to oneshot mode sets
	563	* the bit for all other cpus which are in the general
	564	* (periodic) broadcast mask. So the bit is set and
	565	* would prevent the first broadcast enter after this
	566	* to program the bc device.
	567	*/
	568	tick_broadcast_clear_oneshot(cpu);
9c17bcda	569	}
79bf2bb3 TG	570	}
	571
	572	/*
	573	* Select oneshot operating mode for the broadcast device
	574	*/
	575	void tick_broadcast_switch_to_oneshot(void)
	576	{
	577	struct clock_event_device *bc;
	578	unsigned long flags;
	579
b5f91da0	580	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
fa4da365 SS	581
fa4da365 SS	582	tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
79bf2bb3 TG	583	bc = tick_broadcast_device.evtdev;
	584	if (bc)
	585	tick_broadcast_setup_oneshot(bc);
77b0d60c	586
b5f91da0	587	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
79bf2bb3 TG	588	}
	589
	590
	591	/*
	592	* Remove a dead CPU from broadcasting
	593	*/
	594	void tick_shutdown_broadcast_oneshot(unsigned int *cpup)
	595	{
79bf2bb3 TG	596	unsigned long flags;
	597	unsigned int cpu = *cpup;
	598
b5f91da0	599	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
79bf2bb3	600
31d9b393 TG	601	/*
	602	* Clear the broadcast mask flag for the dead cpu, but do not
	603	* stop the broadcast device!
	604	*/
6b954823	605	cpumask_clear_cpu(cpu, tick_get_broadcast_oneshot_mask());
79bf2bb3	606
b5f91da0	607	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
79bf2bb3 TG	608	}
79bf2bb3 TG	609
27ce4cb4 TG	610	/*
	611	* Check, whether the broadcast device is in one shot mode
	612	*/
	613	int tick_broadcast_oneshot_active(void)
	614	{
	615	return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT;
	616	}
	617
3a142a06 TG	618	/*
	619	* Check whether the broadcast device supports oneshot.
	620	*/
	621	bool tick_broadcast_oneshot_available(void)
	622	{
	623	struct clock_event_device *bc = tick_broadcast_device.evtdev;
	624
	625	return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false;
	626	}
	627
79bf2bb3	628	#endif