[deliverable/linux.git] / kernel / rcu / waketorture.c

/*
 * Specific stress-testing of wakeup logic in the presence of hotplug
 * operations.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, you can access it online at
 * http://www.gnu.org/licenses/gpl-2.0.html.
 *
 * Copyright (C) IBM Corporation, 2016
 *
 * Author: Paul E. McKenney <paulmck@us.ibm.com>
 */
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/kthread.h>
#include <linux/err.h>
#include <linux/spinlock.h>
#include <linux/smp.h>
#include <linux/rcupdate.h>
#include <linux/interrupt.h>
#include <linux/sched.h>
#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/completion.h>
#include <linux/moduleparam.h>
#include <linux/percpu.h>
#include <linux/notifier.h>
#include <linux/reboot.h>
#include <linux/freezer.h>
#include <linux/cpu.h>
#include <linux/delay.h>
#include <linux/stat.h>
#include <linux/srcu.h>
#include <linux/slab.h>
#include <linux/trace_clock.h>
#include <asm/byteorder.h>
#include <linux/torture.h>
#include <linux/vmalloc.h>

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com>");


torture_param(int, nwaiters, -1, "Number of timed-wait threads");
torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
torture_param(int, onoff_interval, 0,
	     "Time between CPU hotplugs (jiffies), 0=disable");
torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable.");
torture_param(int, stat_interval, 60,
	     "Number of seconds between stats printk()s");
torture_param(bool, verbose, true,
	     "Enable verbose debugging printk()s");
torture_param(int, wait_duration, 127,
	     "Number of microseconds to wait each iteration");
torture_param(int, wait_grace, 20,
	     "Number of jiffies before complaining about long wait");

static char *torture_type = "sh";
module_param(torture_type, charp, 0444);
MODULE_PARM_DESC(torture_type, "Type of wait to torture (sti, stui, ...)");

static int nrealwaiters;
static struct task_struct **waiter_tasks;
static struct task_struct *stats_task;
static struct task_struct *onoff_task;

/* Yes, these cache-thrash, and it is inherent to the concurrency design. */
static bool *waiter_done;		/* Waiter is done, don't wake. */
static unsigned long *waiter_iter;	/* Number of wait iterations. */
static bool *waiter_cts;		/* Waiter already checked. */
static unsigned long *waiter_kicks;	/* Number of waiter starvations. */
static unsigned long *waiter_ts;	/* Jiffies last run. */
static DEFINE_MUTEX(waiter_mutex);
static DEFINE_PER_CPU(u64, waiter_cputime); /* Nanoseconds. */
static u64 starttime;

static int onoff_cpu = -1;
static long n_offline_attempts;
static long n_offline_successes;
static unsigned long sum_offline;
static int min_offline = -1;
static int max_offline;
static long n_online_attempts;
static long n_online_successes;
static unsigned long sum_online;
static int min_online = -1;
static int max_online;

static int torture_runnable = IS_ENABLED(MODULE);
module_param(torture_runnable, int, 0444);
MODULE_PARM_DESC(torture_runnable, "Start waketorture at boot");

/*
 * Operations vector for selecting different types of tests.
 */

struct wake_torture_ops {
	void (*wait)(void);
	const char *name;
};

static struct wake_torture_ops *cur_ops;

/*
 * Definitions for schedule_hrtimeout() torture testing.
 */

static void wait_schedule_hrtimeout(void)
{
	ktime_t wait = ns_to_ktime(wait_duration * 1000);

	set_current_state(TASK_UNINTERRUPTIBLE);
	schedule_hrtimeout(&wait, HRTIMER_MODE_REL);
}

static struct wake_torture_ops sh_ops = {
	.wait		= wait_schedule_hrtimeout,
	.name		= "sh"
};

/*
 * Definitions for schedule_timeout_interruptible() torture testing.
 */

static void wait_schedule_timeout_interruptible(void)
{
	schedule_timeout_interruptible((wait_duration + 999) / 1000);
}

static struct wake_torture_ops sti_ops = {
	.wait		= wait_schedule_timeout_interruptible,
	.name		= "sti"
};

/*
 * Definitions for schedule_timeout_uninterruptible() torture testing.
 */

static void wait_schedule_timeout_uninterruptible(void)
{
	schedule_timeout_uninterruptible((wait_duration + 999) / 1000);
}

static struct wake_torture_ops stui_ops = {
	.wait		= wait_schedule_timeout_uninterruptible,
	.name		= "stui"
};

/*
 * Has the specified waiter thread run recently?
 */
static bool kthread_ran_recently(int tnum)
{
	smp_mb(); /* Ensure waiter_cts[] read before waiter_ts[].  [A] */
	return time_before(READ_ONCE(waiter_ts[tnum]), jiffies + wait_grace);
}

/*
 * Wakeup torture fake writer kthread.  Repeatedly calls sync, with a random
 * delay between calls.
 */
static int wake_torture_waiter(void *arg)
{
	int i;
	long me = (long)arg;
	u64 ts;

	VERBOSE_TOROUT_STRING("wake_torture_waiter task started");
	set_user_nice(current, MAX_NICE);

	preempt_disable();
	ts = trace_clock_local();
	do {
		waiter_ts[me] = jiffies;
		smp_mb(); /* Ensure waiter_ts[] written before waiter_cts[]. */
			  /* Pairs with [A]. */
		waiter_cts[me] = false;
		__this_cpu_add(waiter_cputime, trace_clock_local() - ts);
		preempt_enable();
		cur_ops->wait();
		preempt_disable();
		ts = trace_clock_local();
		waiter_iter[me]++;
		for (i = 0; i < nrealwaiters; i++) {
			if (waiter_done[i] ||
			    waiter_cts[i] ||
			    kthread_ran_recently(i))
				continue;
			if (!mutex_trylock(&waiter_mutex)) {
				break; /* Keep lock contention to dull roar. */
			} else if (waiter_done[i] ||
				   waiter_cts[i] ||
				   kthread_ran_recently(i)) {
				mutex_unlock(&waiter_mutex);
			} else {
				waiter_cts[i] = true;
				waiter_kicks[i]++;
				pr_alert("%s%s wake_torture_waiter(): P%d (%#lx) failing to awaken!\n",
					 torture_type, TORTURE_FLAG,
					 waiter_tasks[i]->pid,
					 waiter_tasks[i]->state);
				rcu_ftrace_dump(DUMP_ALL);
				wake_up_process(waiter_tasks[i]);
				mutex_unlock(&waiter_mutex);
			}
		}
		__this_cpu_add(waiter_cputime, trace_clock_local() - ts);
		preempt_enable();
		torture_shutdown_absorb("wake_torture_waiter");
		preempt_disable();
		ts = trace_clock_local();
	} while (!torture_must_stop());
	__this_cpu_add(waiter_cputime, trace_clock_local() - ts);
	preempt_enable();
	mutex_lock(&waiter_mutex);
	waiter_done[me] = true;
	mutex_unlock(&waiter_mutex);
	torture_kthread_stopping("wake_torture_waiter");
	return 0;
}

/*
 * Find a hotpluggable CPU and repeatedly take it online and offline.
 */
static int wake_torture_onoff(void *args)
{
	cpumask_var_t cm;
	int cpu;
	int i;

	VERBOSE_TOROUT_STRING("wake_torture_onoff task started");
	if (onoff_holdoff > 0) {
		VERBOSE_TOROUT_STRING("wake_torture_onoff begin holdoff");
		schedule_timeout_interruptible(onoff_holdoff * HZ);
		VERBOSE_TOROUT_STRING("wake_torture_onoff end holdoff");
	}

	/*
	 * Find the last hotpluggable CPU, and affinity the waiter
	 * tasks elsewhere.
	 */
	for_each_online_cpu(cpu) {
		if (cpu_is_hotpluggable(cpu))
			onoff_cpu = cpu;
	}
	if (onoff_cpu < 0) {
		VERBOSE_TOROUT_STRING("wake_torture_onoff: no hotpluggable CPUs!");
		if (shutdown_secs > 0) {
			VERBOSE_TOROUT_STRING("wake_torture_onoff: Shutting down");
			kernel_power_off();
			VERBOSE_TOROUT_STRING("wake_torture_onoff: Survived kernel_power_off()?");
		}
	}
	pr_alert("%s" TORTURE_FLAG " wake_torture_onoff: onoff_cpu: %d\n", torture_type, onoff_cpu);
	if (!zalloc_cpumask_var(&cm, GFP_KERNEL)) {
		VERBOSE_TOROUT_STRING("wake_torture_onoff: Out of memory, no affinity");
	} else {
		cpumask_copy(cm, cpu_online_mask);
		cpumask_clear_cpu(onoff_cpu, cm);
		if (cpumask_weight(cm) == 0)
			cpumask_setall(cm);
		for (i = 0; i < nrealwaiters; i++)
			set_cpus_allowed_ptr(waiter_tasks[i], cm);
	}

	/* Cycle the victim CPU online and offline! */
	while (!torture_must_stop() && onoff_cpu >= 0) {
		if (!torture_offline(onoff_cpu,
				    &n_offline_attempts, &n_offline_successes,
				    &sum_offline, &min_offline, &max_offline))
			torture_online(onoff_cpu,
				       &n_online_attempts, &n_online_successes,
				       &sum_online, &min_online, &max_online);
		schedule_timeout_interruptible(onoff_interval);
	}
	torture_kthread_stopping("wake_torture_onoff");
	return 0;
}

/*
 * Initiate waketorture-specific online-offline handling, which
 * focuses on a single CPU.
 */
static int wake_torture_onoff_init(void)
{
	int ret = 0;

	if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
		return ret;
	if (onoff_interval <= 0)
		return 0;
	ret = torture_create_kthread(wake_torture_onoff, NULL, onoff_task);
	return ret;
}

/*
 * Clean up after waketorture-specific online-offline handling.
 */
static void wake_torture_onoff_cleanup(void)
{
	if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
		return;
	VERBOSE_TOROUT_STRING("Stopping wake_torture_onoff task");
	kthread_stop(onoff_task);
	onoff_task = NULL;
}

/*
 * Print torture statistics.  Caller must ensure that there is only one
 * call to this function at a given time!!!  This is normally accomplished
 * by relying on the module system to only have one copy of the module
 * loaded, and then by giving the wake_torture_stats kthread full control
 * (or the init/cleanup functions when wake_torture_stats thread is not
 * running).
 *
 * Note that some care is required because this can be called once during
 * cleanup processing after a failed startup attempt.
 */
static void
wake_torture_stats_print(void)
{
	int i;
	bool tardy = false;
	u64 timediff;
	u64 timetot;

	if (!waiter_done || !waiter_iter || !waiter_cts ||
	    !waiter_kicks || !waiter_ts) {
		TOROUT_STRING("Partial initialization, no stats print.\n");
		return;
	}
	for (i = 0; i < nrealwaiters; i++)
		if (waiter_kicks[i]) {
			if (!tardy)
				pr_alert("%s" TORTURE_FLAG " Tardy kthreads:",
					 torture_type);
			tardy = true;
			pr_cont("  P%d%c: %lud/%lu",
				waiter_tasks && waiter_tasks[i]
					? waiter_tasks[i]->pid
					: -1,
				"!."[kthread_ran_recently(i)],
				waiter_kicks[i], waiter_iter[i]);
		}
	if (tardy)
		pr_cont("\n");
	else
		TOROUT_STRING(" No tardy kthreads");
	timediff = (trace_clock_global() - starttime) / 1000;
	timetot = 0;
	for_each_possible_cpu(i)
		timetot += READ_ONCE(per_cpu(waiter_cputime, i));
	timetot /= nr_cpu_ids;
	timetot /= timediff;
	pr_alert("%s" TORTURE_FLAG " timediff: %llu utilization: %llu.%llu nr_cpu_ids: %d onoff: %ld/%ld:%ld/%ld %d,%d:%d,%d %lu:%lu (HZ=%d)\n",
		 torture_type, timediff,
		 timetot / 1000ULL, timetot % 1000ULL, nr_cpu_ids,
		 n_online_successes, n_online_attempts,
		 n_offline_successes, n_offline_attempts,
		 min_online, max_online,
		 min_offline, max_offline,
		 sum_online, sum_offline, HZ);
}

/*
 * Periodically prints torture statistics, if periodic statistics printing
 * was specified via the stat_interval module parameter.
 */
static int
wake_torture_stats(void *arg)
{
	VERBOSE_TOROUT_STRING("wake_torture_stats task started");
	do {
		schedule_timeout_interruptible(stat_interval * HZ);
		wake_torture_stats_print();
		torture_shutdown_absorb("wake_torture_stats");
	} while (!torture_must_stop());
	torture_kthread_stopping("wake_torture_stats");
	return 0;
}

static inline void
wake_torture_print_module_parms(struct wake_torture_ops *cur_ops,
				const char *tag)
{
	pr_alert("%s" TORTURE_FLAG
		 "--- %s: nwaiters=%d onoff_holdoff=%d onoff_interval=%d shutdown_secs=%d stat_interval=%d verbose=%d wait_duration=%d wait_grace=%d\n",
		 torture_type, tag,
		 nrealwaiters, onoff_holdoff, onoff_interval,
		 shutdown_secs, stat_interval, verbose,
		 wait_duration, wait_grace);
}

static void
wake_torture_cleanup(void)
{
	int i;
	bool success;

	(void)torture_cleanup_begin();

	if (onoff_task)
		wake_torture_onoff_cleanup();

	if (waiter_tasks) {
		for (i = 0; i < nrealwaiters; i++)
			torture_stop_kthread(wake_torture_waiter,
					     waiter_tasks[i]);
		kfree(waiter_tasks);
	}

	torture_stop_kthread(wake_torture_stats, stats_task);

	wake_torture_stats_print();  /* -After- the stats thread is stopped! */

	success = !!waiter_kicks;
	for (i = 0; i < nrealwaiters; i++)
		if (!success || waiter_kicks[i]) {
			success = false;
			break;
		}

	kfree(waiter_done);
	kfree(waiter_iter);
	kfree(waiter_cts);
	kfree(waiter_kicks);
	kfree(waiter_ts);

	wake_torture_print_module_parms(cur_ops,
					success ? "End of test: SUCCESS"
						: "End of test: FAILURE");
	torture_cleanup_end();
}

static int __init
wake_torture_init(void)
{
	int i;
	int firsterr = 0;
	static struct wake_torture_ops *torture_ops[] = {
		&sh_ops, &sti_ops, &stui_ops
	};

	if (!torture_init_begin(torture_type, verbose, &torture_runnable))
		return -EBUSY;
	starttime = trace_clock_global();

	/* Process args and tell the world that the torturer is on the job. */
	for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
		cur_ops = torture_ops[i];
		if (strcmp(torture_type, cur_ops->name) == 0)
			break;
	}
	if (i == ARRAY_SIZE(torture_ops)) {
		pr_alert("wake-torture: invalid torture type: \"%s\"\n",
			 torture_type);
		pr_alert("wake-torture types:");
		for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
			pr_alert(" %s", torture_ops[i]->name);
		pr_alert("\n");
		firsterr = -EINVAL;
		goto unwind;
	}

	if (nwaiters >= 0) {
		nrealwaiters = nwaiters;
	} else {
		nrealwaiters = num_online_cpus() - 2 - nwaiters;
		if (nrealwaiters <= 0)
			nrealwaiters = 1;
	}
	wake_torture_print_module_parms(cur_ops, "Start of test");

	/* Initialize the statistics so that each run gets its own numbers. */

	waiter_done = kcalloc(nrealwaiters, sizeof(*waiter_done), GFP_KERNEL);
	waiter_iter = kcalloc(nrealwaiters, sizeof(*waiter_iter), GFP_KERNEL);
	waiter_cts = kcalloc(nrealwaiters, sizeof(*waiter_cts), GFP_KERNEL);
	waiter_kicks = kcalloc(nrealwaiters, sizeof(*waiter_kicks), GFP_KERNEL);
	waiter_ts = kcalloc(nrealwaiters, sizeof(*waiter_ts), GFP_KERNEL);
	if (!waiter_done || !waiter_iter || !waiter_cts || !waiter_kicks ||
	    !waiter_ts) {
		VERBOSE_TOROUT_ERRSTRING("out of memory");
		firsterr = -ENOMEM;
		goto unwind;
	}

	/* Start up the kthreads. */

	waiter_tasks = kcalloc(nrealwaiters, sizeof(waiter_tasks[0]),
			       GFP_KERNEL);
	if (!waiter_tasks) {
		VERBOSE_TOROUT_ERRSTRING("out of memory");
		firsterr = -ENOMEM;
		goto unwind;
	}
	for (i = 0; i < nrealwaiters; i++) {
		firsterr = torture_create_kthread(wake_torture_waiter,
						  NULL, waiter_tasks[i]);
		if (firsterr)
			goto unwind;
	}
	if (stat_interval > 0) {
		firsterr = torture_create_kthread(wake_torture_stats, NULL,
						  stats_task);
		if (firsterr)
			goto unwind;
	}
	firsterr = torture_shutdown_init(shutdown_secs, wake_torture_cleanup);
	if (firsterr)
		goto unwind;
	firsterr = wake_torture_onoff_init();
	if (firsterr)
		goto unwind;
	torture_init_end();
	return 0;

unwind:
	torture_init_end();
	wake_torture_cleanup();
	return firsterr;
}

module_init(wake_torture_init);
module_exit(wake_torture_cleanup);
Commit	Line	Data
7a0f68f8 PM	1	/*
	2	* Specific stress-testing of wakeup logic in the presence of hotplug
	3	* operations.
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License as published by
	7	* the Free Software Foundation; either version 2 of the License, or
	8	* (at your option) any later version.
	9	*
	10	* This program is distributed in the hope that it will be useful,
	11	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	* GNU General Public License for more details.
	14	*
	15	* You should have received a copy of the GNU General Public License
	16	* along with this program; if not, you can access it online at
	17	* http://www.gnu.org/licenses/gpl-2.0.html.
	18	*
	19	* Copyright (C) IBM Corporation, 2016
	20	*
	21	* Author: Paul E. McKenney <paulmck@us.ibm.com>
	22	*/
	23	#include <linux/types.h>
	24	#include <linux/kernel.h>
	25	#include <linux/init.h>
	26	#include <linux/module.h>
	27	#include <linux/kthread.h>
	28	#include <linux/err.h>
	29	#include <linux/spinlock.h>
	30	#include <linux/smp.h>
	31	#include <linux/rcupdate.h>
	32	#include <linux/interrupt.h>
	33	#include <linux/sched.h>
	34	#include <linux/atomic.h>
	35	#include <linux/bitops.h>
	36	#include <linux/completion.h>
	37	#include <linux/moduleparam.h>
	38	#include <linux/percpu.h>
	39	#include <linux/notifier.h>
	40	#include <linux/reboot.h>
	41	#include <linux/freezer.h>
	42	#include <linux/cpu.h>
	43	#include <linux/delay.h>
	44	#include <linux/stat.h>
	45	#include <linux/srcu.h>
	46	#include <linux/slab.h>
	47	#include <linux/trace_clock.h>
	48	#include <asm/byteorder.h>
	49	#include <linux/torture.h>
	50	#include <linux/vmalloc.h>
	51
	52	MODULE_LICENSE("GPL");
	53	MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com>");
	54
	55
	56	torture_param(int, nwaiters, -1, "Number of timed-wait threads");
	57	torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
	58	torture_param(int, onoff_interval, 0,
	59	"Time between CPU hotplugs (jiffies), 0=disable");
	60	torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable.");
	61	torture_param(int, stat_interval, 60,
	62	"Number of seconds between stats printk()s");
	63	torture_param(bool, verbose, true,
	64	"Enable verbose debugging printk()s");
b39611dc PM	65	torture_param(int, wait_duration, 127,
b39611dc PM	66	"Number of microseconds to wait each iteration");
7a0f68f8 PM	67	torture_param(int, wait_grace, 20,
	68	"Number of jiffies before complaining about long wait");
	69
b39611dc	70	static char *torture_type = "sh";
7a0f68f8 PM	71	module_param(torture_type, charp, 0444);
	72	MODULE_PARM_DESC(torture_type, "Type of wait to torture (sti, stui, ...)");
	73
	74	static int nrealwaiters;
	75	static struct task_struct **waiter_tasks;
	76	static struct task_struct *stats_task;
699fe479	77	static struct task_struct *onoff_task;
7a0f68f8 PM	78
	79	/* Yes, these cache-thrash, and it is inherent to the concurrency design. */
	80	static bool waiter_done; / Waiter is done, don't wake. */
	81	static unsigned long waiter_iter; / Number of wait iterations. */
	82	static bool waiter_cts; / Waiter already checked. */
	83	static unsigned long waiter_kicks; / Number of waiter starvations. */
	84	static unsigned long waiter_ts; / Jiffies last run. */
737c981f PM	85	static DEFINE_MUTEX(waiter_mutex);
	86	static DEFINE_PER_CPU(u64, waiter_cputime); /* Nanoseconds. */
	87	static u64 starttime;
7a0f68f8	88
699fe479 PM	89	static int onoff_cpu = -1;
	90	static long n_offline_attempts;
	91	static long n_offline_successes;
	92	static unsigned long sum_offline;
	93	static int min_offline = -1;
	94	static int max_offline;
	95	static long n_online_attempts;
	96	static long n_online_successes;
	97	static unsigned long sum_online;
	98	static int min_online = -1;
	99	static int max_online;
	100
7a0f68f8 PM	101	static int torture_runnable = IS_ENABLED(MODULE);
	102	module_param(torture_runnable, int, 0444);
	103	MODULE_PARM_DESC(torture_runnable, "Start waketorture at boot");
	104
	105	/*
	106	* Operations vector for selecting different types of tests.
	107	*/
	108
	109	struct wake_torture_ops {
b39611dc	110	void (*wait)(void);
7a0f68f8 PM	111	const char *name;
	112	};
	113
	114	static struct wake_torture_ops *cur_ops;
	115
b39611dc PM	116	/*
	117	* Definitions for schedule_hrtimeout() torture testing.
	118	*/
	119
	120	static void wait_schedule_hrtimeout(void)
	121	{
	122	ktime_t wait = ns_to_ktime(wait_duration * 1000);
	123
	124	set_current_state(TASK_UNINTERRUPTIBLE);
	125	schedule_hrtimeout(&wait, HRTIMER_MODE_REL);
	126	}
	127
	128	static struct wake_torture_ops sh_ops = {
	129	.wait = wait_schedule_hrtimeout,
	130	.name = "sh"
	131	};
	132
7a0f68f8 PM	133	/*
	134	* Definitions for schedule_timeout_interruptible() torture testing.
	135	*/
	136
b39611dc PM	137	static void wait_schedule_timeout_interruptible(void)
	138	{
	139	schedule_timeout_interruptible((wait_duration + 999) / 1000);
	140	}
	141
7a0f68f8	142	static struct wake_torture_ops sti_ops = {
b39611dc	143	.wait = wait_schedule_timeout_interruptible,
7a0f68f8 PM	144	.name = "sti"
	145	};
	146
	147	/*
	148	* Definitions for schedule_timeout_uninterruptible() torture testing.
	149	*/
	150
b39611dc PM	151	static void wait_schedule_timeout_uninterruptible(void)
	152	{
	153	schedule_timeout_uninterruptible((wait_duration + 999) / 1000);
	154	}
	155
7a0f68f8	156	static struct wake_torture_ops stui_ops = {
b39611dc	157	.wait = wait_schedule_timeout_uninterruptible,
7a0f68f8 PM	158	.name = "stui"
	159	};
	160
	161	/*
	162	* Has the specified waiter thread run recently?
	163	*/
	164	static bool kthread_ran_recently(int tnum)
	165	{
	166	smp_mb(); /* Ensure waiter_cts[] read before waiter_ts[]. [A] */
	167	return time_before(READ_ONCE(waiter_ts[tnum]), jiffies + wait_grace);
	168	}
	169
	170	/*
	171	* Wakeup torture fake writer kthread. Repeatedly calls sync, with a random
	172	* delay between calls.
	173	*/
	174	static int wake_torture_waiter(void *arg)
	175	{
	176	int i;
	177	long me = (long)arg;
737c981f	178	u64 ts;
7a0f68f8 PM	179
	180	VERBOSE_TOROUT_STRING("wake_torture_waiter task started");
	181	set_user_nice(current, MAX_NICE);
	182
737c981f PM	183	preempt_disable();
737c981f PM	184	ts = trace_clock_local();
7a0f68f8 PM	185	do {
	186	waiter_ts[me] = jiffies;
	187	smp_mb(); /* Ensure waiter_ts[] written before waiter_cts[]. */
	188	/* Pairs with [A]. */
	189	waiter_cts[me] = false;
737c981f PM	190	__this_cpu_add(waiter_cputime, trace_clock_local() - ts);
737c981f PM	191	preempt_enable();
b39611dc	192	cur_ops->wait();
737c981f PM	193	preempt_disable();
737c981f PM	194	ts = trace_clock_local();
7a0f68f8 PM	195	waiter_iter[me]++;
	196	for (i = 0; i < nrealwaiters; i++) {
	197	if (waiter_done[i] \|\|
	198	waiter_cts[i] \|\|
	199	kthread_ran_recently(i))
	200	continue;
	201	if (!mutex_trylock(&waiter_mutex)) {
	202	break; /* Keep lock contention to dull roar. */
	203	} else if (waiter_done[i] \|\|
	204	waiter_cts[i] \|\|
	205	kthread_ran_recently(i)) {
	206	mutex_unlock(&waiter_mutex);
	207	} else {
	208	waiter_cts[i] = true;
	209	waiter_kicks[i]++;
b39611dc PM	210	pr_alert("%s%s wake_torture_waiter(): P%d (%#lx) failing to awaken!\n",
	211	torture_type, TORTURE_FLAG,
	212	waiter_tasks[i]->pid,
	213	waiter_tasks[i]->state);
7a0f68f8 PM	214	rcu_ftrace_dump(DUMP_ALL);
	215	wake_up_process(waiter_tasks[i]);
	216	mutex_unlock(&waiter_mutex);
	217	}
	218	}
737c981f PM	219	__this_cpu_add(waiter_cputime, trace_clock_local() - ts);
737c981f PM	220	preempt_enable();
7a0f68f8	221	torture_shutdown_absorb("wake_torture_waiter");
737c981f PM	222	preempt_disable();
737c981f PM	223	ts = trace_clock_local();
7a0f68f8	224	} while (!torture_must_stop());
737c981f PM	225	__this_cpu_add(waiter_cputime, trace_clock_local() - ts);
737c981f PM	226	preempt_enable();
7a0f68f8 PM	227	mutex_lock(&waiter_mutex);
	228	waiter_done[me] = true;
	229	mutex_unlock(&waiter_mutex);
	230	torture_kthread_stopping("wake_torture_waiter");
	231	return 0;
	232	}
	233
699fe479 PM	234	/*
	235	* Find a hotpluggable CPU and repeatedly take it online and offline.
	236	*/
	237	static int wake_torture_onoff(void *args)
	238	{
034e56af	239	cpumask_var_t cm;
699fe479	240	int cpu;
034e56af	241	int i;
699fe479 PM	242
	243	VERBOSE_TOROUT_STRING("wake_torture_onoff task started");
	244	if (onoff_holdoff > 0) {
	245	VERBOSE_TOROUT_STRING("wake_torture_onoff begin holdoff");
	246	schedule_timeout_interruptible(onoff_holdoff * HZ);
	247	VERBOSE_TOROUT_STRING("wake_torture_onoff end holdoff");
	248	}
034e56af PM	249
	250	/*
	251	* Find the last hotpluggable CPU, and affinity the waiter
	252	* tasks elsewhere.
	253	*/
699fe479 PM	254	for_each_online_cpu(cpu) {
	255	if (cpu_is_hotpluggable(cpu))
	256	onoff_cpu = cpu;
	257	}
d9fce809	258	if (onoff_cpu < 0) {
699fe479	259	VERBOSE_TOROUT_STRING("wake_torture_onoff: no hotpluggable CPUs!");
d9fce809 PM	260	if (shutdown_secs > 0) {
	261	VERBOSE_TOROUT_STRING("wake_torture_onoff: Shutting down");
	262	kernel_power_off();
	263	VERBOSE_TOROUT_STRING("wake_torture_onoff: Survived kernel_power_off()?");
	264	}
	265	}
	266	pr_alert("%s" TORTURE_FLAG " wake_torture_onoff: onoff_cpu: %d\n", torture_type, onoff_cpu);
034e56af PM	267	if (!zalloc_cpumask_var(&cm, GFP_KERNEL)) {
	268	VERBOSE_TOROUT_STRING("wake_torture_onoff: Out of memory, no affinity");
	269	} else {
	270	cpumask_copy(cm, cpu_online_mask);
	271	cpumask_clear_cpu(onoff_cpu, cm);
	272	if (cpumask_weight(cm) == 0)
	273	cpumask_setall(cm);
	274	for (i = 0; i < nrealwaiters; i++)
	275	set_cpus_allowed_ptr(waiter_tasks[i], cm);
	276	}
	277
	278	/* Cycle the victim CPU online and offline! */
699fe479 PM	279	while (!torture_must_stop() && onoff_cpu >= 0) {
	280	if (!torture_offline(onoff_cpu,
	281	&n_offline_attempts, &n_offline_successes,
	282	&sum_offline, &min_offline, &max_offline))
	283	torture_online(onoff_cpu,
	284	&n_online_attempts, &n_online_successes,
	285	&sum_online, &min_online, &max_online);
	286	schedule_timeout_interruptible(onoff_interval);
	287	}
	288	torture_kthread_stopping("wake_torture_onoff");
	289	return 0;
	290	}
	291
	292	/*
	293	* Initiate waketorture-specific online-offline handling, which
	294	* focuses on a single CPU.
	295	*/
	296	static int wake_torture_onoff_init(void)
	297	{
	298	int ret = 0;
	299
	300	if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
	301	return ret;
	302	if (onoff_interval <= 0)
	303	return 0;
	304	ret = torture_create_kthread(wake_torture_onoff, NULL, onoff_task);
	305	return ret;
	306	}
	307
	308	/*
	309	* Clean up after waketorture-specific online-offline handling.
	310	*/
	311	static void wake_torture_onoff_cleanup(void)
	312	{
	313	if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
	314	return;
	315	VERBOSE_TOROUT_STRING("Stopping wake_torture_onoff task");
	316	kthread_stop(onoff_task);
	317	onoff_task = NULL;
	318	}
	319
7a0f68f8 PM	320	/*
	321	* Print torture statistics. Caller must ensure that there is only one
	322	* call to this function at a given time!!! This is normally accomplished
	323	* by relying on the module system to only have one copy of the module
	324	* loaded, and then by giving the wake_torture_stats kthread full control
	325	* (or the init/cleanup functions when wake_torture_stats thread is not
	326	* running).
	327	*
	328	* Note that some care is required because this can be called once during
	329	* cleanup processing after a failed startup attempt.
	330	*/
	331	static void
	332	wake_torture_stats_print(void)
	333	{
	334	int i;
	335	bool tardy = false;
737c981f PM	336	u64 timediff;
737c981f PM	337	u64 timetot;
7a0f68f8 PM	338
	339	if (!waiter_done \|\| !waiter_iter \|\| !waiter_cts \|\|
	340	!waiter_kicks \|\| !waiter_ts) {
	341	TOROUT_STRING("Partial initialization, no stats print.\n");
	342	return;
	343	}
	344	for (i = 0; i < nrealwaiters; i++)
	345	if (waiter_kicks[i]) {
	346	if (!tardy)
	347	pr_alert("%s" TORTURE_FLAG " Tardy kthreads:",
	348	torture_type);
	349	tardy = true;
	350	pr_cont(" P%d%c: %lud/%lu",
	351	waiter_tasks && waiter_tasks[i]
	352	? waiter_tasks[i]->pid
	353	: -1,
	354	"!."[kthread_ran_recently(i)],
	355	waiter_kicks[i], waiter_iter[i]);
	356	}
	357	if (tardy)
	358	pr_cont("\n");
	359	else
	360	TOROUT_STRING(" No tardy kthreads");
737c981f PM	361	timediff = (trace_clock_global() - starttime) / 1000;
	362	timetot = 0;
	363	for_each_possible_cpu(i)
	364	timetot += READ_ONCE(per_cpu(waiter_cputime, i));
	365	timetot /= nr_cpu_ids;
	366	timetot /= timediff;
699fe479	367	pr_alert("%s" TORTURE_FLAG " timediff: %llu utilization: %llu.%llu nr_cpu_ids: %d onoff: %ld/%ld:%ld/%ld %d,%d:%d,%d %lu:%lu (HZ=%d)\n",
737c981f	368	torture_type, timediff,
699fe479 PM	369	timetot / 1000ULL, timetot % 1000ULL, nr_cpu_ids,
	370	n_online_successes, n_online_attempts,
	371	n_offline_successes, n_offline_attempts,
	372	min_online, max_online,
	373	min_offline, max_offline,
	374	sum_online, sum_offline, HZ);
7a0f68f8 PM	375	}
	376
	377	/*
	378	* Periodically prints torture statistics, if periodic statistics printing
	379	* was specified via the stat_interval module parameter.
	380	*/
	381	static int
	382	wake_torture_stats(void *arg)
	383	{
	384	VERBOSE_TOROUT_STRING("wake_torture_stats task started");
	385	do {
	386	schedule_timeout_interruptible(stat_interval * HZ);
	387	wake_torture_stats_print();
	388	torture_shutdown_absorb("wake_torture_stats");
	389	} while (!torture_must_stop());
	390	torture_kthread_stopping("wake_torture_stats");
	391	return 0;
	392	}
	393
	394	static inline void
	395	wake_torture_print_module_parms(struct wake_torture_ops *cur_ops,
	396	const char *tag)
	397	{
	398	pr_alert("%s" TORTURE_FLAG
	399	"--- %s: nwaiters=%d onoff_holdoff=%d onoff_interval=%d shutdown_secs=%d stat_interval=%d verbose=%d wait_duration=%d wait_grace=%d\n",
	400	torture_type, tag,
	401	nrealwaiters, onoff_holdoff, onoff_interval,
	402	shutdown_secs, stat_interval, verbose,
	403	wait_duration, wait_grace);
	404	}
	405
	406	static void
	407	wake_torture_cleanup(void)
	408	{
	409	int i;
	410	bool success;
	411
	412	(void)torture_cleanup_begin();
	413
699fe479 PM	414	if (onoff_task)
	415	wake_torture_onoff_cleanup();
	416
7a0f68f8 PM	417	if (waiter_tasks) {
	418	for (i = 0; i < nrealwaiters; i++)
	419	torture_stop_kthread(wake_torture_waiter,
	420	waiter_tasks[i]);
	421	kfree(waiter_tasks);
	422	}
	423
	424	torture_stop_kthread(wake_torture_stats, stats_task);
	425
	426	wake_torture_stats_print(); /* -After- the stats thread is stopped! */
	427
	428	success = !!waiter_kicks;
	429	for (i = 0; i < nrealwaiters; i++)
	430	if (!success \|\| waiter_kicks[i]) {
	431	success = false;
	432	break;
	433	}
	434
	435	kfree(waiter_done);
	436	kfree(waiter_iter);
	437	kfree(waiter_cts);
	438	kfree(waiter_kicks);
	439	kfree(waiter_ts);
	440
	441	wake_torture_print_module_parms(cur_ops,
	442	success ? "End of test: SUCCESS"
	443	: "End of test: FAILURE");
	444	torture_cleanup_end();
	445	}
	446
	447	static int __init
	448	wake_torture_init(void)
	449	{
	450	int i;
	451	int firsterr = 0;
b39611dc PM	452	static struct wake_torture_ops *torture_ops[] = {
	453	&sh_ops, &sti_ops, &stui_ops
	454	};
7a0f68f8 PM	455
	456	if (!torture_init_begin(torture_type, verbose, &torture_runnable))
	457	return -EBUSY;
737c981f	458	starttime = trace_clock_global();
7a0f68f8 PM	459
	460	/* Process args and tell the world that the torturer is on the job. */
	461	for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
	462	cur_ops = torture_ops[i];
	463	if (strcmp(torture_type, cur_ops->name) == 0)
	464	break;
	465	}
	466	if (i == ARRAY_SIZE(torture_ops)) {
	467	pr_alert("wake-torture: invalid torture type: \"%s\"\n",
	468	torture_type);
	469	pr_alert("wake-torture types:");
	470	for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
	471	pr_alert(" %s", torture_ops[i]->name);
	472	pr_alert("\n");
	473	firsterr = -EINVAL;
	474	goto unwind;
	475	}
	476
	477	if (nwaiters >= 0) {
	478	nrealwaiters = nwaiters;
	479	} else {
	480	nrealwaiters = num_online_cpus() - 2 - nwaiters;
	481	if (nrealwaiters <= 0)
	482	nrealwaiters = 1;
	483	}
	484	wake_torture_print_module_parms(cur_ops, "Start of test");
	485
	486	/* Initialize the statistics so that each run gets its own numbers. */
	487
	488	waiter_done = kcalloc(nrealwaiters, sizeof(*waiter_done), GFP_KERNEL);
	489	waiter_iter = kcalloc(nrealwaiters, sizeof(*waiter_iter), GFP_KERNEL);
	490	waiter_cts = kcalloc(nrealwaiters, sizeof(*waiter_cts), GFP_KERNEL);
	491	waiter_kicks = kcalloc(nrealwaiters, sizeof(*waiter_kicks), GFP_KERNEL);
	492	waiter_ts = kcalloc(nrealwaiters, sizeof(*waiter_ts), GFP_KERNEL);
	493	if (!waiter_done \|\| !waiter_iter \|\| !waiter_cts \|\| !waiter_kicks \|\|
	494	!waiter_ts) {
	495	VERBOSE_TOROUT_ERRSTRING("out of memory");
	496	firsterr = -ENOMEM;
	497	goto unwind;
	498	}
	499
	500	/* Start up the kthreads. */
	501
	502	waiter_tasks = kcalloc(nrealwaiters, sizeof(waiter_tasks[0]),
	503	GFP_KERNEL);
	504	if (!waiter_tasks) {
	505	VERBOSE_TOROUT_ERRSTRING("out of memory");
	506	firsterr = -ENOMEM;
	507	goto unwind;
	508	}
	509	for (i = 0; i < nrealwaiters; i++) {
	510	firsterr = torture_create_kthread(wake_torture_waiter,
	511	NULL, waiter_tasks[i]);
	512	if (firsterr)
	513	goto unwind;
	514	}
	515	if (stat_interval > 0) {
	516	firsterr = torture_create_kthread(wake_torture_stats, NULL,
	517	stats_task);
	518	if (firsterr)
	519	goto unwind;
	520	}
	521	firsterr = torture_shutdown_init(shutdown_secs, wake_torture_cleanup);
	522	if (firsterr)
523	goto unwind;
699fe479	524	firsterr = wake_torture_onoff_init();
7a0f68f8 PM	525	if (firsterr)
	526	goto unwind;
	527	torture_init_end();
	528	return 0;
	529
	530	unwind:
	531	torture_init_end();
	532	wake_torture_cleanup();
	533	return firsterr;
	534	}
	535
	536	module_init(wake_torture_init);
	537	module_exit(wake_torture_cleanup);