kernel/rcutorture.c

   1 /*
   2  * Read-Copy Update module-based torture test facility
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17  *
  18  * Copyright (C) IBM Corporation, 2005
  19  *
  20  * Authors: Paul E. McKenney <paulmck@us.ibm.com>
  21  *
  22  * See also:  Documentation/RCU/torture.txt
  23  */
  24 #include <linux/types.h>
  25 #include <linux/kernel.h>
  26 #include <linux/init.h>
  27 #include <linux/module.h>
  28 #include <linux/kthread.h>
  29 #include <linux/err.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/smp.h>
  32 #include <linux/rcupdate.h>
  33 #include <linux/interrupt.h>
  34 #include <linux/sched.h>
  35 #include <asm/atomic.h>
  36 #include <linux/bitops.h>
  37 #include <linux/module.h>
  38 #include <linux/completion.h>
  39 #include <linux/moduleparam.h>
  40 #include <linux/percpu.h>
  41 #include <linux/notifier.h>
  42 #include <linux/cpu.h>
  43 #include <linux/random.h>
  44 #include <linux/delay.h>
  45 #include <linux/byteorder/swabb.h>
  46 #include <linux/stat.h>
  47 #include <linux/srcu.h>
  48
  49 MODULE_LICENSE("GPL");
  50
  51 static int nreaders = -1;       /* # reader threads, defaults to 2*ncpus */
  52 static int stat_interval;       /* Interval between stats, in seconds. */
  53                                 /*  Defaults to "only at end of test". */
  54 static int verbose;             /* Print more debug info. */
  55 static int test_no_idle_hz;     /* Test RCU's support for tickless idle CPUs. */
  56 static int shuffle_interval = 5; /* Interval between shuffles (in sec)*/
  57 static char *torture_type = "rcu"; /* What to torture: rcu, srcu. */
  58
  59 module_param(nreaders, int, 0);
  60 MODULE_PARM_DESC(nreaders, "Number of RCU reader threads");
  61 module_param(stat_interval, int, 0);
  62 MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s");
  63 module_param(verbose, bool, 0);
  64 MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s");
  65 module_param(test_no_idle_hz, bool, 0);
  66 MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs");
  67 module_param(shuffle_interval, int, 0);
  68 MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles");
  69 module_param(torture_type, charp, 0);
  70 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, srcu)");
  71
  72 #define TORTURE_FLAG "-torture:"
  73 #define PRINTK_STRING(s) \
  74         do { printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)
  75 #define VERBOSE_PRINTK_STRING(s) \
  76         do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)
  77 #define VERBOSE_PRINTK_ERRSTRING(s) \
  78         do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG "!!! " s "\n", torture_type); } while (0)
  79
  80 static char printk_buf[4096];
  81
  82 static int nrealreaders;
  83 static struct task_struct *writer_task;
  84 static struct task_struct **reader_tasks;
  85 static struct task_struct *stats_task;
  86 static struct task_struct *shuffler_task;
  87
  88 #define RCU_TORTURE_PIPE_LEN 10
  89
  90 struct rcu_torture {
  91         struct rcu_head rtort_rcu;
  92         int rtort_pipe_count;
  93         struct list_head rtort_free;
  94         int rtort_mbtest;
  95 };
  96
  97 static int fullstop = 0;        /* stop generating callbacks at test end. */
  98 static LIST_HEAD(rcu_torture_freelist);
  99 static struct rcu_torture *rcu_torture_current = NULL;
 100 static long rcu_torture_current_version = 0;
 101 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
 102 static DEFINE_SPINLOCK(rcu_torture_lock);
 103 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count) =
 104         { 0 };
 105 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch) =
 106         { 0 };
 107 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
 108 static atomic_t n_rcu_torture_alloc;
 109 static atomic_t n_rcu_torture_alloc_fail;
 110 static atomic_t n_rcu_torture_free;
 111 static atomic_t n_rcu_torture_mberror;
 112 static atomic_t n_rcu_torture_error;
 113
 114 /*
 115  * Allocate an element from the rcu_tortures pool.
 116  */
 117 static struct rcu_torture *
 118 rcu_torture_alloc(void)
 119 {
 120         struct list_head *p;
 121
 122         spin_lock_bh(&rcu_torture_lock);
 123         if (list_empty(&rcu_torture_freelist)) {
 124                 atomic_inc(&n_rcu_torture_alloc_fail);
 125                 spin_unlock_bh(&rcu_torture_lock);
 126                 return NULL;
 127         }
 128         atomic_inc(&n_rcu_torture_alloc);
 129         p = rcu_torture_freelist.next;
 130         list_del_init(p);
 131         spin_unlock_bh(&rcu_torture_lock);
 132         return container_of(p, struct rcu_torture, rtort_free);
 133 }
 134
 135 /*
 136  * Free an element to the rcu_tortures pool.
 137  */
 138 static void
 139 rcu_torture_free(struct rcu_torture *p)
 140 {
 141         atomic_inc(&n_rcu_torture_free);
 142         spin_lock_bh(&rcu_torture_lock);
 143         list_add_tail(&p->rtort_free, &rcu_torture_freelist);
 144         spin_unlock_bh(&rcu_torture_lock);
 145 }
 146
 147 struct rcu_random_state {
 148         unsigned long rrs_state;
 149         unsigned long rrs_count;
 150 };
 151
 152 #define RCU_RANDOM_MULT 39916801  /* prime */
 153 #define RCU_RANDOM_ADD  479001701 /* prime */
 154 #define RCU_RANDOM_REFRESH 10000
 155
 156 #define DEFINE_RCU_RANDOM(name) struct rcu_random_state name = { 0, 0 }
 157
 158 /*
 159  * Crude but fast random-number generator.  Uses a linear congruential
 160  * generator, with occasional help from get_random_bytes().
 161  */
 162 static long
 163 rcu_random(struct rcu_random_state *rrsp)
 164 {
 165         long refresh;
 166
 167         if (--rrsp->rrs_count < 0) {
 168                 get_random_bytes(&refresh, sizeof(refresh));
 169                 rrsp->rrs_state += refresh;
 170                 rrsp->rrs_count = RCU_RANDOM_REFRESH;
 171         }
 172         rrsp->rrs_state = rrsp->rrs_state * RCU_RANDOM_MULT + RCU_RANDOM_ADD;
 173         return swahw32(rrsp->rrs_state);
 174 }
 175
 176 /*
 177  * Operations vector for selecting different types of tests.
 178  */
 179
 180 struct rcu_torture_ops {
 181         void (*init)(void);
 182         void (*cleanup)(void);
 183         int (*readlock)(void);
 184         void (*readdelay)(struct rcu_random_state *rrsp);
 185         void (*readunlock)(int idx);
 186         int (*completed)(void);
 187         void (*deferredfree)(struct rcu_torture *p);
 188         int (*stats)(char *page);
 189         char *name;
 190 };
 191 static struct rcu_torture_ops *cur_ops = NULL;
 192
 193 /*
 194  * Definitions for rcu torture testing.
 195  */
 196
 197 static int rcu_torture_read_lock(void) __acquires(RCU)
 198 {
 199         rcu_read_lock();
 200         return 0;
 201 }
 202
 203 static void rcu_read_delay(struct rcu_random_state *rrsp)
 204 {
 205         long delay;
 206         const long longdelay = 200;
 207
 208         /* We want there to be long-running readers, but not all the time. */
 209
 210         delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay);
 211         if (!delay)
 212                 udelay(longdelay);
 213 }
 214
 215 static void rcu_torture_read_unlock(int idx) __releases(RCU)
 216 {
 217         rcu_read_unlock();
 218 }
 219
 220 static int rcu_torture_completed(void)
 221 {
 222         return rcu_batches_completed();
 223 }
 224
 225 static void
 226 rcu_torture_cb(struct rcu_head *p)
 227 {
 228         int i;
 229         struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);
 230
 231         if (fullstop) {
 232                 /* Test is ending, just drop callbacks on the floor. */
 233                 /* The next initialization will pick up the pieces. */
 234                 return;
 235         }
 236         i = rp->rtort_pipe_count;
 237         if (i > RCU_TORTURE_PIPE_LEN)
 238                 i = RCU_TORTURE_PIPE_LEN;
 239         atomic_inc(&rcu_torture_wcount[i]);
 240         if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
 241                 rp->rtort_mbtest = 0;
 242                 rcu_torture_free(rp);
 243         } else
 244                 cur_ops->deferredfree(rp);
 245 }
 246
 247 static void rcu_torture_deferred_free(struct rcu_torture *p)
 248 {
 249         call_rcu(&p->rtort_rcu, rcu_torture_cb);
 250 }
 251
 252 static struct rcu_torture_ops rcu_ops = {
 253         .init = NULL,
 254         .cleanup = NULL,
 255         .readlock = rcu_torture_read_lock,
 256         .readdelay = rcu_read_delay,
 257         .readunlock = rcu_torture_read_unlock,
 258         .completed = rcu_torture_completed,
 259         .deferredfree = rcu_torture_deferred_free,
 260         .stats = NULL,
 261         .name = "rcu"
 262 };
 263
 264 /*
 265  * Definitions for rcu_bh torture testing.
 266  */
 267
 268 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH)
 269 {
 270         rcu_read_lock_bh();
 271         return 0;
 272 }
 273
 274 static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH)
 275 {
 276         rcu_read_unlock_bh();
 277 }
 278
 279 static int rcu_bh_torture_completed(void)
 280 {
 281         return rcu_batches_completed_bh();
 282 }
 283
 284 static void rcu_bh_torture_deferred_free(struct rcu_torture *p)
 285 {
 286         call_rcu_bh(&p->rtort_rcu, rcu_torture_cb);
 287 }
 288
 289 static struct rcu_torture_ops rcu_bh_ops = {
 290         .init = NULL,
 291         .cleanup = NULL,
 292         .readlock = rcu_bh_torture_read_lock,
 293         .readdelay = rcu_read_delay,  /* just reuse rcu's version. */
 294         .readunlock = rcu_bh_torture_read_unlock,
 295         .completed = rcu_bh_torture_completed,
 296         .deferredfree = rcu_bh_torture_deferred_free,
 297         .stats = NULL,
 298         .name = "rcu_bh"
 299 };
 300
 301 /*
 302  * Definitions for srcu torture testing.
 303  */
 304
 305 static struct srcu_struct srcu_ctl;
 306 static struct list_head srcu_removed;
 307
 308 static void srcu_torture_init(void)
 309 {
 310         init_srcu_struct(&srcu_ctl);
 311         INIT_LIST_HEAD(&srcu_removed);
 312 }
 313
 314 static void srcu_torture_cleanup(void)
 315 {
 316         synchronize_srcu(&srcu_ctl);
 317         cleanup_srcu_struct(&srcu_ctl);
 318 }
 319
 320 static int srcu_torture_read_lock(void)
 321 {
 322         return srcu_read_lock(&srcu_ctl);
 323 }
 324
 325 static void srcu_read_delay(struct rcu_random_state *rrsp)
 326 {
 327         long delay;
 328         const long uspertick = 1000000 / HZ;
 329         const long longdelay = 10;
 330
 331         /* We want there to be long-running readers, but not all the time. */
 332
 333         delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick);
 334         if (!delay)
 335                 schedule_timeout_interruptible(longdelay);
 336 }
 337
 338 static void srcu_torture_read_unlock(int idx)
 339 {
 340         srcu_read_unlock(&srcu_ctl, idx);
 341 }
 342
 343 static int srcu_torture_completed(void)
 344 {
 345         return srcu_batches_completed(&srcu_ctl);
 346 }
 347
 348 static void srcu_torture_deferred_free(struct rcu_torture *p)
 349 {
 350         int i;
 351         struct rcu_torture *rp;
 352         struct rcu_torture *rp1;
 353
 354         synchronize_srcu(&srcu_ctl);
 355         list_add(&p->rtort_free, &srcu_removed);
 356         list_for_each_entry_safe(rp, rp1, &srcu_removed, rtort_free) {
 357                 i = rp->rtort_pipe_count;
 358                 if (i > RCU_TORTURE_PIPE_LEN)
 359                         i = RCU_TORTURE_PIPE_LEN;
 360                 atomic_inc(&rcu_torture_wcount[i]);
 361                 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
 362                         rp->rtort_mbtest = 0;
 363                         list_del(&rp->rtort_free);
 364                         rcu_torture_free(rp);
 365                 }
 366         }
 367 }
 368
 369 static int srcu_torture_stats(char *page)
 370 {
 371         int cnt = 0;
 372         int cpu;
 373         int idx = srcu_ctl.completed & 0x1;
 374
 375         cnt += sprintf(&page[cnt], "%s%s per-CPU(idx=%d):",
 376                        torture_type, TORTURE_FLAG, idx);
 377         for_each_possible_cpu(cpu) {
 378                 cnt += sprintf(&page[cnt], " %d(%d,%d)", cpu,
 379                                per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[!idx],
 380                                per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[idx]);
 381         }
 382         cnt += sprintf(&page[cnt], "\n");
 383         return cnt;
 384 }
 385
 386 static struct rcu_torture_ops srcu_ops = {
 387         .init = srcu_torture_init,
 388         .cleanup = srcu_torture_cleanup,
 389         .readlock = srcu_torture_read_lock,
 390         .readdelay = srcu_read_delay,
 391         .readunlock = srcu_torture_read_unlock,
 392         .completed = srcu_torture_completed,
 393         .deferredfree = srcu_torture_deferred_free,
 394         .stats = srcu_torture_stats,
 395         .name = "srcu"
 396 };
 397
 398 static struct rcu_torture_ops *torture_ops[] =
 399         { &rcu_ops, &rcu_bh_ops, &srcu_ops, NULL };
 400
 401 /*
 402  * RCU torture writer kthread.  Repeatedly substitutes a new structure
 403  * for that pointed to by rcu_torture_current, freeing the old structure
 404  * after a series of grace periods (the "pipeline").
 405  */
 406 static int
 407 rcu_torture_writer(void *arg)
 408 {
 409         int i;
 410         long oldbatch = rcu_batches_completed();
 411         struct rcu_torture *rp;
 412         struct rcu_torture *old_rp;
 413         static DEFINE_RCU_RANDOM(rand);
 414
 415         VERBOSE_PRINTK_STRING("rcu_torture_writer task started");
 416         set_user_nice(current, 19);
 417
 418         do {
 419                 schedule_timeout_uninterruptible(1);
 420                 if ((rp = rcu_torture_alloc()) == NULL)
 421                         continue;
 422                 rp->rtort_pipe_count = 0;
 423                 udelay(rcu_random(&rand) & 0x3ff);
 424                 old_rp = rcu_torture_current;
 425                 rp->rtort_mbtest = 1;
 426                 rcu_assign_pointer(rcu_torture_current, rp);
 427                 smp_wmb();
 428                 if (old_rp != NULL) {
 429                         i = old_rp->rtort_pipe_count;
 430                         if (i > RCU_TORTURE_PIPE_LEN)
 431                                 i = RCU_TORTURE_PIPE_LEN;
 432                         atomic_inc(&rcu_torture_wcount[i]);
 433                         old_rp->rtort_pipe_count++;
 434                         cur_ops->deferredfree(old_rp);
 435                 }
 436                 rcu_torture_current_version++;
 437                 oldbatch = cur_ops->completed();
 438         } while (!kthread_should_stop() && !fullstop);
 439         VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping");
 440         while (!kthread_should_stop())
 441                 schedule_timeout_uninterruptible(1);
 442         return 0;
 443 }
 444
 445 /*
 446  * RCU torture reader kthread.  Repeatedly dereferences rcu_torture_current,
 447  * incrementing the corresponding element of the pipeline array.  The
 448  * counter in the element should never be greater than 1, otherwise, the
 449  * RCU implementation is broken.
 450  */
 451 static int
 452 rcu_torture_reader(void *arg)
 453 {
 454         int completed;
 455         int idx;
 456         DEFINE_RCU_RANDOM(rand);
 457         struct rcu_torture *p;
 458         int pipe_count;
 459
 460         VERBOSE_PRINTK_STRING("rcu_torture_reader task started");
 461         set_user_nice(current, 19);
 462
 463         do {
 464                 idx = cur_ops->readlock();
 465                 completed = cur_ops->completed();
 466                 p = rcu_dereference(rcu_torture_current);
 467                 if (p == NULL) {
 468                         /* Wait for rcu_torture_writer to get underway */
 469                         cur_ops->readunlock(idx);
 470                         schedule_timeout_interruptible(HZ);
 471                         continue;
 472                 }
 473                 if (p->rtort_mbtest == 0)
 474                         atomic_inc(&n_rcu_torture_mberror);
 475                 cur_ops->readdelay(&rand);
 476                 preempt_disable();
 477                 pipe_count = p->rtort_pipe_count;
 478                 if (pipe_count > RCU_TORTURE_PIPE_LEN) {
 479                         /* Should not happen, but... */
 480                         pipe_count = RCU_TORTURE_PIPE_LEN;
 481                 }
 482                 ++__get_cpu_var(rcu_torture_count)[pipe_count];
 483                 completed = cur_ops->completed() - completed;
 484                 if (completed > RCU_TORTURE_PIPE_LEN) {
 485                         /* Should not happen, but... */
 486                         completed = RCU_TORTURE_PIPE_LEN;
 487                 }
 488                 ++__get_cpu_var(rcu_torture_batch)[completed];
 489                 preempt_enable();
 490                 cur_ops->readunlock(idx);
 491                 schedule();
 492         } while (!kthread_should_stop() && !fullstop);
 493         VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping");
 494         while (!kthread_should_stop())
 495                 schedule_timeout_uninterruptible(1);
 496         return 0;
 497 }
 498
 499 /*
 500  * Create an RCU-torture statistics message in the specified buffer.
 501  */
 502 static int
 503 rcu_torture_printk(char *page)
 504 {
 505         int cnt = 0;
 506         int cpu;
 507         int i;
 508         long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
 509         long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
 510
 511         for_each_possible_cpu(cpu) {
 512                 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
 513                         pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i];
 514                         batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i];
 515                 }
 516         }
 517         for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
 518                 if (pipesummary[i] != 0)
 519                         break;
 520         }
 521         cnt += sprintf(&page[cnt], "%s%s ", torture_type, TORTURE_FLAG);
 522         cnt += sprintf(&page[cnt],
 523                        "rtc: %p ver: %ld tfle: %d rta: %d rtaf: %d rtf: %d "
 524                        "rtmbe: %d",
 525                        rcu_torture_current,
 526                        rcu_torture_current_version,
 527                        list_empty(&rcu_torture_freelist),
 528                        atomic_read(&n_rcu_torture_alloc),
 529                        atomic_read(&n_rcu_torture_alloc_fail),
 530                        atomic_read(&n_rcu_torture_free),
 531                        atomic_read(&n_rcu_torture_mberror));
 532         if (atomic_read(&n_rcu_torture_mberror) != 0)
 533                 cnt += sprintf(&page[cnt], " !!!");
 534         cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
 535         if (i > 1) {
 536                 cnt += sprintf(&page[cnt], "!!! ");
 537                 atomic_inc(&n_rcu_torture_error);
 538         }
 539         cnt += sprintf(&page[cnt], "Reader Pipe: ");
 540         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
 541                 cnt += sprintf(&page[cnt], " %ld", pipesummary[i]);
 542         cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
 543         cnt += sprintf(&page[cnt], "Reader Batch: ");
 544         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
 545                 cnt += sprintf(&page[cnt], " %ld", batchsummary[i]);
 546         cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
 547         cnt += sprintf(&page[cnt], "Free-Block Circulation: ");
 548         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
 549                 cnt += sprintf(&page[cnt], " %d",
 550                                atomic_read(&rcu_torture_wcount[i]));
 551         }
 552         cnt += sprintf(&page[cnt], "\n");
 553         if (cur_ops->stats != NULL)
 554                 cnt += cur_ops->stats(&page[cnt]);
 555         return cnt;
 556 }
 557
 558 /*
 559  * Print torture statistics.  Caller must ensure that there is only
 560  * one call to this function at a given time!!!  This is normally
 561  * accomplished by relying on the module system to only have one copy
 562  * of the module loaded, and then by giving the rcu_torture_stats
 563  * kthread full control (or the init/cleanup functions when rcu_torture_stats
 564  * thread is not running).
 565  */
 566 static void
 567 rcu_torture_stats_print(void)
 568 {
 569         int cnt;
 570
 571         cnt = rcu_torture_printk(printk_buf);
 572         printk(KERN_ALERT "%s", printk_buf);
 573 }
 574
 575 /*
 576  * Periodically prints torture statistics, if periodic statistics printing
 577  * was specified via the stat_interval module parameter.
 578  *
 579  * No need to worry about fullstop here, since this one doesn't reference
 580  * volatile state or register callbacks.
 581  */
 582 static int
 583 rcu_torture_stats(void *arg)
 584 {
 585         VERBOSE_PRINTK_STRING("rcu_torture_stats task started");
 586         do {
 587                 schedule_timeout_interruptible(stat_interval * HZ);
 588                 rcu_torture_stats_print();
 589         } while (!kthread_should_stop());
 590         VERBOSE_PRINTK_STRING("rcu_torture_stats task stopping");
 591         return 0;
 592 }
 593
 594 static int rcu_idle_cpu;        /* Force all torture tasks off this CPU */
 595
 596 /* Shuffle tasks such that we allow @rcu_idle_cpu to become idle. A special case
 597  * is when @rcu_idle_cpu = -1, when we allow the tasks to run on all CPUs.
 598  */
 599 static void rcu_torture_shuffle_tasks(void)
 600 {
 601         cpumask_t tmp_mask = CPU_MASK_ALL;
 602         int i;
 603
 604         lock_cpu_hotplug();
 605
 606         /* No point in shuffling if there is only one online CPU (ex: UP) */
 607         if (num_online_cpus() == 1) {
 608                 unlock_cpu_hotplug();
 609                 return;
 610         }
 611
 612         if (rcu_idle_cpu != -1)
 613                 cpu_clear(rcu_idle_cpu, tmp_mask);
 614
 615         set_cpus_allowed(current, tmp_mask);
 616
 617         if (reader_tasks != NULL) {
 618                 for (i = 0; i < nrealreaders; i++)
 619                         if (reader_tasks[i])
 620                                 set_cpus_allowed(reader_tasks[i], tmp_mask);
 621         }
 622
 623         if (writer_task)
 624                 set_cpus_allowed(writer_task, tmp_mask);
 625
 626         if (stats_task)
 627                 set_cpus_allowed(stats_task, tmp_mask);
 628
 629         if (rcu_idle_cpu == -1)
 630                 rcu_idle_cpu = num_online_cpus() - 1;
 631         else
 632                 rcu_idle_cpu--;
 633
 634         unlock_cpu_hotplug();
 635 }
 636
 637 /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the
 638  * system to become idle at a time and cut off its timer ticks. This is meant
 639  * to test the support for such tickless idle CPU in RCU.
 640  */
 641 static int
 642 rcu_torture_shuffle(void *arg)
 643 {
 644         VERBOSE_PRINTK_STRING("rcu_torture_shuffle task started");
 645         do {
 646                 schedule_timeout_interruptible(shuffle_interval * HZ);
 647                 rcu_torture_shuffle_tasks();
 648         } while (!kthread_should_stop());
 649         VERBOSE_PRINTK_STRING("rcu_torture_shuffle task stopping");
 650         return 0;
 651 }
 652
 653 static inline void
 654 rcu_torture_print_module_parms(char *tag)
 655 {
 656         printk(KERN_ALERT "%s" TORTURE_FLAG "--- %s: nreaders=%d "
 657                 "stat_interval=%d verbose=%d test_no_idle_hz=%d "
 658                 "shuffle_interval = %d\n",
 659                 torture_type, tag, nrealreaders, stat_interval, verbose,
 660                 test_no_idle_hz, shuffle_interval);
 661 }
 662
 663 static void
 664 rcu_torture_cleanup(void)
 665 {
 666         int i;
 667
 668         fullstop = 1;
 669         if (shuffler_task != NULL) {
 670                 VERBOSE_PRINTK_STRING("Stopping rcu_torture_shuffle task");
 671                 kthread_stop(shuffler_task);
 672         }
 673         shuffler_task = NULL;
 674
 675         if (writer_task != NULL) {
 676                 VERBOSE_PRINTK_STRING("Stopping rcu_torture_writer task");
 677                 kthread_stop(writer_task);
 678         }
 679         writer_task = NULL;
 680
 681         if (reader_tasks != NULL) {
 682                 for (i = 0; i < nrealreaders; i++) {
 683                         if (reader_tasks[i] != NULL) {
 684                                 VERBOSE_PRINTK_STRING(
 685                                         "Stopping rcu_torture_reader task");
 686                                 kthread_stop(reader_tasks[i]);
 687                         }
 688                         reader_tasks[i] = NULL;
 689                 }
 690                 kfree(reader_tasks);
 691                 reader_tasks = NULL;
 692         }
 693         rcu_torture_current = NULL;
 694
 695         if (stats_task != NULL) {
 696                 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stats task");
 697                 kthread_stop(stats_task);
 698         }
 699         stats_task = NULL;
 700
 701         /* Wait for all RCU callbacks to fire.  */
 702         rcu_barrier();
 703
 704         rcu_torture_stats_print();  /* -After- the stats thread is stopped! */
 705
 706         if (cur_ops->cleanup != NULL)
 707                 cur_ops->cleanup();
 708         if (atomic_read(&n_rcu_torture_error))
 709                 rcu_torture_print_module_parms("End of test: FAILURE");
 710         else
 711                 rcu_torture_print_module_parms("End of test: SUCCESS");
 712 }
 713
 714 static int
 715 rcu_torture_init(void)
 716 {
 717         int i;
 718         int cpu;
 719         int firsterr = 0;
 720
 721         /* Process args and tell the world that the torturer is on the job. */
 722
 723         for (i = 0; cur_ops = torture_ops[i], cur_ops != NULL; i++) {
 724                 cur_ops = torture_ops[i];
 725                 if (strcmp(torture_type, cur_ops->name) == 0) {
 726                         break;
 727                 }
 728         }
 729         if (cur_ops == NULL) {
 730                 printk(KERN_ALERT "rcutorture: invalid torture type: \"%s\"\n",
 731                        torture_type);
 732                 return (-EINVAL);
 733         }
 734         if (cur_ops->init != NULL)
 735                 cur_ops->init(); /* no "goto unwind" prior to this point!!! */
 736
 737         if (nreaders >= 0)
 738                 nrealreaders = nreaders;
 739         else
 740                 nrealreaders = 2 * num_online_cpus();
 741         rcu_torture_print_module_parms("Start of test");
 742         fullstop = 0;
 743
 744         /* Set up the freelist. */
 745
 746         INIT_LIST_HEAD(&rcu_torture_freelist);
 747         for (i = 0; i < sizeof(rcu_tortures) / sizeof(rcu_tortures[0]); i++) {
 748                 rcu_tortures[i].rtort_mbtest = 0;
 749                 list_add_tail(&rcu_tortures[i].rtort_free,
 750                               &rcu_torture_freelist);
 751         }
 752
 753         /* Initialize the statistics so that each run gets its own numbers. */
 754
 755         rcu_torture_current = NULL;
 756         rcu_torture_current_version = 0;
 757         atomic_set(&n_rcu_torture_alloc, 0);
 758         atomic_set(&n_rcu_torture_alloc_fail, 0);
 759         atomic_set(&n_rcu_torture_free, 0);
 760         atomic_set(&n_rcu_torture_mberror, 0);
 761         atomic_set(&n_rcu_torture_error, 0);
 762         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
 763                 atomic_set(&rcu_torture_wcount[i], 0);
 764         for_each_possible_cpu(cpu) {
 765                 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
 766                         per_cpu(rcu_torture_count, cpu)[i] = 0;
 767                         per_cpu(rcu_torture_batch, cpu)[i] = 0;
 768                 }
 769         }
 770
 771         /* Start up the kthreads. */
 772
 773         VERBOSE_PRINTK_STRING("Creating rcu_torture_writer task");
 774         writer_task = kthread_run(rcu_torture_writer, NULL,
 775                                   "rcu_torture_writer");
 776         if (IS_ERR(writer_task)) {
 777                 firsterr = PTR_ERR(writer_task);
 778                 VERBOSE_PRINTK_ERRSTRING("Failed to create writer");
 779                 writer_task = NULL;
 780                 goto unwind;
 781         }
 782         reader_tasks = kmalloc(nrealreaders * sizeof(reader_tasks[0]),
 783                                GFP_KERNEL);
 784         if (reader_tasks == NULL) {
 785                 VERBOSE_PRINTK_ERRSTRING("out of memory");
 786                 firsterr = -ENOMEM;
 787                 goto unwind;
 788         }
 789         for (i = 0; i < nrealreaders; i++) {
 790                 VERBOSE_PRINTK_STRING("Creating rcu_torture_reader task");
 791                 reader_tasks[i] = kthread_run(rcu_torture_reader, NULL,
 792                                               "rcu_torture_reader");
 793                 if (IS_ERR(reader_tasks[i])) {
 794                         firsterr = PTR_ERR(reader_tasks[i]);
 795                         VERBOSE_PRINTK_ERRSTRING("Failed to create reader");
 796                         reader_tasks[i] = NULL;
 797                         goto unwind;
 798                 }
 799         }
 800         if (stat_interval > 0) {
 801                 VERBOSE_PRINTK_STRING("Creating rcu_torture_stats task");
 802                 stats_task = kthread_run(rcu_torture_stats, NULL,
 803                                         "rcu_torture_stats");
 804                 if (IS_ERR(stats_task)) {
 805                         firsterr = PTR_ERR(stats_task);
 806                         VERBOSE_PRINTK_ERRSTRING("Failed to create stats");
 807                         stats_task = NULL;
 808                         goto unwind;
 809                 }
 810         }
 811         if (test_no_idle_hz) {
 812                 rcu_idle_cpu = num_online_cpus() - 1;
 813                 /* Create the shuffler thread */
 814                 shuffler_task = kthread_run(rcu_torture_shuffle, NULL,
 815                                           "rcu_torture_shuffle");
 816                 if (IS_ERR(shuffler_task)) {
 817                         firsterr = PTR_ERR(shuffler_task);
 818                         VERBOSE_PRINTK_ERRSTRING("Failed to create shuffler");
 819                         shuffler_task = NULL;
 820                         goto unwind;
 821                 }
 822         }
 823         return 0;
 824
 825 unwind:
 826         rcu_torture_cleanup();
 827         return firsterr;
 828 }
 829
 830 module_init(rcu_torture_init);
 831 module_exit(rcu_torture_cleanup);