tools/testing/ktest/ktest.pl

   1 #!/usr/bin/perl -w
   2 #
   3 # Copyright 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
   4 # Licensed under the terms of the GNU GPL License version 2
   5 #
   6
   7 use strict;
   8 use IPC::Open2;
   9 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
  10 use File::Path qw(mkpath);
  11 use File::Copy qw(cp);
  12 use FileHandle;
  13
  14 my $VERSION = "0.2";
  15
  16 $| = 1;
  17
  18 my %opt;
  19 my %repeat_tests;
  20 my %repeats;
  21
  22 #default opts
  23 my %default = (
  24     "NUM_TESTS"                 => 1,
  25     "TEST_TYPE"                 => "build",
  26     "BUILD_TYPE"                => "randconfig",
  27     "MAKE_CMD"                  => "make",
  28     "TIMEOUT"                   => 120,
  29     "TMP_DIR"                   => "/tmp/ktest/\${MACHINE}",
  30     "SLEEP_TIME"                => 60,  # sleep time between tests
  31     "BUILD_NOCLEAN"             => 0,
  32     "REBOOT_ON_ERROR"           => 0,
  33     "POWEROFF_ON_ERROR"         => 0,
  34     "REBOOT_ON_SUCCESS"         => 1,
  35     "POWEROFF_ON_SUCCESS"       => 0,
  36     "BUILD_OPTIONS"             => "",
  37     "BISECT_SLEEP_TIME"         => 60,   # sleep time between bisects
  38     "PATCHCHECK_SLEEP_TIME"     => 60, # sleep time between patch checks
  39     "CLEAR_LOG"                 => 0,
  40     "BISECT_MANUAL"             => 0,
  41     "BISECT_SKIP"               => 1,
  42     "MIN_CONFIG_TYPE"           => "boot",
  43     "SUCCESS_LINE"              => "login:",
  44     "DETECT_TRIPLE_FAULT"       => 1,
  45     "NO_INSTALL"                => 0,
  46     "BOOTED_TIMEOUT"            => 1,
  47     "DIE_ON_FAILURE"            => 1,
  48     "SSH_EXEC"                  => "ssh \$SSH_USER\@\$MACHINE \$SSH_COMMAND",
  49     "SCP_TO_TARGET"             => "scp \$SRC_FILE \$SSH_USER\@\$MACHINE:\$DST_FILE",
  50     "SCP_TO_TARGET_INSTALL"     => "\${SCP_TO_TARGET}",
  51     "REBOOT"                    => "ssh \$SSH_USER\@\$MACHINE reboot",
  52     "STOP_AFTER_SUCCESS"        => 10,
  53     "STOP_AFTER_FAILURE"        => 60,
  54     "STOP_TEST_AFTER"           => 600,
  55     "MAX_MONITOR_WAIT"          => 1800,
  56
  57 # required, and we will ask users if they don't have them but we keep the default
  58 # value something that is common.
  59     "REBOOT_TYPE"               => "grub",
  60     "LOCALVERSION"              => "-test",
  61     "SSH_USER"                  => "root",
  62     "BUILD_TARGET"              => "arch/x86/boot/bzImage",
  63     "TARGET_IMAGE"              => "/boot/vmlinuz-test",
  64
  65     "LOG_FILE"                  => undef,
  66     "IGNORE_UNUSED"             => 0,
  67 );
  68
  69 my $ktest_config;
  70 my $version;
  71 my $have_version = 0;
  72 my $machine;
  73 my $ssh_user;
  74 my $tmpdir;
  75 my $builddir;
  76 my $outputdir;
  77 my $output_config;
  78 my $test_type;
  79 my $build_type;
  80 my $build_options;
  81 my $final_post_ktest;
  82 my $pre_ktest;
  83 my $post_ktest;
  84 my $pre_test;
  85 my $post_test;
  86 my $pre_build;
  87 my $post_build;
  88 my $pre_build_die;
  89 my $post_build_die;
  90 my $reboot_type;
  91 my $reboot_script;
  92 my $power_cycle;
  93 my $reboot;
  94 my $reboot_on_error;
  95 my $switch_to_good;
  96 my $switch_to_test;
  97 my $poweroff_on_error;
  98 my $reboot_on_success;
  99 my $die_on_failure;
 100 my $powercycle_after_reboot;
 101 my $poweroff_after_halt;
 102 my $max_monitor_wait;
 103 my $ssh_exec;
 104 my $scp_to_target;
 105 my $scp_to_target_install;
 106 my $power_off;
 107 my $grub_menu;
 108 my $grub_number;
 109 my $target;
 110 my $make;
 111 my $pre_install;
 112 my $post_install;
 113 my $no_install;
 114 my $noclean;
 115 my $minconfig;
 116 my $start_minconfig;
 117 my $start_minconfig_defined;
 118 my $output_minconfig;
 119 my $minconfig_type;
 120 my $use_output_minconfig;
 121 my $ignore_config;
 122 my $ignore_errors;
 123 my $addconfig;
 124 my $in_bisect = 0;
 125 my $bisect_bad_commit = "";
 126 my $reverse_bisect;
 127 my $bisect_manual;
 128 my $bisect_skip;
 129 my $config_bisect_good;
 130 my $bisect_ret_good;
 131 my $bisect_ret_bad;
 132 my $bisect_ret_skip;
 133 my $bisect_ret_abort;
 134 my $bisect_ret_default;
 135 my $in_patchcheck = 0;
 136 my $run_test;
 137 my $redirect;
 138 my $buildlog;
 139 my $testlog;
 140 my $dmesg;
 141 my $monitor_fp;
 142 my $monitor_pid;
 143 my $monitor_cnt = 0;
 144 my $sleep_time;
 145 my $bisect_sleep_time;
 146 my $patchcheck_sleep_time;
 147 my $ignore_warnings;
 148 my $store_failures;
 149 my $store_successes;
 150 my $test_name;
 151 my $timeout;
 152 my $booted_timeout;
 153 my $detect_triplefault;
 154 my $console;
 155 my $reboot_success_line;
 156 my $success_line;
 157 my $stop_after_success;
 158 my $stop_after_failure;
 159 my $stop_test_after;
 160 my $build_target;
 161 my $target_image;
 162 my $checkout;
 163 my $localversion;
 164 my $iteration = 0;
 165 my $successes = 0;
 166
 167 my $bisect_good;
 168 my $bisect_bad;
 169 my $bisect_type;
 170 my $bisect_start;
 171 my $bisect_replay;
 172 my $bisect_files;
 173 my $bisect_reverse;
 174 my $bisect_check;
 175
 176 my $config_bisect;
 177 my $config_bisect_type;
 178 my $config_bisect_check;
 179
 180 my $patchcheck_type;
 181 my $patchcheck_start;
 182 my $patchcheck_end;
 183
 184 # set when a test is something other that just building or install
 185 # which would require more options.
 186 my $buildonly = 1;
 187
 188 # set when creating a new config
 189 my $newconfig = 0;
 190
 191 my %entered_configs;
 192 my %config_help;
 193 my %variable;
 194
 195 # force_config is the list of configs that we force enabled (or disabled)
 196 # in a .config file. The MIN_CONFIG and ADD_CONFIG configs.
 197 my %force_config;
 198
 199 # do not force reboots on config problems
 200 my $no_reboot = 1;
 201
 202 # reboot on success
 203 my $reboot_success = 0;
 204
 205 my %option_map = (
 206     "MACHINE"                   => \$machine,
 207     "SSH_USER"                  => \$ssh_user,
 208     "TMP_DIR"                   => \$tmpdir,
 209     "OUTPUT_DIR"                => \$outputdir,
 210     "BUILD_DIR"                 => \$builddir,
 211     "TEST_TYPE"                 => \$test_type,
 212     "PRE_KTEST"                 => \$pre_ktest,
 213     "POST_KTEST"                => \$post_ktest,
 214     "PRE_TEST"                  => \$pre_test,
 215     "POST_TEST"                 => \$post_test,
 216     "BUILD_TYPE"                => \$build_type,
 217     "BUILD_OPTIONS"             => \$build_options,
 218     "PRE_BUILD"                 => \$pre_build,
 219     "POST_BUILD"                => \$post_build,
 220     "PRE_BUILD_DIE"             => \$pre_build_die,
 221     "POST_BUILD_DIE"            => \$post_build_die,
 222     "POWER_CYCLE"               => \$power_cycle,
 223     "REBOOT"                    => \$reboot,
 224     "BUILD_NOCLEAN"             => \$noclean,
 225     "MIN_CONFIG"                => \$minconfig,
 226     "OUTPUT_MIN_CONFIG"         => \$output_minconfig,
 227     "START_MIN_CONFIG"          => \$start_minconfig,
 228     "MIN_CONFIG_TYPE"           => \$minconfig_type,
 229     "USE_OUTPUT_MIN_CONFIG"     => \$use_output_minconfig,
 230     "IGNORE_CONFIG"             => \$ignore_config,
 231     "TEST"                      => \$run_test,
 232     "ADD_CONFIG"                => \$addconfig,
 233     "REBOOT_TYPE"               => \$reboot_type,
 234     "GRUB_MENU"                 => \$grub_menu,
 235     "PRE_INSTALL"               => \$pre_install,
 236     "POST_INSTALL"              => \$post_install,
 237     "NO_INSTALL"                => \$no_install,
 238     "REBOOT_SCRIPT"             => \$reboot_script,
 239     "REBOOT_ON_ERROR"           => \$reboot_on_error,
 240     "SWITCH_TO_GOOD"            => \$switch_to_good,
 241     "SWITCH_TO_TEST"            => \$switch_to_test,
 242     "POWEROFF_ON_ERROR"         => \$poweroff_on_error,
 243     "REBOOT_ON_SUCCESS"         => \$reboot_on_success,
 244     "DIE_ON_FAILURE"            => \$die_on_failure,
 245     "POWER_OFF"                 => \$power_off,
 246     "POWERCYCLE_AFTER_REBOOT"   => \$powercycle_after_reboot,
 247     "POWEROFF_AFTER_HALT"       => \$poweroff_after_halt,
 248     "MAX_MONITOR_WAIT"          => \$max_monitor_wait,
 249     "SLEEP_TIME"                => \$sleep_time,
 250     "BISECT_SLEEP_TIME"         => \$bisect_sleep_time,
 251     "PATCHCHECK_SLEEP_TIME"     => \$patchcheck_sleep_time,
 252     "IGNORE_WARNINGS"           => \$ignore_warnings,
 253     "IGNORE_ERRORS"             => \$ignore_errors,
 254     "BISECT_MANUAL"             => \$bisect_manual,
 255     "BISECT_SKIP"               => \$bisect_skip,
 256     "CONFIG_BISECT_GOOD"        => \$config_bisect_good,
 257     "BISECT_RET_GOOD"           => \$bisect_ret_good,
 258     "BISECT_RET_BAD"            => \$bisect_ret_bad,
 259     "BISECT_RET_SKIP"           => \$bisect_ret_skip,
 260     "BISECT_RET_ABORT"          => \$bisect_ret_abort,
 261     "BISECT_RET_DEFAULT"        => \$bisect_ret_default,
 262     "STORE_FAILURES"            => \$store_failures,
 263     "STORE_SUCCESSES"           => \$store_successes,
 264     "TEST_NAME"                 => \$test_name,
 265     "TIMEOUT"                   => \$timeout,
 266     "BOOTED_TIMEOUT"            => \$booted_timeout,
 267     "CONSOLE"                   => \$console,
 268     "DETECT_TRIPLE_FAULT"       => \$detect_triplefault,
 269     "SUCCESS_LINE"              => \$success_line,
 270     "REBOOT_SUCCESS_LINE"       => \$reboot_success_line,
 271     "STOP_AFTER_SUCCESS"        => \$stop_after_success,
 272     "STOP_AFTER_FAILURE"        => \$stop_after_failure,
 273     "STOP_TEST_AFTER"           => \$stop_test_after,
 274     "BUILD_TARGET"              => \$build_target,
 275     "SSH_EXEC"                  => \$ssh_exec,
 276     "SCP_TO_TARGET"             => \$scp_to_target,
 277     "SCP_TO_TARGET_INSTALL"     => \$scp_to_target_install,
 278     "CHECKOUT"                  => \$checkout,
 279     "TARGET_IMAGE"              => \$target_image,
 280     "LOCALVERSION"              => \$localversion,
 281
 282     "BISECT_GOOD"               => \$bisect_good,
 283     "BISECT_BAD"                => \$bisect_bad,
 284     "BISECT_TYPE"               => \$bisect_type,
 285     "BISECT_START"              => \$bisect_start,
 286     "BISECT_REPLAY"             => \$bisect_replay,
 287     "BISECT_FILES"              => \$bisect_files,
 288     "BISECT_REVERSE"            => \$bisect_reverse,
 289     "BISECT_CHECK"              => \$bisect_check,
 290
 291     "CONFIG_BISECT"             => \$config_bisect,
 292     "CONFIG_BISECT_TYPE"        => \$config_bisect_type,
 293     "CONFIG_BISECT_CHECK"       => \$config_bisect_check,
 294
 295     "PATCHCHECK_TYPE"           => \$patchcheck_type,
 296     "PATCHCHECK_START"          => \$patchcheck_start,
 297     "PATCHCHECK_END"            => \$patchcheck_end,
 298 );
 299
 300 # Options may be used by other options, record them.
 301 my %used_options;
 302
 303 # default variables that can be used
 304 chomp ($variable{"PWD"} = `pwd`);
 305
 306 $config_help{"MACHINE"} = << "EOF"
 307  The machine hostname that you will test.
 308  For build only tests, it is still needed to differentiate log files.
 309 EOF
 310     ;
 311 $config_help{"SSH_USER"} = << "EOF"
 312  The box is expected to have ssh on normal bootup, provide the user
 313   (most likely root, since you need privileged operations)
 314 EOF
 315     ;
 316 $config_help{"BUILD_DIR"} = << "EOF"
 317  The directory that contains the Linux source code (full path).
 318  You can use \${PWD} that will be the path where ktest.pl is run, or use
 319  \${THIS_DIR} which is assigned \${PWD} but may be changed later.
 320 EOF
 321     ;
 322 $config_help{"OUTPUT_DIR"} = << "EOF"
 323  The directory that the objects will be built (full path).
 324  (can not be same as BUILD_DIR)
 325  You can use \${PWD} that will be the path where ktest.pl is run, or use
 326  \${THIS_DIR} which is assigned \${PWD} but may be changed later.
 327 EOF
 328     ;
 329 $config_help{"BUILD_TARGET"} = << "EOF"
 330  The location of the compiled file to copy to the target.
 331  (relative to OUTPUT_DIR)
 332 EOF
 333     ;
 334 $config_help{"BUILD_OPTIONS"} = << "EOF"
 335  Options to add to \"make\" when building.
 336  i.e.  -j20
 337 EOF
 338     ;
 339 $config_help{"TARGET_IMAGE"} = << "EOF"
 340  The place to put your image on the test machine.
 341 EOF
 342     ;
 343 $config_help{"POWER_CYCLE"} = << "EOF"
 344  A script or command to reboot the box.
 345
 346  Here is a digital loggers power switch example
 347  POWER_CYCLE = wget --no-proxy -O /dev/null -q  --auth-no-challenge 'http://admin:admin\@power/outlet?5=CCL'
 348
 349  Here is an example to reboot a virtual box on the current host
 350  with the name "Guest".
 351  POWER_CYCLE = virsh destroy Guest; sleep 5; virsh start Guest
 352 EOF
 353     ;
 354 $config_help{"CONSOLE"} = << "EOF"
 355  The script or command that reads the console
 356
 357   If you use ttywatch server, something like the following would work.
 358 CONSOLE = nc -d localhost 3001
 359
 360  For a virtual machine with guest name "Guest".
 361 CONSOLE =  virsh console Guest
 362 EOF
 363     ;
 364 $config_help{"LOCALVERSION"} = << "EOF"
 365  Required version ending to differentiate the test
 366  from other linux builds on the system.
 367 EOF
 368     ;
 369 $config_help{"REBOOT_TYPE"} = << "EOF"
 370  Way to reboot the box to the test kernel.
 371  Only valid options so far are "grub" and "script".
 372
 373  If you specify grub, it will assume grub version 1
 374  and will search in /boot/grub/menu.lst for the title \$GRUB_MENU
 375  and select that target to reboot to the kernel. If this is not
 376  your setup, then specify "script" and have a command or script
 377  specified in REBOOT_SCRIPT to boot to the target.
 378
 379  The entry in /boot/grub/menu.lst must be entered in manually.
 380  The test will not modify that file.
 381 EOF
 382     ;
 383 $config_help{"GRUB_MENU"} = << "EOF"
 384  The grub title name for the test kernel to boot
 385  (Only mandatory if REBOOT_TYPE = grub)
 386
 387  Note, ktest.pl will not update the grub menu.lst, you need to
 388  manually add an option for the test. ktest.pl will search
 389  the grub menu.lst for this option to find what kernel to
 390  reboot into.
 391
 392  For example, if in the /boot/grub/menu.lst the test kernel title has:
 393  title Test Kernel
 394  kernel vmlinuz-test
 395  GRUB_MENU = Test Kernel
 396 EOF
 397     ;
 398 $config_help{"REBOOT_SCRIPT"} = << "EOF"
 399  A script to reboot the target into the test kernel
 400  (Only mandatory if REBOOT_TYPE = script)
 401 EOF
 402     ;
 403
 404 sub read_prompt {
 405     my ($cancel, $prompt) = @_;
 406
 407     my $ans;
 408
 409     for (;;) {
 410         if ($cancel) {
 411             print "$prompt [y/n/C] ";
 412         } else {
 413             print "$prompt [Y/n] ";
 414         }
 415         $ans = <STDIN>;
 416         chomp $ans;
 417         if ($ans =~ /^\s*$/) {
 418             if ($cancel) {
 419                 $ans = "c";
 420             } else {
 421                 $ans = "y";
 422             }
 423         }
 424         last if ($ans =~ /^y$/i || $ans =~ /^n$/i);
 425         if ($cancel) {
 426             last if ($ans =~ /^c$/i);
 427             print "Please answer either 'y', 'n' or 'c'.\n";
 428         } else {
 429             print "Please answer either 'y' or 'n'.\n";
 430         }
 431     }
 432     if ($ans =~ /^c/i) {
 433         exit;
 434     }
 435     if ($ans !~ /^y$/i) {
 436         return 0;
 437     }
 438     return 1;
 439 }
 440
 441 sub read_yn {
 442     my ($prompt) = @_;
 443
 444     return read_prompt 0, $prompt;
 445 }
 446
 447 sub read_ync {
 448     my ($prompt) = @_;
 449
 450     return read_prompt 1, $prompt;
 451 }
 452
 453 sub get_ktest_config {
 454     my ($config) = @_;
 455     my $ans;
 456
 457     return if (defined($opt{$config}));
 458
 459     if (defined($config_help{$config})) {
 460         print "\n";
 461         print $config_help{$config};
 462     }
 463
 464     for (;;) {
 465         print "$config = ";
 466         if (defined($default{$config}) && length($default{$config})) {
 467             print "\[$default{$config}\] ";
 468         }
 469         $ans = <STDIN>;
 470         $ans =~ s/^\s*(.*\S)\s*$/$1/;
 471         if ($ans =~ /^\s*$/) {
 472             if ($default{$config}) {
 473                 $ans = $default{$config};
 474             } else {
 475                 print "Your answer can not be blank\n";
 476                 next;
 477             }
 478         }
 479         $entered_configs{$config} = ${ans};
 480         last;
 481     }
 482 }
 483
 484 sub get_ktest_configs {
 485     get_ktest_config("MACHINE");
 486     get_ktest_config("BUILD_DIR");
 487     get_ktest_config("OUTPUT_DIR");
 488
 489     if ($newconfig) {
 490         get_ktest_config("BUILD_OPTIONS");
 491     }
 492
 493     # options required for other than just building a kernel
 494     if (!$buildonly) {
 495         get_ktest_config("POWER_CYCLE");
 496         get_ktest_config("CONSOLE");
 497     }
 498
 499     # options required for install and more
 500     if ($buildonly != 1) {
 501         get_ktest_config("SSH_USER");
 502         get_ktest_config("BUILD_TARGET");
 503         get_ktest_config("TARGET_IMAGE");
 504     }
 505
 506     get_ktest_config("LOCALVERSION");
 507
 508     return if ($buildonly);
 509
 510     my $rtype = $opt{"REBOOT_TYPE"};
 511
 512     if (!defined($rtype)) {
 513         if (!defined($opt{"GRUB_MENU"})) {
 514             get_ktest_config("REBOOT_TYPE");
 515             $rtype = $entered_configs{"REBOOT_TYPE"};
 516         } else {
 517             $rtype = "grub";
 518         }
 519     }
 520
 521     if ($rtype eq "grub") {
 522         get_ktest_config("GRUB_MENU");
 523     }
 524 }
 525
 526 sub process_variables {
 527     my ($value, $remove_undef) = @_;
 528     my $retval = "";
 529
 530     # We want to check for '\', and it is just easier
 531     # to check the previous characet of '$' and not need
 532     # to worry if '$' is the first character. By adding
 533     # a space to $value, we can just check [^\\]\$ and
 534     # it will still work.
 535     $value = " $value";
 536
 537     while ($value =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
 538         my $begin = $1;
 539         my $var = $2;
 540         my $end = $3;
 541         # append beginning of value to retval
 542         $retval = "$retval$begin";
 543         if (defined($variable{$var})) {
 544             $retval = "$retval$variable{$var}";
 545         } elsif (defined($remove_undef) && $remove_undef) {
 546             # for if statements, any variable that is not defined,
 547             # we simple convert to 0
 548             $retval = "${retval}0";
 549         } else {
 550             # put back the origin piece.
 551             $retval = "$retval\$\{$var\}";
 552             # This could be an option that is used later, save
 553             # it so we don't warn if this option is not one of
 554             # ktests options.
 555             $used_options{$var} = 1;
 556         }
 557         $value = $end;
 558     }
 559     $retval = "$retval$value";
 560
 561     # remove the space added in the beginning
 562     $retval =~ s/ //;
 563
 564     return "$retval"
 565 }
 566
 567 sub set_value {
 568     my ($lvalue, $rvalue, $override, $overrides, $name) = @_;
 569
 570     my $prvalue = process_variables($rvalue);
 571
 572     if ($buildonly && $lvalue =~ /^TEST_TYPE(\[.*\])?$/ && $prvalue ne "build") {
 573         # Note if a test is something other than build, then we
 574         # will need other manditory options.
 575         if ($prvalue ne "install") {
 576             $buildonly = 0;
 577         } else {
 578             # install still limits some manditory options.
 579             $buildonly = 2;
 580         }
 581     }
 582
 583     if (defined($opt{$lvalue})) {
 584         if (!$override || defined(${$overrides}{$lvalue})) {
 585             my $extra = "";
 586             if ($override) {
 587                 $extra = "In the same override section!\n";
 588             }
 589             die "$name: $.: Option $lvalue defined more than once!\n$extra";
 590         }
 591         ${$overrides}{$lvalue} = $prvalue;
 592     }
 593     if ($rvalue =~ /^\s*$/) {
 594         delete $opt{$lvalue};
 595     } else {
 596         $opt{$lvalue} = $prvalue;
 597     }
 598 }
 599
 600 sub set_variable {
 601     my ($lvalue, $rvalue) = @_;
 602
 603     if ($rvalue =~ /^\s*$/) {
 604         delete $variable{$lvalue};
 605     } else {
 606         $rvalue = process_variables($rvalue);
 607         $variable{$lvalue} = $rvalue;
 608     }
 609 }
 610
 611 sub process_compare {
 612     my ($lval, $cmp, $rval) = @_;
 613
 614     # remove whitespace
 615
 616     $lval =~ s/^\s*//;
 617     $lval =~ s/\s*$//;
 618
 619     $rval =~ s/^\s*//;
 620     $rval =~ s/\s*$//;
 621
 622     if ($cmp eq "==") {
 623         return $lval eq $rval;
 624     } elsif ($cmp eq "!=") {
 625         return $lval ne $rval;
 626     }
 627
 628     my $statement = "$lval $cmp $rval";
 629     my $ret = eval $statement;
 630
 631     # $@ stores error of eval
 632     if ($@) {
 633         return -1;
 634     }
 635
 636     return $ret;
 637 }
 638
 639 sub value_defined {
 640     my ($val) = @_;
 641
 642     return defined($variable{$2}) ||
 643         defined($opt{$2});
 644 }
 645
 646 my $d = 0;
 647 sub process_expression {
 648     my ($name, $val) = @_;
 649
 650     my $c = $d++;
 651
 652     while ($val =~ s/\(([^\(]*?)\)/\&\&\&\&VAL\&\&\&\&/) {
 653         my $express = $1;
 654
 655         if (process_expression($name, $express)) {
 656             $val =~ s/\&\&\&\&VAL\&\&\&\&/ 1 /;
 657         } else {
 658             $val =~ s/\&\&\&\&VAL\&\&\&\&/ 0 /;
 659         }
 660     }
 661
 662     $d--;
 663     my $OR = "\\|\\|";
 664     my $AND = "\\&\\&";
 665
 666     while ($val =~ s/^(.*?)($OR|$AND)//) {
 667         my $express = $1;
 668         my $op = $2;
 669
 670         if (process_expression($name, $express)) {
 671             if ($op eq "||") {
 672                 return 1;
 673             }
 674         } else {
 675             if ($op eq "&&") {
 676                 return 0;
 677             }
 678         }
 679     }
 680
 681     if ($val =~ /(.*)(==|\!=|>=|<=|>|<)(.*)/) {
 682         my $ret = process_compare($1, $2, $3);
 683         if ($ret < 0) {
 684             die "$name: $.: Unable to process comparison\n";
 685         }
 686         return $ret;
 687     }
 688
 689     if ($val =~ /^\s*(NOT\s*)?DEFINED\s+(\S+)\s*$/) {
 690         if (defined $1) {
 691             return !value_defined($2);
 692         } else {
 693             return value_defined($2);
 694         }
 695     }
 696
 697     if ($val =~ /^\s*0\s*$/) {
 698         return 0;
 699     } elsif ($val =~ /^\s*\d+\s*$/) {
 700         return 1;
 701     }
 702
 703     die ("$name: $.: Undefined content $val in if statement\n");
 704 }
 705
 706 sub process_if {
 707     my ($name, $value) = @_;
 708
 709     # Convert variables and replace undefined ones with 0
 710     my $val = process_variables($value, 1);
 711     my $ret = process_expression $name, $val;
 712
 713     return $ret;
 714 }
 715
 716 sub __read_config {
 717     my ($config, $current_test_num) = @_;
 718
 719     my $in;
 720     open($in, $config) || die "can't read file $config";
 721
 722     my $name = $config;
 723     $name =~ s,.*/(.*),$1,;
 724
 725     my $test_num = $$current_test_num;
 726     my $default = 1;
 727     my $repeat = 1;
 728     my $num_tests_set = 0;
 729     my $skip = 0;
 730     my $rest;
 731     my $line;
 732     my $test_case = 0;
 733     my $if = 0;
 734     my $if_set = 0;
 735     my $override = 0;
 736
 737     my %overrides;
 738
 739     while (<$in>) {
 740
 741         # ignore blank lines and comments
 742         next if (/^\s*$/ || /\s*\#/);
 743
 744         if (/^\s*(TEST_START|DEFAULTS)\b(.*)/) {
 745
 746             my $type = $1;
 747             $rest = $2;
 748             $line = $2;
 749
 750             my $old_test_num;
 751             my $old_repeat;
 752             $override = 0;
 753
 754             if ($type eq "TEST_START") {
 755
 756                 if ($num_tests_set) {
 757                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 758                 }
 759
 760                 $old_test_num = $test_num;
 761                 $old_repeat = $repeat;
 762
 763                 $test_num += $repeat;
 764                 $default = 0;
 765                 $repeat = 1;
 766             } else {
 767                 $default = 1;
 768             }
 769
 770             # If SKIP is anywhere in the line, the command will be skipped
 771             if ($rest =~ s/\s+SKIP\b//) {
 772                 $skip = 1;
 773             } else {
 774                 $test_case = 1;
 775                 $skip = 0;
 776             }
 777
 778             if ($rest =~ s/\sELSE\b//) {
 779                 if (!$if) {
 780                     die "$name: $.: ELSE found with out matching IF section\n$_";
 781                 }
 782                 $if = 0;
 783
 784                 if ($if_set) {
 785                     $skip = 1;
 786                 } else {
 787                     $skip = 0;
 788                 }
 789             }
 790
 791             if ($rest =~ s/\sIF\s+(.*)//) {
 792                 if (process_if($name, $1)) {
 793                     $if_set = 1;
 794                 } else {
 795                     $skip = 1;
 796                 }
 797                 $if = 1;
 798             } else {
 799                 $if = 0;
 800                 $if_set = 0;
 801             }
 802
 803             if (!$skip) {
 804                 if ($type eq "TEST_START") {
 805                     if ($rest =~ s/\s+ITERATE\s+(\d+)//) {
 806                         $repeat = $1;
 807                         $repeat_tests{"$test_num"} = $repeat;
 808                     }
 809                 } elsif ($rest =~ s/\sOVERRIDE\b//) {
 810                     # DEFAULT only
 811                     $override = 1;
 812                     # Clear previous overrides
 813                     %overrides = ();
 814                 }
 815             }
 816
 817             if (!$skip && $rest !~ /^\s*$/) {
 818                 die "$name: $.: Gargbage found after $type\n$_";
 819             }
 820
 821             if ($skip && $type eq "TEST_START") {
 822                 $test_num = $old_test_num;
 823                 $repeat = $old_repeat;
 824             }
 825
 826         } elsif (/^\s*ELSE\b(.*)$/) {
 827             if (!$if) {
 828                 die "$name: $.: ELSE found with out matching IF section\n$_";
 829             }
 830             $rest = $1;
 831             if ($if_set) {
 832                 $skip = 1;
 833                 $rest = "";
 834             } else {
 835                 $skip = 0;
 836
 837                 if ($rest =~ /\sIF\s+(.*)/) {
 838                     # May be a ELSE IF section.
 839                     if (!process_if($name, $1)) {
 840                         $skip = 1;
 841                     }
 842                     $rest = "";
 843                 } else {
 844                     $if = 0;
 845                 }
 846             }
 847
 848             if ($rest !~ /^\s*$/) {
 849                 die "$name: $.: Gargbage found after DEFAULTS\n$_";
 850             }
 851
 852         } elsif (/^\s*INCLUDE\s+(\S+)/) {
 853
 854             next if ($skip);
 855
 856             if (!$default) {
 857                 die "$name: $.: INCLUDE can only be done in default sections\n$_";
 858             }
 859
 860             my $file = process_variables($1);
 861
 862             if ($file !~ m,^/,) {
 863                 # check the path of the config file first
 864                 if ($config =~ m,(.*)/,) {
 865                     if (-f "$1/$file") {
 866                         $file = "$1/$file";
 867                     }
 868                 }
 869             }
 870
 871             if ( ! -r $file ) {
 872                 die "$name: $.: Can't read file $file\n$_";
 873             }
 874
 875             if (__read_config($file, \$test_num)) {
 876                 $test_case = 1;
 877             }
 878
 879         } elsif (/^\s*([A-Z_\[\]\d]+)\s*=\s*(.*?)\s*$/) {
 880
 881             next if ($skip);
 882
 883             my $lvalue = $1;
 884             my $rvalue = $2;
 885
 886             if (!$default &&
 887                 ($lvalue eq "NUM_TESTS" ||
 888                  $lvalue eq "LOG_FILE" ||
 889                  $lvalue eq "CLEAR_LOG")) {
 890                 die "$name: $.: $lvalue must be set in DEFAULTS section\n";
 891             }
 892
 893             if ($lvalue eq "NUM_TESTS") {
 894                 if ($test_num) {
 895                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 896                 }
 897                 if (!$default) {
 898                     die "$name: $.: NUM_TESTS must be set in default section\n";
 899                 }
 900                 $num_tests_set = 1;
 901             }
 902
 903             if ($default || $lvalue =~ /\[\d+\]$/) {
 904                 set_value($lvalue, $rvalue, $override, \%overrides, $name);
 905             } else {
 906                 my $val = "$lvalue\[$test_num\]";
 907                 set_value($val, $rvalue, $override, \%overrides, $name);
 908
 909                 if ($repeat > 1) {
 910                     $repeats{$val} = $repeat;
 911                 }
 912             }
 913         } elsif (/^\s*([A-Z_\[\]\d]+)\s*:=\s*(.*?)\s*$/) {
 914             next if ($skip);
 915
 916             my $lvalue = $1;
 917             my $rvalue = $2;
 918
 919             # process config variables.
 920             # Config variables are only active while reading the
 921             # config and can be defined anywhere. They also ignore
 922             # TEST_START and DEFAULTS, but are skipped if they are in
 923             # on of these sections that have SKIP defined.
 924             # The save variable can be
 925             # defined multiple times and the new one simply overrides
 926             # the prevous one.
 927             set_variable($lvalue, $rvalue);
 928
 929         } else {
 930             die "$name: $.: Garbage found in config\n$_";
 931         }
 932     }
 933
 934     if ($test_num) {
 935         $test_num += $repeat - 1;
 936         $opt{"NUM_TESTS"} = $test_num;
 937     }
 938
 939     close($in);
 940
 941     $$current_test_num = $test_num;
 942
 943     return $test_case;
 944 }
 945
 946 sub get_test_case {
 947         print "What test case would you like to run?\n";
 948         print " (build, install or boot)\n";
 949         print " Other tests are available but require editing the config file\n";
 950         my $ans = <STDIN>;
 951         chomp $ans;
 952         $default{"TEST_TYPE"} = $ans;
 953 }
 954
 955 sub read_config {
 956     my ($config) = @_;
 957
 958     my $test_case;
 959     my $test_num = 0;
 960
 961     $test_case = __read_config $config, \$test_num;
 962
 963     # make sure we have all mandatory configs
 964     get_ktest_configs;
 965
 966     # was a test specified?
 967     if (!$test_case) {
 968         print "No test case specified.\n";
 969         get_test_case;
 970     }
 971
 972     # set any defaults
 973
 974     foreach my $default (keys %default) {
 975         if (!defined($opt{$default})) {
 976             $opt{$default} = $default{$default};
 977         }
 978     }
 979
 980     if ($opt{"IGNORE_UNUSED"} == 1) {
 981         return;
 982     }
 983
 984     my %not_used;
 985
 986     # check if there are any stragglers (typos?)
 987     foreach my $option (keys %opt) {
 988         my $op = $option;
 989         # remove per test labels.
 990         $op =~ s/\[.*\]//;
 991         if (!exists($option_map{$op}) &&
 992             !exists($default{$op}) &&
 993             !exists($used_options{$op})) {
 994             $not_used{$op} = 1;
 995         }
 996     }
 997
 998     if (%not_used) {
 999         my $s = "s are";
1000         $s = " is" if (keys %not_used == 1);
1001         print "The following option$s not used; could be a typo:\n";
1002         foreach my $option (keys %not_used) {
1003             print "$option\n";
1004         }
1005         print "Set IGRNORE_UNUSED = 1 to have ktest ignore unused variables\n";
1006         if (!read_yn "Do you want to continue?") {
1007             exit -1;
1008         }
1009     }
1010 }
1011
1012 sub __eval_option {
1013     my ($option, $i) = @_;
1014
1015     # Add space to evaluate the character before $
1016     $option = " $option";
1017     my $retval = "";
1018     my $repeated = 0;
1019     my $parent = 0;
1020
1021     foreach my $test (keys %repeat_tests) {
1022         if ($i >= $test &&
1023             $i < $test + $repeat_tests{$test}) {
1024
1025             $repeated = 1;
1026             $parent = $test;
1027             last;
1028         }
1029     }
1030
1031     while ($option =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
1032         my $start = $1;
1033         my $var = $2;
1034         my $end = $3;
1035
1036         # Append beginning of line
1037         $retval = "$retval$start";
1038
1039         # If the iteration option OPT[$i] exists, then use that.
1040         # otherwise see if the default OPT (without [$i]) exists.
1041
1042         my $o = "$var\[$i\]";
1043         my $parento = "$var\[$parent\]";
1044
1045         if (defined($opt{$o})) {
1046             $o = $opt{$o};
1047             $retval = "$retval$o";
1048         } elsif ($repeated && defined($opt{$parento})) {
1049             $o = $opt{$parento};
1050             $retval = "$retval$o";
1051         } elsif (defined($opt{$var})) {
1052             $o = $opt{$var};
1053             $retval = "$retval$o";
1054         } else {
1055             $retval = "$retval\$\{$var\}";
1056         }
1057
1058         $option = $end;
1059     }
1060
1061     $retval = "$retval$option";
1062
1063     $retval =~ s/^ //;
1064
1065     return $retval;
1066 }
1067
1068 sub eval_option {
1069     my ($option, $i) = @_;
1070
1071     my $prev = "";
1072
1073     # Since an option can evaluate to another option,
1074     # keep iterating until we do not evaluate any more
1075     # options.
1076     my $r = 0;
1077     while ($prev ne $option) {
1078         # Check for recursive evaluations.
1079         # 100 deep should be more than enough.
1080         if ($r++ > 100) {
1081             die "Over 100 evaluations accurred with $option\n" .
1082                 "Check for recursive variables\n";
1083         }
1084         $prev = $option;
1085         $option = __eval_option($option, $i);
1086     }
1087
1088     return $option;
1089 }
1090
1091 sub _logit {
1092     if (defined($opt{"LOG_FILE"})) {
1093         open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
1094         print OUT @_;
1095         close(OUT);
1096     }
1097 }
1098
1099 sub logit {
1100     if (defined($opt{"LOG_FILE"})) {
1101         _logit @_;
1102     } else {
1103         print @_;
1104     }
1105 }
1106
1107 sub doprint {
1108     print @_;
1109     _logit @_;
1110 }
1111
1112 sub run_command;
1113 sub start_monitor;
1114 sub end_monitor;
1115 sub wait_for_monitor;
1116
1117 sub reboot {
1118     my ($time) = @_;
1119
1120     if (defined($time)) {
1121         start_monitor;
1122         # flush out current monitor
1123         # May contain the reboot success line
1124         wait_for_monitor 1;
1125     }
1126
1127     # try to reboot normally
1128     if (run_command $reboot) {
1129         if (defined($powercycle_after_reboot)) {
1130             sleep $powercycle_after_reboot;
1131             run_command "$power_cycle";
1132         }
1133     } else {
1134         # nope? power cycle it.
1135         run_command "$power_cycle";
1136     }
1137
1138     if (defined($time)) {
1139         if (wait_for_monitor($time, $reboot_success_line)) {
1140             # reboot got stuck?
1141             doprint "Reboot did not finish. Forcing power cycle\n";
1142             run_command "$power_cycle";
1143         }
1144         end_monitor;
1145     }
1146 }
1147
1148 sub reboot_to_good {
1149     my ($time) = @_;
1150
1151     if (defined($switch_to_good)) {
1152         run_command $switch_to_good;
1153     }
1154
1155     reboot $time;
1156 }
1157
1158 sub do_not_reboot {
1159     my $i = $iteration;
1160
1161     return $test_type eq "build" || $no_reboot ||
1162         ($test_type eq "patchcheck" && $opt{"PATCHCHECK_TYPE[$i]"} eq "build") ||
1163         ($test_type eq "bisect" && $opt{"BISECT_TYPE[$i]"} eq "build");
1164 }
1165
1166 sub dodie {
1167     doprint "CRITICAL FAILURE... ", @_, "\n";
1168
1169     my $i = $iteration;
1170
1171     if ($reboot_on_error && !do_not_reboot) {
1172
1173         doprint "REBOOTING\n";
1174         reboot_to_good;
1175
1176     } elsif ($poweroff_on_error && defined($power_off)) {
1177         doprint "POWERING OFF\n";
1178         `$power_off`;
1179     }
1180
1181     if (defined($opt{"LOG_FILE"})) {
1182         print " See $opt{LOG_FILE} for more info.\n";
1183     }
1184
1185     die @_, "\n";
1186 }
1187
1188 sub open_console {
1189     my ($fp) = @_;
1190
1191     my $flags;
1192
1193     my $pid = open($fp, "$console|") or
1194         dodie "Can't open console $console";
1195
1196     $flags = fcntl($fp, F_GETFL, 0) or
1197         dodie "Can't get flags for the socket: $!";
1198     $flags = fcntl($fp, F_SETFL, $flags | O_NONBLOCK) or
1199         dodie "Can't set flags for the socket: $!";
1200
1201     return $pid;
1202 }
1203
1204 sub close_console {
1205     my ($fp, $pid) = @_;
1206
1207     doprint "kill child process $pid\n";
1208     kill 2, $pid;
1209
1210     print "closing!\n";
1211     close($fp);
1212 }
1213
1214 sub start_monitor {
1215     if ($monitor_cnt++) {
1216         return;
1217     }
1218     $monitor_fp = \*MONFD;
1219     $monitor_pid = open_console $monitor_fp;
1220
1221     return;
1222
1223     open(MONFD, "Stop perl from warning about single use of MONFD");
1224 }
1225
1226 sub end_monitor {
1227     if (--$monitor_cnt) {
1228         return;
1229     }
1230     close_console($monitor_fp, $monitor_pid);
1231 }
1232
1233 sub wait_for_monitor {
1234     my ($time, $stop) = @_;
1235     my $full_line = "";
1236     my $line;
1237     my $booted = 0;
1238     my $start_time = time;
1239     my $skip_call_trace = 0;
1240     my $bug = 0;
1241     my $bug_ignored = 0;
1242     my $now;
1243
1244     doprint "** Wait for monitor to settle down **\n";
1245
1246     # read the monitor and wait for the system to calm down
1247     while (!$booted) {
1248         $line = wait_for_input($monitor_fp, $time);
1249         last if (!defined($line));
1250         print "$line";
1251         $full_line .= $line;
1252
1253         if (defined($stop) && $full_line =~ /$stop/) {
1254             doprint "wait for monitor detected $stop\n";
1255             $booted = 1;
1256         }
1257
1258         if ($full_line =~ /\[ backtrace testing \]/) {
1259             $skip_call_trace = 1;
1260         }
1261
1262         if ($full_line =~ /call trace:/i) {
1263             if (!$bug && !$skip_call_trace) {
1264                 if ($ignore_errors) {
1265                     $bug_ignored = 1;
1266                 } else {
1267                     $bug = 1;
1268                 }
1269             }
1270         }
1271
1272         if ($full_line =~ /\[ end of backtrace testing \]/) {
1273             $skip_call_trace = 0;
1274         }
1275
1276         if ($full_line =~ /Kernel panic -/) {
1277             $bug = 1;
1278         }
1279
1280         if ($line =~ /\n/) {
1281             $full_line = "";
1282         }
1283         $now = time;
1284         if ($now - $start_time >= $max_monitor_wait) {
1285             doprint "Exiting monitor flush due to hitting MAX_MONITOR_WAIT\n";
1286             return 1;
1287         }
1288     }
1289     print "** Monitor flushed **\n";
1290     return $bug;
1291 }
1292
1293 sub save_logs {
1294         my ($result, $basedir) = @_;
1295         my @t = localtime;
1296         my $date = sprintf "%04d%02d%02d%02d%02d%02d",
1297                 1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
1298
1299         my $type = $build_type;
1300         if ($type =~ /useconfig/) {
1301             $type = "useconfig";
1302         }
1303
1304         my $dir = "$machine-$test_type-$type-$result-$date";
1305
1306         $dir = "$basedir/$dir";
1307
1308         if (!-d $dir) {
1309             mkpath($dir) or
1310                 die "can't create $dir";
1311         }
1312
1313         my %files = (
1314                 "config" => $output_config,
1315                 "buildlog" => $buildlog,
1316                 "dmesg" => $dmesg,
1317                 "testlog" => $testlog,
1318         );
1319
1320         while (my ($name, $source) = each(%files)) {
1321                 if (-f "$source") {
1322                         cp "$source", "$dir/$name" or
1323                                 die "failed to copy $source";
1324                 }
1325         }
1326
1327         doprint "*** Saved info to $dir ***\n";
1328 }
1329
1330 sub fail {
1331
1332         if (defined($post_test)) {
1333                 run_command $post_test;
1334         }
1335
1336         if ($die_on_failure) {
1337                 dodie @_;
1338         }
1339
1340         doprint "FAILED\n";
1341
1342         my $i = $iteration;
1343
1344         # no need to reboot for just building.
1345         if (!do_not_reboot) {
1346             doprint "REBOOTING\n";
1347             reboot_to_good $sleep_time;
1348         }
1349
1350         my $name = "";
1351
1352         if (defined($test_name)) {
1353             $name = " ($test_name)";
1354         }
1355
1356         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1357         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1358         doprint "KTEST RESULT: TEST $i$name Failed: ", @_, "\n";
1359         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1360         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1361
1362         if (defined($store_failures)) {
1363             save_logs "fail", $store_failures;
1364         }
1365
1366         return 1;
1367 }
1368
1369 sub run_command {
1370     my ($command) = @_;
1371     my $dolog = 0;
1372     my $dord = 0;
1373     my $pid;
1374
1375     $command =~ s/\$SSH_USER/$ssh_user/g;
1376     $command =~ s/\$MACHINE/$machine/g;
1377
1378     doprint("$command ... ");
1379
1380     $pid = open(CMD, "$command 2>&1 |") or
1381         (fail "unable to exec $command" and return 0);
1382
1383     if (defined($opt{"LOG_FILE"})) {
1384         open(LOG, ">>$opt{LOG_FILE}") or
1385             dodie "failed to write to log";
1386         $dolog = 1;
1387     }
1388
1389     if (defined($redirect)) {
1390         open (RD, ">$redirect") or
1391             dodie "failed to write to redirect $redirect";
1392         $dord = 1;
1393     }
1394
1395     while (<CMD>) {
1396         print LOG if ($dolog);
1397         print RD  if ($dord);
1398     }
1399
1400     waitpid($pid, 0);
1401     my $failed = $?;
1402
1403     close(CMD);
1404     close(LOG) if ($dolog);
1405     close(RD)  if ($dord);
1406
1407     if ($failed) {
1408         doprint "FAILED!\n";
1409     } else {
1410         doprint "SUCCESS\n";
1411     }
1412
1413     return !$failed;
1414 }
1415
1416 sub run_ssh {
1417     my ($cmd) = @_;
1418     my $cp_exec = $ssh_exec;
1419
1420     $cp_exec =~ s/\$SSH_COMMAND/$cmd/g;
1421     return run_command "$cp_exec";
1422 }
1423
1424 sub run_scp {
1425     my ($src, $dst, $cp_scp) = @_;
1426
1427     $cp_scp =~ s/\$SRC_FILE/$src/g;
1428     $cp_scp =~ s/\$DST_FILE/$dst/g;
1429
1430     return run_command "$cp_scp";
1431 }
1432
1433 sub run_scp_install {
1434     my ($src, $dst) = @_;
1435
1436     my $cp_scp = $scp_to_target_install;
1437
1438     return run_scp($src, $dst, $cp_scp);
1439 }
1440
1441 sub run_scp_mod {
1442     my ($src, $dst) = @_;
1443
1444     my $cp_scp = $scp_to_target;
1445
1446     return run_scp($src, $dst, $cp_scp);
1447 }
1448
1449 sub get_grub_index {
1450
1451     if ($reboot_type ne "grub") {
1452         return;
1453     }
1454     return if (defined($grub_number));
1455
1456     doprint "Find grub menu ... ";
1457     $grub_number = -1;
1458
1459     my $ssh_grub = $ssh_exec;
1460     $ssh_grub =~ s,\$SSH_COMMAND,cat /boot/grub/menu.lst,g;
1461
1462     open(IN, "$ssh_grub |")
1463         or die "unable to get menu.lst";
1464
1465     my $found = 0;
1466
1467     while (<IN>) {
1468         if (/^\s*title\s+$grub_menu\s*$/) {
1469             $grub_number++;
1470             $found = 1;
1471             last;
1472         } elsif (/^\s*title\s/) {
1473             $grub_number++;
1474         }
1475     }
1476     close(IN);
1477
1478     die "Could not find '$grub_menu' in /boot/grub/menu on $machine"
1479         if (!$found);
1480     doprint "$grub_number\n";
1481 }
1482
1483 sub wait_for_input
1484 {
1485     my ($fp, $time) = @_;
1486     my $rin;
1487     my $ready;
1488     my $line;
1489     my $ch;
1490
1491     if (!defined($time)) {
1492         $time = $timeout;
1493     }
1494
1495     $rin = '';
1496     vec($rin, fileno($fp), 1) = 1;
1497     $ready = select($rin, undef, undef, $time);
1498
1499     $line = "";
1500
1501     # try to read one char at a time
1502     while (sysread $fp, $ch, 1) {
1503         $line .= $ch;
1504         last if ($ch eq "\n");
1505     }
1506
1507     if (!length($line)) {
1508         return undef;
1509     }
1510
1511     return $line;
1512 }
1513
1514 sub reboot_to {
1515     if (defined($switch_to_test)) {
1516         run_command $switch_to_test;
1517     }
1518
1519     if ($reboot_type eq "grub") {
1520         run_ssh "'(echo \"savedefault --default=$grub_number --once\" | grub --batch)'";
1521     } elsif (defined $reboot_script) {
1522         run_command "$reboot_script";
1523     }
1524     reboot;
1525 }
1526
1527 sub get_sha1 {
1528     my ($commit) = @_;
1529
1530     doprint "git rev-list --max-count=1 $commit ... ";
1531     my $sha1 = `git rev-list --max-count=1 $commit`;
1532     my $ret = $?;
1533
1534     logit $sha1;
1535
1536     if ($ret) {
1537         doprint "FAILED\n";
1538         dodie "Failed to get git $commit";
1539     }
1540
1541     print "SUCCESS\n";
1542
1543     chomp $sha1;
1544
1545     return $sha1;
1546 }
1547
1548 sub monitor {
1549     my $booted = 0;
1550     my $bug = 0;
1551     my $bug_ignored = 0;
1552     my $skip_call_trace = 0;
1553     my $loops;
1554
1555     wait_for_monitor 5;
1556
1557     my $line;
1558     my $full_line = "";
1559
1560     open(DMESG, "> $dmesg") or
1561         die "unable to write to $dmesg";
1562
1563     reboot_to;
1564
1565     my $success_start;
1566     my $failure_start;
1567     my $monitor_start = time;
1568     my $done = 0;
1569     my $version_found = 0;
1570
1571     while (!$done) {
1572
1573         if ($bug && defined($stop_after_failure) &&
1574             $stop_after_failure >= 0) {
1575             my $time = $stop_after_failure - (time - $failure_start);
1576             $line = wait_for_input($monitor_fp, $time);
1577             if (!defined($line)) {
1578                 doprint "bug timed out after $booted_timeout seconds\n";
1579                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
1580                 last;
1581             }
1582         } elsif ($booted) {
1583             $line = wait_for_input($monitor_fp, $booted_timeout);
1584             if (!defined($line)) {
1585                 my $s = $booted_timeout == 1 ? "" : "s";
1586                 doprint "Successful boot found: break after $booted_timeout second$s\n";
1587                 last;
1588             }
1589         } else {
1590             $line = wait_for_input($monitor_fp);
1591             if (!defined($line)) {
1592                 my $s = $timeout == 1 ? "" : "s";
1593                 doprint "Timed out after $timeout second$s\n";
1594                 last;
1595             }
1596         }
1597
1598         doprint $line;
1599         print DMESG $line;
1600
1601         # we are not guaranteed to get a full line
1602         $full_line .= $line;
1603
1604         if ($full_line =~ /$success_line/) {
1605             $booted = 1;
1606             $success_start = time;
1607         }
1608
1609         if ($booted && defined($stop_after_success) &&
1610             $stop_after_success >= 0) {
1611             my $now = time;
1612             if ($now - $success_start >= $stop_after_success) {
1613                 doprint "Test forced to stop after $stop_after_success seconds after success\n";
1614                 last;
1615             }
1616         }
1617
1618         if ($full_line =~ /\[ backtrace testing \]/) {
1619             $skip_call_trace = 1;
1620         }
1621
1622         if ($full_line =~ /call trace:/i) {
1623             if (!$bug && !$skip_call_trace) {
1624                 if ($ignore_errors) {
1625                     $bug_ignored = 1;
1626                 } else {
1627                     $bug = 1;
1628                     $failure_start = time;
1629                 }
1630             }
1631         }
1632
1633         if ($bug && defined($stop_after_failure) &&
1634             $stop_after_failure >= 0) {
1635             my $now = time;
1636             if ($now - $failure_start >= $stop_after_failure) {
1637                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
1638                 last;
1639             }
1640         }
1641
1642         if ($full_line =~ /\[ end of backtrace testing \]/) {
1643             $skip_call_trace = 0;
1644         }
1645
1646         if ($full_line =~ /Kernel panic -/) {
1647             $failure_start = time;
1648             $bug = 1;
1649         }
1650
1651         # Detect triple faults by testing the banner
1652         if ($full_line =~ /\bLinux version (\S+).*\n/) {
1653             if ($1 eq $version) {
1654                 $version_found = 1;
1655             } elsif ($version_found && $detect_triplefault) {
1656                 # We already booted into the kernel we are testing,
1657                 # but now we booted into another kernel?
1658                 # Consider this a triple fault.
1659                 doprint "Aleady booted in Linux kernel $version, but now\n";
1660                 doprint "we booted into Linux kernel $1.\n";
1661                 doprint "Assuming that this is a triple fault.\n";
1662                 doprint "To disable this: set DETECT_TRIPLE_FAULT to 0\n";
1663                 last;
1664             }
1665         }
1666
1667         if ($line =~ /\n/) {
1668             $full_line = "";
1669         }
1670
1671         if ($stop_test_after > 0 && !$booted && !$bug) {
1672             if (time - $monitor_start > $stop_test_after) {
1673                 doprint "STOP_TEST_AFTER ($stop_test_after seconds) timed out\n";
1674                 $done = 1;
1675             }
1676         }
1677     }
1678
1679     close(DMESG);
1680
1681     if ($bug) {
1682         return 0 if ($in_bisect);
1683         fail "failed - got a bug report" and return 0;
1684     }
1685
1686     if (!$booted) {
1687         return 0 if ($in_bisect);
1688         fail "failed - never got a boot prompt." and return 0;
1689     }
1690
1691     if ($bug_ignored) {
1692         doprint "WARNING: Call Trace detected but ignored due to IGNORE_ERRORS=1\n";
1693     }
1694
1695     return 1;
1696 }
1697
1698 sub eval_kernel_version {
1699     my ($option) = @_;
1700
1701     $option =~ s/\$KERNEL_VERSION/$version/g;
1702
1703     return $option;
1704 }
1705
1706 sub do_post_install {
1707
1708     return if (!defined($post_install));
1709
1710     my $cp_post_install = eval_kernel_version $post_install;
1711     run_command "$cp_post_install" or
1712         dodie "Failed to run post install";
1713 }
1714
1715 sub install {
1716
1717     return if ($no_install);
1718
1719     if (defined($pre_install)) {
1720         my $cp_pre_install = eval_kernel_version $pre_install;
1721         run_command "$cp_pre_install" or
1722             dodie "Failed to run pre install";
1723     }
1724
1725     my $cp_target = eval_kernel_version $target_image;
1726
1727     run_scp_install "$outputdir/$build_target", "$cp_target" or
1728         dodie "failed to copy image";
1729
1730     my $install_mods = 0;
1731
1732     # should we process modules?
1733     $install_mods = 0;
1734     open(IN, "$output_config") or dodie("Can't read config file");
1735     while (<IN>) {
1736         if (/CONFIG_MODULES(=y)?/) {
1737             $install_mods = 1 if (defined($1));
1738             last;
1739         }
1740     }
1741     close(IN);
1742
1743     if (!$install_mods) {
1744         do_post_install;
1745         doprint "No modules needed\n";
1746         return;
1747     }
1748
1749     run_command "$make INSTALL_MOD_STRIP=1 INSTALL_MOD_PATH=$tmpdir modules_install" or
1750         dodie "Failed to install modules";
1751
1752     my $modlib = "/lib/modules/$version";
1753     my $modtar = "ktest-mods.tar.bz2";
1754
1755     run_ssh "rm -rf $modlib" or
1756         dodie "failed to remove old mods: $modlib";
1757
1758     # would be nice if scp -r did not follow symbolic links
1759     run_command "cd $tmpdir && tar -cjf $modtar lib/modules/$version" or
1760         dodie "making tarball";
1761
1762     run_scp_mod "$tmpdir/$modtar", "/tmp" or
1763         dodie "failed to copy modules";
1764
1765     unlink "$tmpdir/$modtar";
1766
1767     run_ssh "'(cd / && tar xjf /tmp/$modtar)'" or
1768         dodie "failed to tar modules";
1769
1770     run_ssh "rm -f /tmp/$modtar";
1771
1772     do_post_install;
1773 }
1774
1775 sub get_version {
1776     # get the release name
1777     return if ($have_version);
1778     doprint "$make kernelrelease ... ";
1779     $version = `$make kernelrelease | tail -1`;
1780     chomp($version);
1781     doprint "$version\n";
1782     $have_version = 1;
1783 }
1784
1785 sub start_monitor_and_boot {
1786     # Make sure the stable kernel has finished booting
1787     start_monitor;
1788     wait_for_monitor 5;
1789     end_monitor;
1790
1791     get_grub_index;
1792     get_version;
1793     install;
1794
1795     start_monitor;
1796     return monitor;
1797 }
1798
1799 sub check_buildlog {
1800     my ($patch) = @_;
1801
1802     my @files = `git show $patch | diffstat -l`;
1803
1804     open(IN, "git show $patch |") or
1805         dodie "failed to show $patch";
1806     while (<IN>) {
1807         if (m,^--- a/(.*),) {
1808             chomp $1;
1809             $files[$#files] = $1;
1810         }
1811     }
1812     close(IN);
1813
1814     open(IN, $buildlog) or dodie "Can't open $buildlog";
1815     while (<IN>) {
1816         if (/^\s*(.*?):.*(warning|error)/) {
1817             my $err = $1;
1818             foreach my $file (@files) {
1819                 my $fullpath = "$builddir/$file";
1820                 if ($file eq $err || $fullpath eq $err) {
1821                     fail "$file built with warnings" and return 0;
1822                 }
1823             }
1824         }
1825     }
1826     close(IN);
1827
1828     return 1;
1829 }
1830
1831 sub apply_min_config {
1832     my $outconfig = "$output_config.new";
1833
1834     # Read the config file and remove anything that
1835     # is in the force_config hash (from minconfig and others)
1836     # then add the force config back.
1837
1838     doprint "Applying minimum configurations into $output_config.new\n";
1839
1840     open (OUT, ">$outconfig") or
1841         dodie "Can't create $outconfig";
1842
1843     if (-f $output_config) {
1844         open (IN, $output_config) or
1845             dodie "Failed to open $output_config";
1846         while (<IN>) {
1847             if (/^(# )?(CONFIG_[^\s=]*)/) {
1848                 next if (defined($force_config{$2}));
1849             }
1850             print OUT;
1851         }
1852         close IN;
1853     }
1854     foreach my $config (keys %force_config) {
1855         print OUT "$force_config{$config}\n";
1856     }
1857     close OUT;
1858
1859     run_command "mv $outconfig $output_config";
1860 }
1861
1862 sub make_oldconfig {
1863
1864     my @force_list = keys %force_config;
1865
1866     if ($#force_list >= 0) {
1867         apply_min_config;
1868     }
1869
1870     if (!run_command "$make oldnoconfig") {
1871         # Perhaps oldnoconfig doesn't exist in this version of the kernel
1872         # try a yes '' | oldconfig
1873         doprint "oldnoconfig failed, trying yes '' | make oldconfig\n";
1874         run_command "yes '' | $make oldconfig" or
1875             dodie "failed make config oldconfig";
1876     }
1877 }
1878
1879 # read a config file and use this to force new configs.
1880 sub load_force_config {
1881     my ($config) = @_;
1882
1883     doprint "Loading force configs from $config\n";
1884     open(IN, $config) or
1885         dodie "failed to read $config";
1886     while (<IN>) {
1887         chomp;
1888         if (/^(CONFIG[^\s=]*)(\s*=.*)/) {
1889             $force_config{$1} = $_;
1890         } elsif (/^# (CONFIG_\S*) is not set/) {
1891             $force_config{$1} = $_;
1892         }
1893     }
1894     close IN;
1895 }
1896
1897 sub build {
1898     my ($type) = @_;
1899
1900     unlink $buildlog;
1901
1902     # Failed builds should not reboot the target
1903     my $save_no_reboot = $no_reboot;
1904     $no_reboot = 1;
1905
1906     # Calculate a new version from here.
1907     $have_version = 0;
1908
1909     if (defined($pre_build)) {
1910         my $ret = run_command $pre_build;
1911         if (!$ret && defined($pre_build_die) &&
1912             $pre_build_die) {
1913             dodie "failed to pre_build\n";
1914         }
1915     }
1916
1917     if ($type =~ /^useconfig:(.*)/) {
1918         run_command "cp $1 $output_config" or
1919             dodie "could not copy $1 to .config";
1920
1921         $type = "oldconfig";
1922     }
1923
1924     # old config can ask questions
1925     if ($type eq "oldconfig") {
1926         $type = "oldnoconfig";
1927
1928         # allow for empty configs
1929         run_command "touch $output_config";
1930
1931         if (!$noclean) {
1932             run_command "mv $output_config $outputdir/config_temp" or
1933                 dodie "moving .config";
1934
1935             run_command "$make mrproper" or dodie "make mrproper";
1936
1937             run_command "mv $outputdir/config_temp $output_config" or
1938                 dodie "moving config_temp";
1939         }
1940
1941     } elsif (!$noclean) {
1942         unlink "$output_config";
1943         run_command "$make mrproper" or
1944             dodie "make mrproper";
1945     }
1946
1947     # add something to distinguish this build
1948     open(OUT, "> $outputdir/localversion") or dodie("Can't make localversion file");
1949     print OUT "$localversion\n";
1950     close(OUT);
1951
1952     if (defined($minconfig)) {
1953         load_force_config($minconfig);
1954     }
1955
1956     if ($type ne "oldnoconfig") {
1957         run_command "$make $type" or
1958             dodie "failed make config";
1959     }
1960     # Run old config regardless, to enforce min configurations
1961     make_oldconfig;
1962
1963     $redirect = "$buildlog";
1964     my $build_ret = run_command "$make $build_options";
1965     undef $redirect;
1966
1967     if (defined($post_build)) {
1968         # Because a post build may change the kernel version
1969         # do it now.
1970         get_version;
1971         my $ret = run_command $post_build;
1972         if (!$ret && defined($post_build_die) &&
1973             $post_build_die) {
1974             dodie "failed to post_build\n";
1975         }
1976     }
1977
1978     if (!$build_ret) {
1979         # bisect may need this to pass
1980         if ($in_bisect) {
1981             $no_reboot = $save_no_reboot;
1982             return 0;
1983         }
1984         fail "failed build" and return 0;
1985     }
1986
1987     $no_reboot = $save_no_reboot;
1988
1989     return 1;
1990 }
1991
1992 sub halt {
1993     if (!run_ssh "halt" or defined($power_off)) {
1994         if (defined($poweroff_after_halt)) {
1995             sleep $poweroff_after_halt;
1996             run_command "$power_off";
1997         }
1998     } else {
1999         # nope? the zap it!
2000         run_command "$power_off";
2001     }
2002 }
2003
2004 sub success {
2005     my ($i) = @_;
2006
2007     if (defined($post_test)) {
2008         run_command $post_test;
2009     }
2010
2011     $successes++;
2012
2013     my $name = "";
2014
2015     if (defined($test_name)) {
2016         $name = " ($test_name)";
2017     }
2018
2019     doprint "\n\n*******************************************\n";
2020     doprint     "*******************************************\n";
2021     doprint     "KTEST RESULT: TEST $i$name SUCCESS!!!!         **\n";
2022     doprint     "*******************************************\n";
2023     doprint     "*******************************************\n";
2024
2025     if (defined($store_successes)) {
2026         save_logs "success", $store_successes;
2027     }
2028
2029     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
2030         doprint "Reboot and wait $sleep_time seconds\n";
2031         reboot_to_good $sleep_time;
2032     }
2033 }
2034
2035 sub answer_bisect {
2036     for (;;) {
2037         doprint "Pass or fail? [p/f]";
2038         my $ans = <STDIN>;
2039         chomp $ans;
2040         if ($ans eq "p" || $ans eq "P") {
2041             return 1;
2042         } elsif ($ans eq "f" || $ans eq "F") {
2043             return 0;
2044         } else {
2045             print "Please answer 'P' or 'F'\n";
2046         }
2047     }
2048 }
2049
2050 sub child_run_test {
2051     my $failed = 0;
2052
2053     # child should have no power
2054     $reboot_on_error = 0;
2055     $poweroff_on_error = 0;
2056     $die_on_failure = 1;
2057
2058     $redirect = "$testlog";
2059     run_command $run_test or $failed = 1;
2060     undef $redirect;
2061
2062     exit $failed;
2063 }
2064
2065 my $child_done;
2066
2067 sub child_finished {
2068     $child_done = 1;
2069 }
2070
2071 sub do_run_test {
2072     my $child_pid;
2073     my $child_exit;
2074     my $line;
2075     my $full_line;
2076     my $bug = 0;
2077     my $bug_ignored = 0;
2078
2079     wait_for_monitor 1;
2080
2081     doprint "run test $run_test\n";
2082
2083     $child_done = 0;
2084
2085     $SIG{CHLD} = qw(child_finished);
2086
2087     $child_pid = fork;
2088
2089     child_run_test if (!$child_pid);
2090
2091     $full_line = "";
2092
2093     do {
2094         $line = wait_for_input($monitor_fp, 1);
2095         if (defined($line)) {
2096
2097             # we are not guaranteed to get a full line
2098             $full_line .= $line;
2099             doprint $line;
2100
2101             if ($full_line =~ /call trace:/i) {
2102                 if ($ignore_errors) {
2103                     $bug_ignored = 1;
2104                 } else {
2105                     $bug = 1;
2106                 }
2107             }
2108
2109             if ($full_line =~ /Kernel panic -/) {
2110                 $bug = 1;
2111             }
2112
2113             if ($line =~ /\n/) {
2114                 $full_line = "";
2115             }
2116         }
2117     } while (!$child_done && !$bug);
2118
2119     if (!$bug && $bug_ignored) {
2120         doprint "WARNING: Call Trace detected but ignored due to IGNORE_ERRORS=1\n";
2121     }
2122
2123     if ($bug) {
2124         my $failure_start = time;
2125         my $now;
2126         do {
2127             $line = wait_for_input($monitor_fp, 1);
2128             if (defined($line)) {
2129                 doprint $line;
2130             }
2131             $now = time;
2132             if ($now - $failure_start >= $stop_after_failure) {
2133                 last;
2134             }
2135         } while (defined($line));
2136
2137         doprint "Detected kernel crash!\n";
2138         # kill the child with extreme prejudice
2139         kill 9, $child_pid;
2140     }
2141
2142     waitpid $child_pid, 0;
2143     $child_exit = $?;
2144
2145     if (!$bug && $in_bisect) {
2146         if (defined($bisect_ret_good)) {
2147             if ($child_exit == $bisect_ret_good) {
2148                 return 1;
2149             }
2150         }
2151         if (defined($bisect_ret_skip)) {
2152             if ($child_exit == $bisect_ret_skip) {
2153                 return -1;
2154             }
2155         }
2156         if (defined($bisect_ret_abort)) {
2157             if ($child_exit == $bisect_ret_abort) {
2158                 fail "test abort" and return -2;
2159             }
2160         }
2161         if (defined($bisect_ret_bad)) {
2162             if ($child_exit == $bisect_ret_skip) {
2163                 return 0;
2164             }
2165         }
2166         if (defined($bisect_ret_default)) {
2167             if ($bisect_ret_default eq "good") {
2168                 return 1;
2169             } elsif ($bisect_ret_default eq "bad") {
2170                 return 0;
2171             } elsif ($bisect_ret_default eq "skip") {
2172                 return -1;
2173             } elsif ($bisect_ret_default eq "abort") {
2174                 return -2;
2175             } else {
2176                 fail "unknown default action: $bisect_ret_default"
2177                     and return -2;
2178             }
2179         }
2180     }
2181
2182     if ($bug || $child_exit) {
2183         return 0 if $in_bisect;
2184         fail "test failed" and return 0;
2185     }
2186     return 1;
2187 }
2188
2189 sub run_git_bisect {
2190     my ($command) = @_;
2191
2192     doprint "$command ... ";
2193
2194     my $output = `$command 2>&1`;
2195     my $ret = $?;
2196
2197     logit $output;
2198
2199     if ($ret) {
2200         doprint "FAILED\n";
2201         dodie "Failed to git bisect";
2202     }
2203
2204     doprint "SUCCESS\n";
2205     if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
2206         doprint "$1 [$2]\n";
2207     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
2208         $bisect_bad_commit = $1;
2209         doprint "Found bad commit... $1\n";
2210         return 0;
2211     } else {
2212         # we already logged it, just print it now.
2213         print $output;
2214     }
2215
2216     return 1;
2217 }
2218
2219 sub bisect_reboot {
2220     doprint "Reboot and sleep $bisect_sleep_time seconds\n";
2221     reboot_to_good $bisect_sleep_time;
2222 }
2223
2224 # returns 1 on success, 0 on failure, -1 on skip
2225 sub run_bisect_test {
2226     my ($type, $buildtype) = @_;
2227
2228     my $failed = 0;
2229     my $result;
2230     my $output;
2231     my $ret;
2232
2233     $in_bisect = 1;
2234
2235     build $buildtype or $failed = 1;
2236
2237     if ($type ne "build") {
2238         if ($failed && $bisect_skip) {
2239             $in_bisect = 0;
2240             return -1;
2241         }
2242         dodie "Failed on build" if $failed;
2243
2244         # Now boot the box
2245         start_monitor_and_boot or $failed = 1;
2246
2247         if ($type ne "boot") {
2248             if ($failed && $bisect_skip) {
2249                 end_monitor;
2250                 bisect_reboot;
2251                 $in_bisect = 0;
2252                 return -1;
2253             }
2254             dodie "Failed on boot" if $failed;
2255
2256             do_run_test or $failed = 1;
2257         }
2258         end_monitor;
2259     }
2260
2261     if ($failed) {
2262         $result = 0;
2263     } else {
2264         $result = 1;
2265     }
2266
2267     # reboot the box to a kernel we can ssh to
2268     if ($type ne "build") {
2269         bisect_reboot;
2270     }
2271     $in_bisect = 0;
2272
2273     return $result;
2274 }
2275
2276 sub run_bisect {
2277     my ($type) = @_;
2278     my $buildtype = "oldconfig";
2279
2280     # We should have a minconfig to use?
2281     if (defined($minconfig)) {
2282         $buildtype = "useconfig:$minconfig";
2283     }
2284
2285     my $ret = run_bisect_test $type, $buildtype;
2286
2287     if ($bisect_manual) {
2288         $ret = answer_bisect;
2289     }
2290
2291     # Are we looking for where it worked, not failed?
2292     if ($reverse_bisect && $ret >= 0) {
2293         $ret = !$ret;
2294     }
2295
2296     if ($ret > 0) {
2297         return "good";
2298     } elsif ($ret == 0) {
2299         return  "bad";
2300     } elsif ($bisect_skip) {
2301         doprint "HIT A BAD COMMIT ... SKIPPING\n";
2302         return "skip";
2303     }
2304 }
2305
2306 sub update_bisect_replay {
2307     my $tmp_log = "$tmpdir/ktest_bisect_log";
2308     run_command "git bisect log > $tmp_log" or
2309         die "can't create bisect log";
2310     return $tmp_log;
2311 }
2312
2313 sub bisect {
2314     my ($i) = @_;
2315
2316     my $result;
2317
2318     die "BISECT_GOOD[$i] not defined\n" if (!defined($bisect_good));
2319     die "BISECT_BAD[$i] not defined\n"  if (!defined($bisect_bad));
2320     die "BISECT_TYPE[$i] not defined\n" if (!defined($bisect_type));
2321
2322     my $good = $bisect_good;
2323     my $bad = $bisect_bad;
2324     my $type = $bisect_type;
2325     my $start = $bisect_start;
2326     my $replay = $bisect_replay;
2327     my $start_files = $bisect_files;
2328
2329     if (defined($start_files)) {
2330         $start_files = " -- " . $start_files;
2331     } else {
2332         $start_files = "";
2333     }
2334
2335     # convert to true sha1's
2336     $good = get_sha1($good);
2337     $bad = get_sha1($bad);
2338
2339     if (defined($bisect_reverse) && $bisect_reverse == 1) {
2340         doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
2341         $reverse_bisect = 1;
2342     } else {
2343         $reverse_bisect = 0;
2344     }
2345
2346     # Can't have a test without having a test to run
2347     if ($type eq "test" && !defined($run_test)) {
2348         $type = "boot";
2349     }
2350
2351     # Check if a bisect was running
2352     my $bisect_start_file = "$builddir/.git/BISECT_START";
2353
2354     my $check = $bisect_check;
2355     my $do_check = defined($check) && $check ne "0";
2356
2357     if ( -f $bisect_start_file ) {
2358         print "Bisect in progress found\n";
2359         if ($do_check) {
2360             print " If you say yes, then no checks of good or bad will be done\n";
2361         }
2362         if (defined($replay)) {
2363             print "** BISECT_REPLAY is defined in config file **";
2364             print " Ignore config option and perform new git bisect log?\n";
2365             if (read_ync " (yes, no, or cancel) ") {
2366                 $replay = update_bisect_replay;
2367                 $do_check = 0;
2368             }
2369         } elsif (read_yn "read git log and continue?") {
2370             $replay = update_bisect_replay;
2371             $do_check = 0;
2372         }
2373     }
2374
2375     if ($do_check) {
2376
2377         # get current HEAD
2378         my $head = get_sha1("HEAD");
2379
2380         if ($check ne "good") {
2381             doprint "TESTING BISECT BAD [$bad]\n";
2382             run_command "git checkout $bad" or
2383                 die "Failed to checkout $bad";
2384
2385             $result = run_bisect $type;
2386
2387             if ($result ne "bad") {
2388                 fail "Tested BISECT_BAD [$bad] and it succeeded" and return 0;
2389             }
2390         }
2391
2392         if ($check ne "bad") {
2393             doprint "TESTING BISECT GOOD [$good]\n";
2394             run_command "git checkout $good" or
2395                 die "Failed to checkout $good";
2396
2397             $result = run_bisect $type;
2398
2399             if ($result ne "good") {
2400                 fail "Tested BISECT_GOOD [$good] and it failed" and return 0;
2401             }
2402         }
2403
2404         # checkout where we started
2405         run_command "git checkout $head" or
2406             die "Failed to checkout $head";
2407     }
2408
2409     run_command "git bisect start$start_files" or
2410         dodie "could not start bisect";
2411
2412     run_command "git bisect good $good" or
2413         dodie "could not set bisect good to $good";
2414
2415     run_git_bisect "git bisect bad $bad" or
2416         dodie "could not set bisect bad to $bad";
2417
2418     if (defined($replay)) {
2419         run_command "git bisect replay $replay" or
2420             dodie "failed to run replay";
2421     }
2422
2423     if (defined($start)) {
2424         run_command "git checkout $start" or
2425             dodie "failed to checkout $start";
2426     }
2427
2428     my $test;
2429     do {
2430         $result = run_bisect $type;
2431         $test = run_git_bisect "git bisect $result";
2432     } while ($test);
2433
2434     run_command "git bisect log" or
2435         dodie "could not capture git bisect log";
2436
2437     run_command "git bisect reset" or
2438         dodie "could not reset git bisect";
2439
2440     doprint "Bad commit was [$bisect_bad_commit]\n";
2441
2442     success $i;
2443 }
2444
2445 # config_ignore holds the configs that were set (or unset) for
2446 # a good config and we will ignore these configs for the rest
2447 # of a config bisect. These configs stay as they were.
2448 my %config_ignore;
2449
2450 # config_set holds what all configs were set as.
2451 my %config_set;
2452
2453 # config_off holds the set of configs that the bad config had disabled.
2454 # We need to record them and set them in the .config when running
2455 # oldnoconfig, because oldnoconfig does not turn off new symbols, but
2456 # instead just keeps the defaults.
2457 my %config_off;
2458
2459 # config_off_tmp holds a set of configs to turn off for now
2460 my @config_off_tmp;
2461
2462 # config_list is the set of configs that are being tested
2463 my %config_list;
2464 my %null_config;
2465
2466 my %dependency;
2467
2468 sub assign_configs {
2469     my ($hash, $config) = @_;
2470
2471     open (IN, $config)
2472         or dodie "Failed to read $config";
2473
2474     while (<IN>) {
2475         if (/^((CONFIG\S*)=.*)/) {
2476             ${$hash}{$2} = $1;
2477         }
2478     }
2479
2480     close(IN);
2481 }
2482
2483 sub process_config_ignore {
2484     my ($config) = @_;
2485
2486     assign_configs \%config_ignore, $config;
2487 }
2488
2489 sub read_current_config {
2490     my ($config_ref) = @_;
2491
2492     %{$config_ref} = ();
2493     undef %{$config_ref};
2494
2495     my @key = keys %{$config_ref};
2496     if ($#key >= 0) {
2497         print "did not delete!\n";
2498         exit;
2499     }
2500     open (IN, "$output_config");
2501
2502     while (<IN>) {
2503         if (/^(CONFIG\S+)=(.*)/) {
2504             ${$config_ref}{$1} = $2;
2505         }
2506     }
2507     close(IN);
2508 }
2509
2510 sub get_dependencies {
2511     my ($config) = @_;
2512
2513     my $arr = $dependency{$config};
2514     if (!defined($arr)) {
2515         return ();
2516     }
2517
2518     my @deps = @{$arr};
2519
2520     foreach my $dep (@{$arr}) {
2521         print "ADD DEP $dep\n";
2522         @deps = (@deps, get_dependencies $dep);
2523     }
2524
2525     return @deps;
2526 }
2527
2528 sub create_config {
2529     my @configs = @_;
2530
2531     open(OUT, ">$output_config") or dodie "Can not write to $output_config";
2532
2533     foreach my $config (@configs) {
2534         print OUT "$config_set{$config}\n";
2535         my @deps = get_dependencies $config;
2536         foreach my $dep (@deps) {
2537             print OUT "$config_set{$dep}\n";
2538         }
2539     }
2540
2541     # turn off configs to keep off
2542     foreach my $config (keys %config_off) {
2543         print OUT "# $config is not set\n";
2544     }
2545
2546     # turn off configs that should be off for now
2547     foreach my $config (@config_off_tmp) {
2548         print OUT "# $config is not set\n";
2549     }
2550
2551     foreach my $config (keys %config_ignore) {
2552         print OUT "$config_ignore{$config}\n";
2553     }
2554     close(OUT);
2555
2556     make_oldconfig;
2557 }
2558
2559 sub compare_configs {
2560     my (%a, %b) = @_;
2561
2562     foreach my $item (keys %a) {
2563         if (!defined($b{$item})) {
2564             print "diff $item\n";
2565             return 1;
2566         }
2567         delete $b{$item};
2568     }
2569
2570     my @keys = keys %b;
2571     if ($#keys) {
2572         print "diff2 $keys[0]\n";
2573     }
2574     return -1 if ($#keys >= 0);
2575
2576     return 0;
2577 }
2578
2579 sub run_config_bisect_test {
2580     my ($type) = @_;
2581
2582     return run_bisect_test $type, "oldconfig";
2583 }
2584
2585 sub process_passed {
2586     my (%configs) = @_;
2587
2588     doprint "These configs had no failure: (Enabling them for further compiles)\n";
2589     # Passed! All these configs are part of a good compile.
2590     # Add them to the min options.
2591     foreach my $config (keys %configs) {
2592         if (defined($config_list{$config})) {
2593             doprint " removing $config\n";
2594             $config_ignore{$config} = $config_list{$config};
2595             delete $config_list{$config};
2596         }
2597     }
2598     doprint "config copied to $outputdir/config_good\n";
2599     run_command "cp -f $output_config $outputdir/config_good";
2600 }
2601
2602 sub process_failed {
2603     my ($config) = @_;
2604
2605     doprint "\n\n***************************************\n";
2606     doprint "Found bad config: $config\n";
2607     doprint "***************************************\n\n";
2608 }
2609
2610 sub run_config_bisect {
2611
2612     my @start_list = keys %config_list;
2613
2614     if ($#start_list < 0) {
2615         doprint "No more configs to test!!!\n";
2616         return -1;
2617     }
2618
2619     doprint "***** RUN TEST ***\n";
2620     my $type = $config_bisect_type;
2621     my $ret;
2622     my %current_config;
2623
2624     my $count = $#start_list + 1;
2625     doprint "  $count configs to test\n";
2626
2627     my $half = int($#start_list / 2);
2628
2629     do {
2630         my @tophalf = @start_list[0 .. $half];
2631
2632         # keep the bottom half off
2633         if ($half < $#start_list) {
2634             @config_off_tmp = @start_list[$half + 1 .. $#start_list];
2635         } else {
2636             @config_off_tmp = ();
2637         }
2638
2639         create_config @tophalf;
2640         read_current_config \%current_config;
2641
2642         $count = $#tophalf + 1;
2643         doprint "Testing $count configs\n";
2644         my $found = 0;
2645         # make sure we test something
2646         foreach my $config (@tophalf) {
2647             if (defined($current_config{$config})) {
2648                 logit " $config\n";
2649                 $found = 1;
2650             }
2651         }
2652         if (!$found) {
2653             # try the other half
2654             doprint "Top half produced no set configs, trying bottom half\n";
2655
2656             # keep the top half off
2657             @config_off_tmp = @tophalf;
2658             @tophalf = @start_list[$half + 1 .. $#start_list];
2659
2660             create_config @tophalf;
2661             read_current_config \%current_config;
2662             foreach my $config (@tophalf) {
2663                 if (defined($current_config{$config})) {
2664                     logit " $config\n";
2665                     $found = 1;
2666                 }
2667             }
2668             if (!$found) {
2669                 doprint "Failed: Can't make new config with current configs\n";
2670                 foreach my $config (@start_list) {
2671                     doprint "  CONFIG: $config\n";
2672                 }
2673                 return -1;
2674             }
2675             $count = $#tophalf + 1;
2676             doprint "Testing $count configs\n";
2677         }
2678
2679         $ret = run_config_bisect_test $type;
2680         if ($bisect_manual) {
2681             $ret = answer_bisect;
2682         }
2683         if ($ret) {
2684             process_passed %current_config;
2685             return 0;
2686         }
2687
2688         doprint "This config had a failure.\n";
2689         doprint "Removing these configs that were not set in this config:\n";
2690         doprint "config copied to $outputdir/config_bad\n";
2691         run_command "cp -f $output_config $outputdir/config_bad";
2692
2693         # A config exists in this group that was bad.
2694         foreach my $config (keys %config_list) {
2695             if (!defined($current_config{$config})) {
2696                 doprint " removing $config\n";
2697                 delete $config_list{$config};
2698             }
2699         }
2700
2701         @start_list = @tophalf;
2702
2703         if ($#start_list == 0) {
2704             process_failed $start_list[0];
2705             return 1;
2706         }
2707
2708         # remove half the configs we are looking at and see if
2709         # they are good.
2710         $half = int($#start_list / 2);
2711     } while ($#start_list > 0);
2712
2713     # we found a single config, try it again unless we are running manually
2714
2715     if ($bisect_manual) {
2716         process_failed $start_list[0];
2717         return 1;
2718     }
2719
2720     my @tophalf = @start_list[0 .. 0];
2721
2722     $ret = run_config_bisect_test $type;
2723     if ($ret) {
2724         process_passed %current_config;
2725         return 0;
2726     }
2727
2728     process_failed $start_list[0];
2729     return 1;
2730 }
2731
2732 sub config_bisect {
2733     my ($i) = @_;
2734
2735     my $start_config = $config_bisect;
2736
2737     my $tmpconfig = "$tmpdir/use_config";
2738
2739     if (defined($config_bisect_good)) {
2740         process_config_ignore $config_bisect_good;
2741     }
2742
2743     # Make the file with the bad config and the min config
2744     if (defined($minconfig)) {
2745         # read the min config for things to ignore
2746         run_command "cp $minconfig $tmpconfig" or
2747             dodie "failed to copy $minconfig to $tmpconfig";
2748     } else {
2749         unlink $tmpconfig;
2750     }
2751
2752     if (-f $tmpconfig) {
2753         load_force_config($tmpconfig);
2754         process_config_ignore $tmpconfig;
2755     }
2756
2757     # now process the start config
2758     run_command "cp $start_config $output_config" or
2759         dodie "failed to copy $start_config to $output_config";
2760
2761     # read directly what we want to check
2762     my %config_check;
2763     open (IN, $output_config)
2764         or dodie "failed to open $output_config";
2765
2766     while (<IN>) {
2767         if (/^((CONFIG\S*)=.*)/) {
2768             $config_check{$2} = $1;
2769         }
2770     }
2771     close(IN);
2772
2773     # Now run oldconfig with the minconfig
2774     make_oldconfig;
2775
2776     # check to see what we lost (or gained)
2777     open (IN, $output_config)
2778         or dodie "Failed to read $start_config";
2779
2780     my %removed_configs;
2781     my %added_configs;
2782
2783     while (<IN>) {
2784         if (/^((CONFIG\S*)=.*)/) {
2785             # save off all options
2786             $config_set{$2} = $1;
2787             if (defined($config_check{$2})) {
2788                 if (defined($config_ignore{$2})) {
2789                     $removed_configs{$2} = $1;
2790                 } else {
2791                     $config_list{$2} = $1;
2792                 }
2793             } elsif (!defined($config_ignore{$2})) {
2794                 $added_configs{$2} = $1;
2795                 $config_list{$2} = $1;
2796             }
2797         } elsif (/^# ((CONFIG\S*).*)/) {
2798             # Keep these configs disabled
2799             $config_set{$2} = $1;
2800             $config_off{$2} = $1;
2801         }
2802     }
2803     close(IN);
2804
2805     my @confs = keys %removed_configs;
2806     if ($#confs >= 0) {
2807         doprint "Configs overridden by default configs and removed from check:\n";
2808         foreach my $config (@confs) {
2809             doprint " $config\n";
2810         }
2811     }
2812     @confs = keys %added_configs;
2813     if ($#confs >= 0) {
2814         doprint "Configs appearing in make oldconfig and added:\n";
2815         foreach my $config (@confs) {
2816             doprint " $config\n";
2817         }
2818     }
2819
2820     my %config_test;
2821     my $once = 0;
2822
2823     @config_off_tmp = ();
2824
2825     # Sometimes kconfig does weird things. We must make sure
2826     # that the config we autocreate has everything we need
2827     # to test, otherwise we may miss testing configs, or
2828     # may not be able to create a new config.
2829     # Here we create a config with everything set.
2830     create_config (keys %config_list);
2831     read_current_config \%config_test;
2832     foreach my $config (keys %config_list) {
2833         if (!defined($config_test{$config})) {
2834             if (!$once) {
2835                 $once = 1;
2836                 doprint "Configs not produced by kconfig (will not be checked):\n";
2837             }
2838             doprint "  $config\n";
2839             delete $config_list{$config};
2840         }
2841     }
2842     my $ret;
2843
2844     if (defined($config_bisect_check) && $config_bisect_check) {
2845         doprint " Checking to make sure bad config with min config fails\n";
2846         create_config keys %config_list;
2847         $ret = run_config_bisect_test $config_bisect_type;
2848         if ($ret) {
2849             doprint " FAILED! Bad config with min config boots fine\n";
2850             return -1;
2851         }
2852         doprint " Bad config with min config fails as expected\n";
2853     }
2854
2855     do {
2856         $ret = run_config_bisect;
2857     } while (!$ret);
2858
2859     return $ret if ($ret < 0);
2860
2861     success $i;
2862 }
2863
2864 sub patchcheck_reboot {
2865     doprint "Reboot and sleep $patchcheck_sleep_time seconds\n";
2866     reboot_to_good $patchcheck_sleep_time;
2867 }
2868
2869 sub patchcheck {
2870     my ($i) = @_;
2871
2872     die "PATCHCHECK_START[$i] not defined\n"
2873         if (!defined($patchcheck_start));
2874     die "PATCHCHECK_TYPE[$i] not defined\n"
2875         if (!defined($patchcheck_type));
2876
2877     my $start = $patchcheck_start;
2878
2879     my $end = "HEAD";
2880     if (defined($patchcheck_end)) {
2881         $end = $patchcheck_end;
2882     }
2883
2884     # Get the true sha1's since we can use things like HEAD~3
2885     $start = get_sha1($start);
2886     $end = get_sha1($end);
2887
2888     my $type = $patchcheck_type;
2889
2890     # Can't have a test without having a test to run
2891     if ($type eq "test" && !defined($run_test)) {
2892         $type = "boot";
2893     }
2894
2895     open (IN, "git log --pretty=oneline $end|") or
2896         dodie "could not get git list";
2897
2898     my @list;
2899
2900     while (<IN>) {
2901         chomp;
2902         $list[$#list+1] = $_;
2903         last if (/^$start/);
2904     }
2905     close(IN);
2906
2907     if ($list[$#list] !~ /^$start/) {
2908         fail "SHA1 $start not found";
2909     }
2910
2911     # go backwards in the list
2912     @list = reverse @list;
2913
2914     my $save_clean = $noclean;
2915     my %ignored_warnings;
2916
2917     if (defined($ignore_warnings)) {
2918         foreach my $sha1 (split /\s+/, $ignore_warnings) {
2919             $ignored_warnings{$sha1} = 1;
2920         }
2921     }
2922
2923     $in_patchcheck = 1;
2924     foreach my $item (@list) {
2925         my $sha1 = $item;
2926         $sha1 =~ s/^([[:xdigit:]]+).*/$1/;
2927
2928         doprint "\nProcessing commit $item\n\n";
2929
2930         run_command "git checkout $sha1" or
2931             die "Failed to checkout $sha1";
2932
2933         # only clean on the first and last patch
2934         if ($item eq $list[0] ||
2935             $item eq $list[$#list]) {
2936             $noclean = $save_clean;
2937         } else {
2938             $noclean = 1;
2939         }
2940
2941         if (defined($minconfig)) {
2942             build "useconfig:$minconfig" or return 0;
2943         } else {
2944             # ?? no config to use?
2945             build "oldconfig" or return 0;
2946         }
2947
2948
2949         if (!defined($ignored_warnings{$sha1})) {
2950             check_buildlog $sha1 or return 0;
2951         }
2952
2953         next if ($type eq "build");
2954
2955         my $failed = 0;
2956
2957         start_monitor_and_boot or $failed = 1;
2958
2959         if (!$failed && $type ne "boot"){
2960             do_run_test or $failed = 1;
2961         }
2962         end_monitor;
2963         return 0 if ($failed);
2964
2965         patchcheck_reboot;
2966
2967     }
2968     $in_patchcheck = 0;
2969     success $i;
2970
2971     return 1;
2972 }
2973
2974 my %depends;
2975 my %depcount;
2976 my $iflevel = 0;
2977 my @ifdeps;
2978
2979 # prevent recursion
2980 my %read_kconfigs;
2981
2982 sub add_dep {
2983     # $config depends on $dep
2984     my ($config, $dep) = @_;
2985
2986     if (defined($depends{$config})) {
2987         $depends{$config} .= " " . $dep;
2988     } else {
2989         $depends{$config} = $dep;
2990     }
2991
2992     # record the number of configs depending on $dep
2993     if (defined $depcount{$dep}) {
2994         $depcount{$dep}++;
2995     } else {
2996         $depcount{$dep} = 1;
2997     }
2998 }
2999
3000 # taken from streamline_config.pl
3001 sub read_kconfig {
3002     my ($kconfig) = @_;
3003
3004     my $state = "NONE";
3005     my $config;
3006     my @kconfigs;
3007
3008     my $cont = 0;
3009     my $line;
3010
3011
3012     if (! -f $kconfig) {
3013         doprint "file $kconfig does not exist, skipping\n";
3014         return;
3015     }
3016
3017     open(KIN, "$kconfig")
3018         or die "Can't open $kconfig";
3019     while (<KIN>) {
3020         chomp;
3021
3022         # Make sure that lines ending with \ continue
3023         if ($cont) {
3024             $_ = $line . " " . $_;
3025         }
3026
3027         if (s/\\$//) {
3028             $cont = 1;
3029             $line = $_;
3030             next;
3031         }
3032
3033         $cont = 0;
3034
3035         # collect any Kconfig sources
3036         if (/^source\s*"(.*)"/) {
3037             $kconfigs[$#kconfigs+1] = $1;
3038         }
3039
3040         # configs found
3041         if (/^\s*(menu)?config\s+(\S+)\s*$/) {
3042             $state = "NEW";
3043             $config = $2;
3044
3045             for (my $i = 0; $i < $iflevel; $i++) {
3046                 add_dep $config, $ifdeps[$i];
3047             }
3048
3049         # collect the depends for the config
3050         } elsif ($state eq "NEW" && /^\s*depends\s+on\s+(.*)$/) {
3051
3052             add_dep $config, $1;
3053
3054         # Get the configs that select this config
3055         } elsif ($state eq "NEW" && /^\s*select\s+(\S+)/) {
3056
3057             # selected by depends on config
3058             add_dep $1, $config;
3059
3060         # Check for if statements
3061         } elsif (/^if\s+(.*\S)\s*$/) {
3062             my $deps = $1;
3063             # remove beginning and ending non text
3064             $deps =~ s/^[^a-zA-Z0-9_]*//;
3065             $deps =~ s/[^a-zA-Z0-9_]*$//;
3066
3067             my @deps = split /[^a-zA-Z0-9_]+/, $deps;
3068
3069             $ifdeps[$iflevel++] = join ':', @deps;
3070
3071         } elsif (/^endif/) {
3072
3073             $iflevel-- if ($iflevel);
3074
3075         # stop on "help"
3076         } elsif (/^\s*help\s*$/) {
3077             $state = "NONE";
3078         }
3079     }
3080     close(KIN);
3081
3082     # read in any configs that were found.
3083     foreach $kconfig (@kconfigs) {
3084         if (!defined($read_kconfigs{$kconfig})) {
3085             $read_kconfigs{$kconfig} = 1;
3086             read_kconfig("$builddir/$kconfig");
3087         }
3088     }
3089 }
3090
3091 sub read_depends {
3092     # find out which arch this is by the kconfig file
3093     open (IN, $output_config)
3094         or dodie "Failed to read $output_config";
3095     my $arch;
3096     while (<IN>) {
3097         if (m,Linux/(\S+)\s+\S+\s+Kernel Configuration,) {
3098             $arch = $1;
3099             last;
3100         }
3101     }
3102     close IN;
3103
3104     if (!defined($arch)) {
3105         doprint "Could not find arch from config file\n";
3106         doprint "no dependencies used\n";
3107         return;
3108     }
3109
3110     # arch is really the subarch, we need to know
3111     # what directory to look at.
3112     if ($arch eq "i386" || $arch eq "x86_64") {
3113         $arch = "x86";
3114     } elsif ($arch =~ /^tile/) {
3115         $arch = "tile";
3116     }
3117
3118     my $kconfig = "$builddir/arch/$arch/Kconfig";
3119
3120     if (! -f $kconfig && $arch =~ /\d$/) {
3121         my $orig = $arch;
3122         # some subarchs have numbers, truncate them
3123         $arch =~ s/\d*$//;
3124         $kconfig = "$builddir/arch/$arch/Kconfig";
3125         if (! -f $kconfig) {
3126             doprint "No idea what arch dir $orig is for\n";
3127             doprint "no dependencies used\n";
3128             return;
3129         }
3130     }
3131
3132     read_kconfig($kconfig);
3133 }
3134
3135 sub read_config_list {
3136     my ($config) = @_;
3137
3138     open (IN, $config)
3139         or dodie "Failed to read $config";
3140
3141     while (<IN>) {
3142         if (/^((CONFIG\S*)=.*)/) {
3143             if (!defined($config_ignore{$2})) {
3144                 $config_list{$2} = $1;
3145             }
3146         }
3147     }
3148
3149     close(IN);
3150 }
3151
3152 sub read_output_config {
3153     my ($config) = @_;
3154
3155     assign_configs \%config_ignore, $config;
3156 }
3157
3158 sub make_new_config {
3159     my @configs = @_;
3160
3161     open (OUT, ">$output_config")
3162         or dodie "Failed to write $output_config";
3163
3164     foreach my $config (@configs) {
3165         print OUT "$config\n";
3166     }
3167     close OUT;
3168 }
3169
3170 sub chomp_config {
3171     my ($config) = @_;
3172
3173     $config =~ s/CONFIG_//;
3174
3175     return $config;
3176 }
3177
3178 sub get_depends {
3179     my ($dep) = @_;
3180
3181     my $kconfig = chomp_config $dep;
3182
3183     $dep = $depends{"$kconfig"};
3184
3185     # the dep string we have saves the dependencies as they
3186     # were found, including expressions like ! && ||. We
3187     # want to split this out into just an array of configs.
3188
3189     my $valid = "A-Za-z_0-9";
3190
3191     my @configs;
3192
3193     while ($dep =~ /[$valid]/) {
3194
3195         if ($dep =~ /^[^$valid]*([$valid]+)/) {
3196             my $conf = "CONFIG_" . $1;
3197
3198             $configs[$#configs + 1] = $conf;
3199
3200             $dep =~ s/^[^$valid]*[$valid]+//;
3201         } else {
3202             die "this should never happen";
3203         }
3204     }
3205
3206     return @configs;
3207 }
3208
3209 my %min_configs;
3210 my %keep_configs;
3211 my %save_configs;
3212 my %processed_configs;
3213 my %nochange_config;
3214
3215 sub test_this_config {
3216     my ($config) = @_;
3217
3218     my $found;
3219
3220     # if we already processed this config, skip it
3221     if (defined($processed_configs{$config})) {
3222         return undef;
3223     }
3224     $processed_configs{$config} = 1;
3225
3226     # if this config failed during this round, skip it
3227     if (defined($nochange_config{$config})) {
3228         return undef;
3229     }
3230
3231     my $kconfig = chomp_config $config;
3232
3233     # Test dependencies first
3234     if (defined($depends{"$kconfig"})) {
3235         my @parents = get_depends $config;
3236         foreach my $parent (@parents) {
3237             # if the parent is in the min config, check it first
3238             next if (!defined($min_configs{$parent}));
3239             $found = test_this_config($parent);
3240             if (defined($found)) {
3241                 return $found;
3242             }
3243         }
3244     }
3245
3246     # Remove this config from the list of configs
3247     # do a make oldnoconfig and then read the resulting
3248     # .config to make sure it is missing the config that
3249     # we had before
3250     my %configs = %min_configs;
3251     delete $configs{$config};
3252     make_new_config ((values %configs), (values %keep_configs));
3253     make_oldconfig;
3254     undef %configs;
3255     assign_configs \%configs, $output_config;
3256
3257     return $config if (!defined($configs{$config}));
3258
3259     doprint "disabling config $config did not change .config\n";
3260
3261     $nochange_config{$config} = 1;
3262
3263     return undef;
3264 }
3265
3266 sub make_min_config {
3267     my ($i) = @_;
3268
3269     my $type = $minconfig_type;
3270     if ($type ne "boot" && $type ne "test") {
3271         fail "Invalid MIN_CONFIG_TYPE '$minconfig_type'\n" .
3272             " make_min_config works only with 'boot' and 'test'\n" and return;
3273     }
3274
3275     if (!defined($output_minconfig)) {
3276         fail "OUTPUT_MIN_CONFIG not defined" and return;
3277     }
3278
3279     # If output_minconfig exists, and the start_minconfig
3280     # came from min_config, than ask if we should use
3281     # that instead.
3282     if (-f $output_minconfig && !$start_minconfig_defined) {
3283         print "$output_minconfig exists\n";
3284         if (!defined($use_output_minconfig)) {
3285             if (read_yn " Use it as minconfig?") {
3286                 $start_minconfig = $output_minconfig;
3287             }
3288         } elsif ($use_output_minconfig > 0) {
3289             doprint "Using $output_minconfig as MIN_CONFIG\n";
3290             $start_minconfig = $output_minconfig;
3291         } else {
3292             doprint "Set to still use MIN_CONFIG as starting point\n";
3293         }
3294     }
3295
3296     if (!defined($start_minconfig)) {
3297         fail "START_MIN_CONFIG or MIN_CONFIG not defined" and return;
3298     }
3299
3300     my $temp_config = "$tmpdir/temp_config";
3301
3302     # First things first. We build an allnoconfig to find
3303     # out what the defaults are that we can't touch.
3304     # Some are selections, but we really can't handle selections.
3305
3306     my $save_minconfig = $minconfig;
3307     undef $minconfig;
3308
3309     run_command "$make allnoconfig" or return 0;
3310
3311     read_depends;
3312
3313     process_config_ignore $output_config;
3314
3315     undef %save_configs;
3316     undef %min_configs;
3317
3318     if (defined($ignore_config)) {
3319         # make sure the file exists
3320         `touch $ignore_config`;
3321         assign_configs \%save_configs, $ignore_config;
3322     }
3323
3324     %keep_configs = %save_configs;
3325
3326     doprint "Load initial configs from $start_minconfig\n";
3327
3328     # Look at the current min configs, and save off all the
3329     # ones that were set via the allnoconfig
3330     assign_configs \%min_configs, $start_minconfig;
3331
3332     my @config_keys = keys %min_configs;
3333
3334     # All configs need a depcount
3335     foreach my $config (@config_keys) {
3336         my $kconfig = chomp_config $config;
3337         if (!defined $depcount{$kconfig}) {
3338                 $depcount{$kconfig} = 0;
3339         }
3340     }
3341
3342     # Remove anything that was set by the make allnoconfig
3343     # we shouldn't need them as they get set for us anyway.
3344     foreach my $config (@config_keys) {
3345         # Remove anything in the ignore_config
3346         if (defined($keep_configs{$config})) {
3347             my $file = $ignore_config;
3348             $file =~ s,.*/(.*?)$,$1,;
3349             doprint "$config set by $file ... ignored\n";
3350             delete $min_configs{$config};
3351             next;
3352         }
3353         # But make sure the settings are the same. If a min config
3354         # sets a selection, we do not want to get rid of it if
3355         # it is not the same as what we have. Just move it into
3356         # the keep configs.
3357         if (defined($config_ignore{$config})) {
3358             if ($config_ignore{$config} ne $min_configs{$config}) {
3359                 doprint "$config is in allnoconfig as '$config_ignore{$config}'";
3360                 doprint " but it is '$min_configs{$config}' in minconfig .. keeping\n";
3361                 $keep_configs{$config} = $min_configs{$config};
3362             } else {
3363                 doprint "$config set by allnoconfig ... ignored\n";
3364             }
3365             delete $min_configs{$config};
3366         }
3367     }
3368
3369     my $done = 0;
3370     my $take_two = 0;
3371
3372     while (!$done) {
3373
3374         my $config;
3375         my $found;
3376
3377         # Now disable each config one by one and do a make oldconfig
3378         # till we find a config that changes our list.
3379
3380         my @test_configs = keys %min_configs;
3381
3382         # Sort keys by who is most dependent on
3383         @test_configs = sort  { $depcount{chomp_config($b)} <=> $depcount{chomp_config($a)} }
3384                           @test_configs ;
3385
3386         # Put configs that did not modify the config at the end.
3387         my $reset = 1;
3388         for (my $i = 0; $i < $#test_configs; $i++) {
3389             if (!defined($nochange_config{$test_configs[0]})) {
3390                 $reset = 0;
3391                 last;
3392             }
3393             # This config didn't change the .config last time.
3394             # Place it at the end
3395             my $config = shift @test_configs;
3396             push @test_configs, $config;
3397         }
3398
3399         # if every test config has failed to modify the .config file
3400         # in the past, then reset and start over.
3401         if ($reset) {
3402             undef %nochange_config;
3403         }
3404
3405         undef %processed_configs;
3406
3407         foreach my $config (@test_configs) {
3408
3409             $found = test_this_config $config;
3410
3411             last if (defined($found));
3412
3413             # oh well, try another config
3414         }
3415
3416         if (!defined($found)) {
3417             # we could have failed due to the nochange_config hash
3418             # reset and try again
3419             if (!$take_two) {
3420                 undef %nochange_config;
3421                 $take_two = 1;
3422                 next;
3423             }
3424             doprint "No more configs found that we can disable\n";
3425             $done = 1;
3426             last;
3427         }
3428         $take_two = 0;
3429
3430         $config = $found;
3431
3432         doprint "Test with $config disabled\n";
3433
3434         # set in_bisect to keep build and monitor from dieing
3435         $in_bisect = 1;
3436
3437         my $failed = 0;
3438         build "oldconfig" or $failed = 1;
3439         if (!$failed) {
3440                 start_monitor_and_boot or $failed = 1;
3441
3442                 if ($type eq "test" && !$failed) {
3443                     do_run_test or $failed = 1;
3444                 }
3445
3446                 end_monitor;
3447         }
3448
3449         $in_bisect = 0;
3450
3451         if ($failed) {
3452             doprint "$min_configs{$config} is needed to boot the box... keeping\n";
3453             # this config is needed, add it to the ignore list.
3454             $keep_configs{$config} = $min_configs{$config};
3455             $save_configs{$config} = $min_configs{$config};
3456             delete $min_configs{$config};
3457
3458             # update new ignore configs
3459             if (defined($ignore_config)) {
3460                 open (OUT, ">$temp_config")
3461                     or die "Can't write to $temp_config";
3462                 foreach my $config (keys %save_configs) {
3463                     print OUT "$save_configs{$config}\n";
3464                 }
3465                 close OUT;
3466                 run_command "mv $temp_config $ignore_config" or
3467                     dodie "failed to copy update to $ignore_config";
3468             }
3469
3470         } else {
3471             # We booted without this config, remove it from the minconfigs.
3472             doprint "$config is not needed, disabling\n";
3473
3474             delete $min_configs{$config};
3475
3476             # Also disable anything that is not enabled in this config
3477             my %configs;
3478             assign_configs \%configs, $output_config;
3479             my @config_keys = keys %min_configs;
3480             foreach my $config (@config_keys) {
3481                 if (!defined($configs{$config})) {
3482                     doprint "$config is not set, disabling\n";
3483                     delete $min_configs{$config};
3484                 }
3485             }
3486
3487             # Save off all the current mandidory configs
3488             open (OUT, ">$temp_config")
3489                 or die "Can't write to $temp_config";
3490             foreach my $config (keys %keep_configs) {
3491                 print OUT "$keep_configs{$config}\n";
3492             }
3493             foreach my $config (keys %min_configs) {
3494                 print OUT "$min_configs{$config}\n";
3495             }
3496             close OUT;
3497
3498             run_command "mv $temp_config $output_minconfig" or
3499                 dodie "failed to copy update to $output_minconfig";
3500         }
3501
3502         doprint "Reboot and wait $sleep_time seconds\n";
3503         reboot_to_good $sleep_time;
3504     }
3505
3506     success $i;
3507     return 1;
3508 }
3509
3510 $#ARGV < 1 or die "ktest.pl version: $VERSION\n   usage: ktest.pl config-file\n";
3511
3512 if ($#ARGV == 0) {
3513     $ktest_config = $ARGV[0];
3514     if (! -f $ktest_config) {
3515         print "$ktest_config does not exist.\n";
3516         if (!read_yn "Create it?") {
3517             exit 0;
3518         }
3519     }
3520 } else {
3521     $ktest_config = "ktest.conf";
3522 }
3523
3524 if (! -f $ktest_config) {
3525     $newconfig = 1;
3526     get_test_case;
3527     open(OUT, ">$ktest_config") or die "Can not create $ktest_config";
3528     print OUT << "EOF"
3529 # Generated by ktest.pl
3530 #
3531
3532 # PWD is a ktest.pl variable that will result in the process working
3533 # directory that ktest.pl is executed in.
3534
3535 # THIS_DIR is automatically assigned the PWD of the path that generated
3536 # the config file. It is best to use this variable when assigning other
3537 # directory paths within this directory. This allows you to easily
3538 # move the test cases to other locations or to other machines.
3539 #
3540 THIS_DIR := $variable{"PWD"}
3541
3542 # Define each test with TEST_START
3543 # The config options below it will override the defaults
3544 TEST_START
3545 TEST_TYPE = $default{"TEST_TYPE"}
3546
3547 DEFAULTS
3548 EOF
3549 ;
3550     close(OUT);
3551 }
3552 read_config $ktest_config;
3553
3554 if (defined($opt{"LOG_FILE"})) {
3555     $opt{"LOG_FILE"} = eval_option($opt{"LOG_FILE"}, -1);
3556 }
3557
3558 # Append any configs entered in manually to the config file.
3559 my @new_configs = keys %entered_configs;
3560 if ($#new_configs >= 0) {
3561     print "\nAppending entered in configs to $ktest_config\n";
3562     open(OUT, ">>$ktest_config") or die "Can not append to $ktest_config";
3563     foreach my $config (@new_configs) {
3564         print OUT "$config = $entered_configs{$config}\n";
3565         $opt{$config} = process_variables($entered_configs{$config});
3566     }
3567 }
3568
3569 if ($opt{"CLEAR_LOG"} && defined($opt{"LOG_FILE"})) {
3570     unlink $opt{"LOG_FILE"};
3571 }
3572
3573 doprint "\n\nSTARTING AUTOMATED TESTS\n\n";
3574
3575 for (my $i = 0, my $repeat = 1; $i <= $opt{"NUM_TESTS"}; $i += $repeat) {
3576
3577     if (!$i) {
3578         doprint "DEFAULT OPTIONS:\n";
3579     } else {
3580         doprint "\nTEST $i OPTIONS";
3581         if (defined($repeat_tests{$i})) {
3582             $repeat = $repeat_tests{$i};
3583             doprint " ITERATE $repeat";
3584         }
3585         doprint "\n";
3586     }
3587
3588     foreach my $option (sort keys %opt) {
3589
3590         if ($option =~ /\[(\d+)\]$/) {
3591             next if ($i != $1);
3592         } else {
3593             next if ($i);
3594         }
3595
3596         doprint "$option = $opt{$option}\n";
3597     }
3598 }
3599
3600 sub __set_test_option {
3601     my ($name, $i) = @_;
3602
3603     my $option = "$name\[$i\]";
3604
3605     if (defined($opt{$option})) {
3606         return $opt{$option};
3607     }
3608
3609     foreach my $test (keys %repeat_tests) {
3610         if ($i >= $test &&
3611             $i < $test + $repeat_tests{$test}) {
3612             $option = "$name\[$test\]";
3613             if (defined($opt{$option})) {
3614                 return $opt{$option};
3615             }
3616         }
3617     }
3618
3619     if (defined($opt{$name})) {
3620         return $opt{$name};
3621     }
3622
3623     return undef;
3624 }
3625
3626 sub set_test_option {
3627     my ($name, $i) = @_;
3628
3629     my $option = __set_test_option($name, $i);
3630     return $option if (!defined($option));
3631
3632     return eval_option($option, $i);
3633 }
3634
3635 # First we need to do is the builds
3636 for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
3637
3638     # Do not reboot on failing test options
3639     $no_reboot = 1;
3640     $reboot_success = 0;
3641
3642     $have_version = 0;
3643
3644     $iteration = $i;
3645
3646     undef %force_config;
3647
3648     my $makecmd = set_test_option("MAKE_CMD", $i);
3649
3650     # Load all the options into their mapped variable names
3651     foreach my $opt (keys %option_map) {
3652         ${$option_map{$opt}} = set_test_option($opt, $i);
3653     }
3654
3655     $start_minconfig_defined = 1;
3656
3657     # The first test may override the PRE_KTEST option
3658     if (defined($pre_ktest) && $i == 1) {
3659         doprint "\n";
3660         run_command $pre_ktest;
3661     }
3662
3663     # Any test can override the POST_KTEST option
3664     # The last test takes precedence.
3665     if (defined($post_ktest)) {
3666         $final_post_ktest = $post_ktest;
3667     }
3668
3669     if (!defined($start_minconfig)) {
3670         $start_minconfig_defined = 0;
3671         $start_minconfig = $minconfig;
3672     }
3673
3674     chdir $builddir || die "can't change directory to $builddir";
3675
3676     foreach my $dir ($tmpdir, $outputdir) {
3677         if (!-d $dir) {
3678             mkpath($dir) or
3679                 die "can't create $dir";
3680         }
3681     }
3682
3683     $ENV{"SSH_USER"} = $ssh_user;
3684     $ENV{"MACHINE"} = $machine;
3685
3686     $buildlog = "$tmpdir/buildlog-$machine";
3687     $testlog = "$tmpdir/testlog-$machine";
3688     $dmesg = "$tmpdir/dmesg-$machine";
3689     $make = "$makecmd O=$outputdir";
3690     $output_config = "$outputdir/.config";
3691
3692     if (!$buildonly) {
3693         $target = "$ssh_user\@$machine";
3694         if ($reboot_type eq "grub") {
3695             dodie "GRUB_MENU not defined" if (!defined($grub_menu));
3696         }
3697     }
3698
3699     my $run_type = $build_type;
3700     if ($test_type eq "patchcheck") {
3701         $run_type = $patchcheck_type;
3702     } elsif ($test_type eq "bisect") {
3703         $run_type = $bisect_type;
3704     } elsif ($test_type eq "config_bisect") {
3705         $run_type = $config_bisect_type;
3706     }
3707
3708     if ($test_type eq "make_min_config") {
3709         $run_type = "";
3710     }
3711
3712     # mistake in config file?
3713     if (!defined($run_type)) {
3714         $run_type = "ERROR";
3715     }
3716
3717     my $installme = "";
3718     $installme = " no_install" if ($no_install);
3719
3720     doprint "\n\n";
3721     doprint "RUNNING TEST $i of $opt{NUM_TESTS} with option $test_type $run_type$installme\n\n";
3722
3723     if (defined($pre_test)) {
3724         run_command $pre_test;
3725     }
3726
3727     unlink $dmesg;
3728     unlink $buildlog;
3729     unlink $testlog;
3730
3731     if (defined($addconfig)) {
3732         my $min = $minconfig;
3733         if (!defined($minconfig)) {
3734             $min = "";
3735         }
3736         run_command "cat $addconfig $min > $tmpdir/add_config" or
3737             dodie "Failed to create temp config";
3738         $minconfig = "$tmpdir/add_config";
3739     }
3740
3741     if (defined($checkout)) {
3742         run_command "git checkout $checkout" or
3743             die "failed to checkout $checkout";
3744     }
3745
3746     $no_reboot = 0;
3747
3748     # A test may opt to not reboot the box
3749     if ($reboot_on_success) {
3750         $reboot_success = 1;
3751     }
3752
3753     if ($test_type eq "bisect") {
3754         bisect $i;
3755         next;
3756     } elsif ($test_type eq "config_bisect") {
3757         config_bisect $i;
3758         next;
3759     } elsif ($test_type eq "patchcheck") {
3760         patchcheck $i;
3761         next;
3762     } elsif ($test_type eq "make_min_config") {
3763         make_min_config $i;
3764         next;
3765     }
3766
3767     if ($build_type ne "nobuild") {
3768         build $build_type or next;
3769     }
3770
3771     if ($test_type eq "install") {
3772         get_version;
3773         install;
3774         success $i;
3775         next;
3776     }
3777
3778     if ($test_type ne "build") {
3779         my $failed = 0;
3780         start_monitor_and_boot or $failed = 1;
3781
3782         if (!$failed && $test_type ne "boot" && defined($run_test)) {
3783             do_run_test or $failed = 1;
3784         }
3785         end_monitor;
3786         next if ($failed);
3787     }
3788
3789     success $i;
3790 }
3791
3792 if (defined($final_post_ktest)) {
3793     run_command $final_post_ktest;
3794 }
3795
3796 if ($opt{"POWEROFF_ON_SUCCESS"}) {
3797     halt;
3798 } elsif ($opt{"REBOOT_ON_SUCCESS"} && !do_not_reboot && $reboot_success) {
3799     reboot_to_good;
3800 } elsif (defined($switch_to_good)) {
3801     # still need to get to the good kernel
3802     run_command $switch_to_good;
3803 }
3804
3805
3806 doprint "\n    $successes of $opt{NUM_TESTS} tests were successful\n\n";
3807
3808 exit 0;