tools/testing/ktest/ktest.pl

   1 #!/usr/bin/perl -w
   2 #
   3 # Copyright 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
   4 # Licensed under the terms of the GNU GPL License version 2
   5 #
   6
   7 use strict;
   8 use IPC::Open2;
   9 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
  10 use File::Path qw(mkpath);
  11 use File::Copy qw(cp);
  12 use FileHandle;
  13
  14 my $VERSION = "0.2";
  15
  16 $| = 1;
  17
  18 my %opt;
  19 my %repeat_tests;
  20 my %repeats;
  21
  22 #default opts
  23 my %default = (
  24     "NUM_TESTS"                 => 1,
  25     "TEST_TYPE"                 => "build",
  26     "BUILD_TYPE"                => "randconfig",
  27     "MAKE_CMD"                  => "make",
  28     "TIMEOUT"                   => 120,
  29     "TMP_DIR"                   => "/tmp/ktest/\${MACHINE}",
  30     "SLEEP_TIME"                => 60,  # sleep time between tests
  31     "BUILD_NOCLEAN"             => 0,
  32     "REBOOT_ON_ERROR"           => 0,
  33     "POWEROFF_ON_ERROR"         => 0,
  34     "REBOOT_ON_SUCCESS"         => 1,
  35     "POWEROFF_ON_SUCCESS"       => 0,
  36     "BUILD_OPTIONS"             => "",
  37     "BISECT_SLEEP_TIME"         => 60,   # sleep time between bisects
  38     "PATCHCHECK_SLEEP_TIME"     => 60, # sleep time between patch checks
  39     "CLEAR_LOG"                 => 0,
  40     "BISECT_MANUAL"             => 0,
  41     "BISECT_SKIP"               => 1,
  42     "MIN_CONFIG_TYPE"           => "boot",
  43     "SUCCESS_LINE"              => "login:",
  44     "DETECT_TRIPLE_FAULT"       => 1,
  45     "NO_INSTALL"                => 0,
  46     "BOOTED_TIMEOUT"            => 1,
  47     "DIE_ON_FAILURE"            => 1,
  48     "SSH_EXEC"                  => "ssh \$SSH_USER\@\$MACHINE \$SSH_COMMAND",
  49     "SCP_TO_TARGET"             => "scp \$SRC_FILE \$SSH_USER\@\$MACHINE:\$DST_FILE",
  50     "SCP_TO_TARGET_INSTALL"     => "\${SCP_TO_TARGET}",
  51     "REBOOT"                    => "ssh \$SSH_USER\@\$MACHINE reboot",
  52     "STOP_AFTER_SUCCESS"        => 10,
  53     "STOP_AFTER_FAILURE"        => 60,
  54     "STOP_TEST_AFTER"           => 600,
  55     "MAX_MONITOR_WAIT"          => 1800,
  56
  57 # required, and we will ask users if they don't have them but we keep the default
  58 # value something that is common.
  59     "REBOOT_TYPE"               => "grub",
  60     "LOCALVERSION"              => "-test",
  61     "SSH_USER"                  => "root",
  62     "BUILD_TARGET"              => "arch/x86/boot/bzImage",
  63     "TARGET_IMAGE"              => "/boot/vmlinuz-test",
  64
  65     "LOG_FILE"                  => undef,
  66     "IGNORE_UNUSED"             => 0,
  67 );
  68
  69 my $ktest_config;
  70 my $version;
  71 my $have_version = 0;
  72 my $machine;
  73 my $ssh_user;
  74 my $tmpdir;
  75 my $builddir;
  76 my $outputdir;
  77 my $output_config;
  78 my $test_type;
  79 my $build_type;
  80 my $build_options;
  81 my $final_post_ktest;
  82 my $pre_ktest;
  83 my $post_ktest;
  84 my $pre_test;
  85 my $post_test;
  86 my $pre_build;
  87 my $post_build;
  88 my $pre_build_die;
  89 my $post_build_die;
  90 my $reboot_type;
  91 my $reboot_script;
  92 my $power_cycle;
  93 my $reboot;
  94 my $reboot_on_error;
  95 my $switch_to_good;
  96 my $switch_to_test;
  97 my $poweroff_on_error;
  98 my $reboot_on_success;
  99 my $die_on_failure;
 100 my $powercycle_after_reboot;
 101 my $poweroff_after_halt;
 102 my $max_monitor_wait;
 103 my $ssh_exec;
 104 my $scp_to_target;
 105 my $scp_to_target_install;
 106 my $power_off;
 107 my $grub_menu;
 108 my $grub_number;
 109 my $target;
 110 my $make;
 111 my $pre_install;
 112 my $post_install;
 113 my $no_install;
 114 my $noclean;
 115 my $minconfig;
 116 my $start_minconfig;
 117 my $start_minconfig_defined;
 118 my $output_minconfig;
 119 my $minconfig_type;
 120 my $use_output_minconfig;
 121 my $ignore_config;
 122 my $ignore_errors;
 123 my $addconfig;
 124 my $in_bisect = 0;
 125 my $bisect_bad_commit = "";
 126 my $reverse_bisect;
 127 my $bisect_manual;
 128 my $bisect_skip;
 129 my $config_bisect_good;
 130 my $bisect_ret_good;
 131 my $bisect_ret_bad;
 132 my $bisect_ret_skip;
 133 my $bisect_ret_abort;
 134 my $bisect_ret_default;
 135 my $in_patchcheck = 0;
 136 my $run_test;
 137 my $redirect;
 138 my $buildlog;
 139 my $testlog;
 140 my $dmesg;
 141 my $monitor_fp;
 142 my $monitor_pid;
 143 my $monitor_cnt = 0;
 144 my $sleep_time;
 145 my $bisect_sleep_time;
 146 my $patchcheck_sleep_time;
 147 my $ignore_warnings;
 148 my $store_failures;
 149 my $store_successes;
 150 my $test_name;
 151 my $timeout;
 152 my $booted_timeout;
 153 my $detect_triplefault;
 154 my $console;
 155 my $reboot_success_line;
 156 my $success_line;
 157 my $stop_after_success;
 158 my $stop_after_failure;
 159 my $stop_test_after;
 160 my $build_target;
 161 my $target_image;
 162 my $checkout;
 163 my $localversion;
 164 my $iteration = 0;
 165 my $successes = 0;
 166
 167 my $bisect_good;
 168 my $bisect_bad;
 169 my $bisect_type;
 170 my $bisect_start;
 171 my $bisect_replay;
 172 my $bisect_files;
 173 my $bisect_reverse;
 174 my $bisect_check;
 175
 176 my $config_bisect;
 177 my $config_bisect_type;
 178 my $config_bisect_check;
 179
 180 my $patchcheck_type;
 181 my $patchcheck_start;
 182 my $patchcheck_end;
 183
 184 # set when a test is something other that just building or install
 185 # which would require more options.
 186 my $buildonly = 1;
 187
 188 # set when creating a new config
 189 my $newconfig = 0;
 190
 191 my %entered_configs;
 192 my %config_help;
 193 my %variable;
 194
 195 # force_config is the list of configs that we force enabled (or disabled)
 196 # in a .config file. The MIN_CONFIG and ADD_CONFIG configs.
 197 my %force_config;
 198
 199 # do not force reboots on config problems
 200 my $no_reboot = 1;
 201
 202 # reboot on success
 203 my $reboot_success = 0;
 204
 205 my %option_map = (
 206     "MACHINE"                   => \$machine,
 207     "SSH_USER"                  => \$ssh_user,
 208     "TMP_DIR"                   => \$tmpdir,
 209     "OUTPUT_DIR"                => \$outputdir,
 210     "BUILD_DIR"                 => \$builddir,
 211     "TEST_TYPE"                 => \$test_type,
 212     "PRE_KTEST"                 => \$pre_ktest,
 213     "POST_KTEST"                => \$post_ktest,
 214     "PRE_TEST"                  => \$pre_test,
 215     "POST_TEST"                 => \$post_test,
 216     "BUILD_TYPE"                => \$build_type,
 217     "BUILD_OPTIONS"             => \$build_options,
 218     "PRE_BUILD"                 => \$pre_build,
 219     "POST_BUILD"                => \$post_build,
 220     "PRE_BUILD_DIE"             => \$pre_build_die,
 221     "POST_BUILD_DIE"            => \$post_build_die,
 222     "POWER_CYCLE"               => \$power_cycle,
 223     "REBOOT"                    => \$reboot,
 224     "BUILD_NOCLEAN"             => \$noclean,
 225     "MIN_CONFIG"                => \$minconfig,
 226     "OUTPUT_MIN_CONFIG"         => \$output_minconfig,
 227     "START_MIN_CONFIG"          => \$start_minconfig,
 228     "MIN_CONFIG_TYPE"           => \$minconfig_type,
 229     "USE_OUTPUT_MIN_CONFIG"     => \$use_output_minconfig,
 230     "IGNORE_CONFIG"             => \$ignore_config,
 231     "TEST"                      => \$run_test,
 232     "ADD_CONFIG"                => \$addconfig,
 233     "REBOOT_TYPE"               => \$reboot_type,
 234     "GRUB_MENU"                 => \$grub_menu,
 235     "PRE_INSTALL"               => \$pre_install,
 236     "POST_INSTALL"              => \$post_install,
 237     "NO_INSTALL"                => \$no_install,
 238     "REBOOT_SCRIPT"             => \$reboot_script,
 239     "REBOOT_ON_ERROR"           => \$reboot_on_error,
 240     "SWITCH_TO_GOOD"            => \$switch_to_good,
 241     "SWITCH_TO_TEST"            => \$switch_to_test,
 242     "POWEROFF_ON_ERROR"         => \$poweroff_on_error,
 243     "REBOOT_ON_SUCCESS"         => \$reboot_on_success,
 244     "DIE_ON_FAILURE"            => \$die_on_failure,
 245     "POWER_OFF"                 => \$power_off,
 246     "POWERCYCLE_AFTER_REBOOT"   => \$powercycle_after_reboot,
 247     "POWEROFF_AFTER_HALT"       => \$poweroff_after_halt,
 248     "MAX_MONITOR_WAIT"          => \$max_monitor_wait,
 249     "SLEEP_TIME"                => \$sleep_time,
 250     "BISECT_SLEEP_TIME"         => \$bisect_sleep_time,
 251     "PATCHCHECK_SLEEP_TIME"     => \$patchcheck_sleep_time,
 252     "IGNORE_WARNINGS"           => \$ignore_warnings,
 253     "IGNORE_ERRORS"             => \$ignore_errors,
 254     "BISECT_MANUAL"             => \$bisect_manual,
 255     "BISECT_SKIP"               => \$bisect_skip,
 256     "CONFIG_BISECT_GOOD"        => \$config_bisect_good,
 257     "BISECT_RET_GOOD"           => \$bisect_ret_good,
 258     "BISECT_RET_BAD"            => \$bisect_ret_bad,
 259     "BISECT_RET_SKIP"           => \$bisect_ret_skip,
 260     "BISECT_RET_ABORT"          => \$bisect_ret_abort,
 261     "BISECT_RET_DEFAULT"        => \$bisect_ret_default,
 262     "STORE_FAILURES"            => \$store_failures,
 263     "STORE_SUCCESSES"           => \$store_successes,
 264     "TEST_NAME"                 => \$test_name,
 265     "TIMEOUT"                   => \$timeout,
 266     "BOOTED_TIMEOUT"            => \$booted_timeout,
 267     "CONSOLE"                   => \$console,
 268     "DETECT_TRIPLE_FAULT"       => \$detect_triplefault,
 269     "SUCCESS_LINE"              => \$success_line,
 270     "REBOOT_SUCCESS_LINE"       => \$reboot_success_line,
 271     "STOP_AFTER_SUCCESS"        => \$stop_after_success,
 272     "STOP_AFTER_FAILURE"        => \$stop_after_failure,
 273     "STOP_TEST_AFTER"           => \$stop_test_after,
 274     "BUILD_TARGET"              => \$build_target,
 275     "SSH_EXEC"                  => \$ssh_exec,
 276     "SCP_TO_TARGET"             => \$scp_to_target,
 277     "SCP_TO_TARGET_INSTALL"     => \$scp_to_target_install,
 278     "CHECKOUT"                  => \$checkout,
 279     "TARGET_IMAGE"              => \$target_image,
 280     "LOCALVERSION"              => \$localversion,
 281
 282     "BISECT_GOOD"               => \$bisect_good,
 283     "BISECT_BAD"                => \$bisect_bad,
 284     "BISECT_TYPE"               => \$bisect_type,
 285     "BISECT_START"              => \$bisect_start,
 286     "BISECT_REPLAY"             => \$bisect_replay,
 287     "BISECT_FILES"              => \$bisect_files,
 288     "BISECT_REVERSE"            => \$bisect_reverse,
 289     "BISECT_CHECK"              => \$bisect_check,
 290
 291     "CONFIG_BISECT"             => \$config_bisect,
 292     "CONFIG_BISECT_TYPE"        => \$config_bisect_type,
 293     "CONFIG_BISECT_CHECK"       => \$config_bisect_check,
 294
 295     "PATCHCHECK_TYPE"           => \$patchcheck_type,
 296     "PATCHCHECK_START"          => \$patchcheck_start,
 297     "PATCHCHECK_END"            => \$patchcheck_end,
 298 );
 299
 300 # Options may be used by other options, record them.
 301 my %used_options;
 302
 303 # default variables that can be used
 304 chomp ($variable{"PWD"} = `pwd`);
 305
 306 $config_help{"MACHINE"} = << "EOF"
 307  The machine hostname that you will test.
 308  For build only tests, it is still needed to differentiate log files.
 309 EOF
 310     ;
 311 $config_help{"SSH_USER"} = << "EOF"
 312  The box is expected to have ssh on normal bootup, provide the user
 313   (most likely root, since you need privileged operations)
 314 EOF
 315     ;
 316 $config_help{"BUILD_DIR"} = << "EOF"
 317  The directory that contains the Linux source code (full path).
 318  You can use \${PWD} that will be the path where ktest.pl is run, or use
 319  \${THIS_DIR} which is assigned \${PWD} but may be changed later.
 320 EOF
 321     ;
 322 $config_help{"OUTPUT_DIR"} = << "EOF"
 323  The directory that the objects will be built (full path).
 324  (can not be same as BUILD_DIR)
 325  You can use \${PWD} that will be the path where ktest.pl is run, or use
 326  \${THIS_DIR} which is assigned \${PWD} but may be changed later.
 327 EOF
 328     ;
 329 $config_help{"BUILD_TARGET"} = << "EOF"
 330  The location of the compiled file to copy to the target.
 331  (relative to OUTPUT_DIR)
 332 EOF
 333     ;
 334 $config_help{"BUILD_OPTIONS"} = << "EOF"
 335  Options to add to \"make\" when building.
 336  i.e.  -j20
 337 EOF
 338     ;
 339 $config_help{"TARGET_IMAGE"} = << "EOF"
 340  The place to put your image on the test machine.
 341 EOF
 342     ;
 343 $config_help{"POWER_CYCLE"} = << "EOF"
 344  A script or command to reboot the box.
 345
 346  Here is a digital loggers power switch example
 347  POWER_CYCLE = wget --no-proxy -O /dev/null -q  --auth-no-challenge 'http://admin:admin\@power/outlet?5=CCL'
 348
 349  Here is an example to reboot a virtual box on the current host
 350  with the name "Guest".
 351  POWER_CYCLE = virsh destroy Guest; sleep 5; virsh start Guest
 352 EOF
 353     ;
 354 $config_help{"CONSOLE"} = << "EOF"
 355  The script or command that reads the console
 356
 357   If you use ttywatch server, something like the following would work.
 358 CONSOLE = nc -d localhost 3001
 359
 360  For a virtual machine with guest name "Guest".
 361 CONSOLE =  virsh console Guest
 362 EOF
 363     ;
 364 $config_help{"LOCALVERSION"} = << "EOF"
 365  Required version ending to differentiate the test
 366  from other linux builds on the system.
 367 EOF
 368     ;
 369 $config_help{"REBOOT_TYPE"} = << "EOF"
 370  Way to reboot the box to the test kernel.
 371  Only valid options so far are "grub" and "script".
 372
 373  If you specify grub, it will assume grub version 1
 374  and will search in /boot/grub/menu.lst for the title \$GRUB_MENU
 375  and select that target to reboot to the kernel. If this is not
 376  your setup, then specify "script" and have a command or script
 377  specified in REBOOT_SCRIPT to boot to the target.
 378
 379  The entry in /boot/grub/menu.lst must be entered in manually.
 380  The test will not modify that file.
 381 EOF
 382     ;
 383 $config_help{"GRUB_MENU"} = << "EOF"
 384  The grub title name for the test kernel to boot
 385  (Only mandatory if REBOOT_TYPE = grub)
 386
 387  Note, ktest.pl will not update the grub menu.lst, you need to
 388  manually add an option for the test. ktest.pl will search
 389  the grub menu.lst for this option to find what kernel to
 390  reboot into.
 391
 392  For example, if in the /boot/grub/menu.lst the test kernel title has:
 393  title Test Kernel
 394  kernel vmlinuz-test
 395  GRUB_MENU = Test Kernel
 396 EOF
 397     ;
 398 $config_help{"REBOOT_SCRIPT"} = << "EOF"
 399  A script to reboot the target into the test kernel
 400  (Only mandatory if REBOOT_TYPE = script)
 401 EOF
 402     ;
 403
 404 sub read_prompt {
 405     my ($cancel, $prompt) = @_;
 406
 407     my $ans;
 408
 409     for (;;) {
 410         if ($cancel) {
 411             print "$prompt [y/n/C] ";
 412         } else {
 413             print "$prompt [Y/n] ";
 414         }
 415         $ans = <STDIN>;
 416         chomp $ans;
 417         if ($ans =~ /^\s*$/) {
 418             if ($cancel) {
 419                 $ans = "c";
 420             } else {
 421                 $ans = "y";
 422             }
 423         }
 424         last if ($ans =~ /^y$/i || $ans =~ /^n$/i);
 425         if ($cancel) {
 426             last if ($ans =~ /^c$/i);
 427             print "Please answer either 'y', 'n' or 'c'.\n";
 428         } else {
 429             print "Please answer either 'y' or 'n'.\n";
 430         }
 431     }
 432     if ($ans =~ /^c/i) {
 433         exit;
 434     }
 435     if ($ans !~ /^y$/i) {
 436         return 0;
 437     }
 438     return 1;
 439 }
 440
 441 sub read_yn {
 442     my ($prompt) = @_;
 443
 444     return read_prompt 0, $prompt;
 445 }
 446
 447 sub read_ync {
 448     my ($prompt) = @_;
 449
 450     return read_prompt 1, $prompt;
 451 }
 452
 453 sub get_ktest_config {
 454     my ($config) = @_;
 455     my $ans;
 456
 457     return if (defined($opt{$config}));
 458
 459     if (defined($config_help{$config})) {
 460         print "\n";
 461         print $config_help{$config};
 462     }
 463
 464     for (;;) {
 465         print "$config = ";
 466         if (defined($default{$config}) && length($default{$config})) {
 467             print "\[$default{$config}\] ";
 468         }
 469         $ans = <STDIN>;
 470         $ans =~ s/^\s*(.*\S)\s*$/$1/;
 471         if ($ans =~ /^\s*$/) {
 472             if ($default{$config}) {
 473                 $ans = $default{$config};
 474             } else {
 475                 print "Your answer can not be blank\n";
 476                 next;
 477             }
 478         }
 479         $entered_configs{$config} = ${ans};
 480         last;
 481     }
 482 }
 483
 484 sub get_ktest_configs {
 485     get_ktest_config("MACHINE");
 486     get_ktest_config("BUILD_DIR");
 487     get_ktest_config("OUTPUT_DIR");
 488
 489     if ($newconfig) {
 490         get_ktest_config("BUILD_OPTIONS");
 491     }
 492
 493     # options required for other than just building a kernel
 494     if (!$buildonly) {
 495         get_ktest_config("POWER_CYCLE");
 496         get_ktest_config("CONSOLE");
 497     }
 498
 499     # options required for install and more
 500     if ($buildonly != 1) {
 501         get_ktest_config("SSH_USER");
 502         get_ktest_config("BUILD_TARGET");
 503         get_ktest_config("TARGET_IMAGE");
 504     }
 505
 506     get_ktest_config("LOCALVERSION");
 507
 508     return if ($buildonly);
 509
 510     my $rtype = $opt{"REBOOT_TYPE"};
 511
 512     if (!defined($rtype)) {
 513         if (!defined($opt{"GRUB_MENU"})) {
 514             get_ktest_config("REBOOT_TYPE");
 515             $rtype = $entered_configs{"REBOOT_TYPE"};
 516         } else {
 517             $rtype = "grub";
 518         }
 519     }
 520
 521     if ($rtype eq "grub") {
 522         get_ktest_config("GRUB_MENU");
 523     }
 524 }
 525
 526 sub process_variables {
 527     my ($value, $remove_undef) = @_;
 528     my $retval = "";
 529
 530     # We want to check for '\', and it is just easier
 531     # to check the previous characet of '$' and not need
 532     # to worry if '$' is the first character. By adding
 533     # a space to $value, we can just check [^\\]\$ and
 534     # it will still work.
 535     $value = " $value";
 536
 537     while ($value =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
 538         my $begin = $1;
 539         my $var = $2;
 540         my $end = $3;
 541         # append beginning of value to retval
 542         $retval = "$retval$begin";
 543         if (defined($variable{$var})) {
 544             $retval = "$retval$variable{$var}";
 545         } elsif (defined($remove_undef) && $remove_undef) {
 546             # for if statements, any variable that is not defined,
 547             # we simple convert to 0
 548             $retval = "${retval}0";
 549         } else {
 550             # put back the origin piece.
 551             $retval = "$retval\$\{$var\}";
 552             # This could be an option that is used later, save
 553             # it so we don't warn if this option is not one of
 554             # ktests options.
 555             $used_options{$var} = 1;
 556         }
 557         $value = $end;
 558     }
 559     $retval = "$retval$value";
 560
 561     # remove the space added in the beginning
 562     $retval =~ s/ //;
 563
 564     return "$retval"
 565 }
 566
 567 sub set_value {
 568     my ($lvalue, $rvalue, $override, $overrides, $name) = @_;
 569
 570     my $prvalue = process_variables($rvalue);
 571
 572     if ($buildonly && $lvalue =~ /^TEST_TYPE(\[.*\])?$/ && $prvalue ne "build") {
 573         # Note if a test is something other than build, then we
 574         # will need other manditory options.
 575         if ($prvalue ne "install") {
 576             $buildonly = 0;
 577         } else {
 578             # install still limits some manditory options.
 579             $buildonly = 2;
 580         }
 581     }
 582
 583     if (defined($opt{$lvalue})) {
 584         if (!$override || defined(${$overrides}{$lvalue})) {
 585             my $extra = "";
 586             if ($override) {
 587                 $extra = "In the same override section!\n";
 588             }
 589             die "$name: $.: Option $lvalue defined more than once!\n$extra";
 590         }
 591         ${$overrides}{$lvalue} = $prvalue;
 592     }
 593     if ($rvalue =~ /^\s*$/) {
 594         delete $opt{$lvalue};
 595     } else {
 596         $opt{$lvalue} = $prvalue;
 597     }
 598 }
 599
 600 sub set_variable {
 601     my ($lvalue, $rvalue) = @_;
 602
 603     if ($rvalue =~ /^\s*$/) {
 604         delete $variable{$lvalue};
 605     } else {
 606         $rvalue = process_variables($rvalue);
 607         $variable{$lvalue} = $rvalue;
 608     }
 609 }
 610
 611 sub process_compare {
 612     my ($lval, $cmp, $rval) = @_;
 613
 614     # remove whitespace
 615
 616     $lval =~ s/^\s*//;
 617     $lval =~ s/\s*$//;
 618
 619     $rval =~ s/^\s*//;
 620     $rval =~ s/\s*$//;
 621
 622     if ($cmp eq "==") {
 623         return $lval eq $rval;
 624     } elsif ($cmp eq "!=") {
 625         return $lval ne $rval;
 626     }
 627
 628     my $statement = "$lval $cmp $rval";
 629     my $ret = eval $statement;
 630
 631     # $@ stores error of eval
 632     if ($@) {
 633         return -1;
 634     }
 635
 636     return $ret;
 637 }
 638
 639 sub value_defined {
 640     my ($val) = @_;
 641
 642     return defined($variable{$2}) ||
 643         defined($opt{$2});
 644 }
 645
 646 my $d = 0;
 647 sub process_expression {
 648     my ($name, $val) = @_;
 649
 650     my $c = $d++;
 651
 652     while ($val =~ s/\(([^\(]*?)\)/\&\&\&\&VAL\&\&\&\&/) {
 653         my $express = $1;
 654
 655         if (process_expression($name, $express)) {
 656             $val =~ s/\&\&\&\&VAL\&\&\&\&/ 1 /;
 657         } else {
 658             $val =~ s/\&\&\&\&VAL\&\&\&\&/ 0 /;
 659         }
 660     }
 661
 662     $d--;
 663     my $OR = "\\|\\|";
 664     my $AND = "\\&\\&";
 665
 666     while ($val =~ s/^(.*?)($OR|$AND)//) {
 667         my $express = $1;
 668         my $op = $2;
 669
 670         if (process_expression($name, $express)) {
 671             if ($op eq "||") {
 672                 return 1;
 673             }
 674         } else {
 675             if ($op eq "&&") {
 676                 return 0;
 677             }
 678         }
 679     }
 680
 681     if ($val =~ /(.*)(==|\!=|>=|<=|>|<)(.*)/) {
 682         my $ret = process_compare($1, $2, $3);
 683         if ($ret < 0) {
 684             die "$name: $.: Unable to process comparison\n";
 685         }
 686         return $ret;
 687     }
 688
 689     if ($val =~ /^\s*(NOT\s*)?DEFINED\s+(\S+)\s*$/) {
 690         if (defined $1) {
 691             return !value_defined($2);
 692         } else {
 693             return value_defined($2);
 694         }
 695     }
 696
 697     if ($val =~ /^\s*0\s*$/) {
 698         return 0;
 699     } elsif ($val =~ /^\s*\d+\s*$/) {
 700         return 1;
 701     }
 702
 703     die ("$name: $.: Undefined content $val in if statement\n");
 704 }
 705
 706 sub process_if {
 707     my ($name, $value) = @_;
 708
 709     # Convert variables and replace undefined ones with 0
 710     my $val = process_variables($value, 1);
 711     my $ret = process_expression $name, $val;
 712
 713     return $ret;
 714 }
 715
 716 sub __read_config {
 717     my ($config, $current_test_num) = @_;
 718
 719     my $in;
 720     open($in, $config) || die "can't read file $config";
 721
 722     my $name = $config;
 723     $name =~ s,.*/(.*),$1,;
 724
 725     my $test_num = $$current_test_num;
 726     my $default = 1;
 727     my $repeat = 1;
 728     my $num_tests_set = 0;
 729     my $skip = 0;
 730     my $rest;
 731     my $line;
 732     my $test_case = 0;
 733     my $if = 0;
 734     my $if_set = 0;
 735     my $override = 0;
 736
 737     my %overrides;
 738
 739     while (<$in>) {
 740
 741         # ignore blank lines and comments
 742         next if (/^\s*$/ || /\s*\#/);
 743
 744         if (/^\s*(TEST_START|DEFAULTS)\b(.*)/) {
 745
 746             my $type = $1;
 747             $rest = $2;
 748             $line = $2;
 749
 750             my $old_test_num;
 751             my $old_repeat;
 752             $override = 0;
 753
 754             if ($type eq "TEST_START") {
 755
 756                 if ($num_tests_set) {
 757                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 758                 }
 759
 760                 $old_test_num = $test_num;
 761                 $old_repeat = $repeat;
 762
 763                 $test_num += $repeat;
 764                 $default = 0;
 765                 $repeat = 1;
 766             } else {
 767                 $default = 1;
 768             }
 769
 770             # If SKIP is anywhere in the line, the command will be skipped
 771             if ($rest =~ s/\s+SKIP\b//) {
 772                 $skip = 1;
 773             } else {
 774                 $test_case = 1;
 775                 $skip = 0;
 776             }
 777
 778             if ($rest =~ s/\sELSE\b//) {
 779                 if (!$if) {
 780                     die "$name: $.: ELSE found with out matching IF section\n$_";
 781                 }
 782                 $if = 0;
 783
 784                 if ($if_set) {
 785                     $skip = 1;
 786                 } else {
 787                     $skip = 0;
 788                 }
 789             }
 790
 791             if ($rest =~ s/\sIF\s+(.*)//) {
 792                 if (process_if($name, $1)) {
 793                     $if_set = 1;
 794                 } else {
 795                     $skip = 1;
 796                 }
 797                 $if = 1;
 798             } else {
 799                 $if = 0;
 800                 $if_set = 0;
 801             }
 802
 803             if (!$skip) {
 804                 if ($type eq "TEST_START") {
 805                     if ($rest =~ s/\s+ITERATE\s+(\d+)//) {
 806                         $repeat = $1;
 807                         $repeat_tests{"$test_num"} = $repeat;
 808                     }
 809                 } elsif ($rest =~ s/\sOVERRIDE\b//) {
 810                     # DEFAULT only
 811                     $override = 1;
 812                     # Clear previous overrides
 813                     %overrides = ();
 814                 }
 815             }
 816
 817             if (!$skip && $rest !~ /^\s*$/) {
 818                 die "$name: $.: Gargbage found after $type\n$_";
 819             }
 820
 821             if ($skip && $type eq "TEST_START") {
 822                 $test_num = $old_test_num;
 823                 $repeat = $old_repeat;
 824             }
 825
 826         } elsif (/^\s*ELSE\b(.*)$/) {
 827             if (!$if) {
 828                 die "$name: $.: ELSE found with out matching IF section\n$_";
 829             }
 830             $rest = $1;
 831             if ($if_set) {
 832                 $skip = 1;
 833                 $rest = "";
 834             } else {
 835                 $skip = 0;
 836
 837                 if ($rest =~ /\sIF\s+(.*)/) {
 838                     # May be a ELSE IF section.
 839                     if (!process_if($name, $1)) {
 840                         $skip = 1;
 841                     }
 842                     $rest = "";
 843                 } else {
 844                     $if = 0;
 845                 }
 846             }
 847
 848             if ($rest !~ /^\s*$/) {
 849                 die "$name: $.: Gargbage found after DEFAULTS\n$_";
 850             }
 851
 852         } elsif (/^\s*INCLUDE\s+(\S+)/) {
 853
 854             next if ($skip);
 855
 856             if (!$default) {
 857                 die "$name: $.: INCLUDE can only be done in default sections\n$_";
 858             }
 859
 860             my $file = process_variables($1);
 861
 862             if ($file !~ m,^/,) {
 863                 # check the path of the config file first
 864                 if ($config =~ m,(.*)/,) {
 865                     if (-f "$1/$file") {
 866                         $file = "$1/$file";
 867                     }
 868                 }
 869             }
 870
 871             if ( ! -r $file ) {
 872                 die "$name: $.: Can't read file $file\n$_";
 873             }
 874
 875             if (__read_config($file, \$test_num)) {
 876                 $test_case = 1;
 877             }
 878
 879         } elsif (/^\s*([A-Z_\[\]\d]+)\s*=\s*(.*?)\s*$/) {
 880
 881             next if ($skip);
 882
 883             my $lvalue = $1;
 884             my $rvalue = $2;
 885
 886             if (!$default &&
 887                 ($lvalue eq "NUM_TESTS" ||
 888                  $lvalue eq "LOG_FILE" ||
 889                  $lvalue eq "CLEAR_LOG")) {
 890                 die "$name: $.: $lvalue must be set in DEFAULTS section\n";
 891             }
 892
 893             if ($lvalue eq "NUM_TESTS") {
 894                 if ($test_num) {
 895                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 896                 }
 897                 if (!$default) {
 898                     die "$name: $.: NUM_TESTS must be set in default section\n";
 899                 }
 900                 $num_tests_set = 1;
 901             }
 902
 903             if ($default || $lvalue =~ /\[\d+\]$/) {
 904                 set_value($lvalue, $rvalue, $override, \%overrides, $name);
 905             } else {
 906                 my $val = "$lvalue\[$test_num\]";
 907                 set_value($val, $rvalue, $override, \%overrides, $name);
 908
 909                 if ($repeat > 1) {
 910                     $repeats{$val} = $repeat;
 911                 }
 912             }
 913         } elsif (/^\s*([A-Z_\[\]\d]+)\s*:=\s*(.*?)\s*$/) {
 914             next if ($skip);
 915
 916             my $lvalue = $1;
 917             my $rvalue = $2;
 918
 919             # process config variables.
 920             # Config variables are only active while reading the
 921             # config and can be defined anywhere. They also ignore
 922             # TEST_START and DEFAULTS, but are skipped if they are in
 923             # on of these sections that have SKIP defined.
 924             # The save variable can be
 925             # defined multiple times and the new one simply overrides
 926             # the prevous one.
 927             set_variable($lvalue, $rvalue);
 928
 929         } else {
 930             die "$name: $.: Garbage found in config\n$_";
 931         }
 932     }
 933
 934     if ($test_num) {
 935         $test_num += $repeat - 1;
 936         $opt{"NUM_TESTS"} = $test_num;
 937     }
 938
 939     close($in);
 940
 941     $$current_test_num = $test_num;
 942
 943     return $test_case;
 944 }
 945
 946 sub get_test_case {
 947         print "What test case would you like to run?\n";
 948         print " (build, install or boot)\n";
 949         print " Other tests are available but require editing the config file\n";
 950         my $ans = <STDIN>;
 951         chomp $ans;
 952         $default{"TEST_TYPE"} = $ans;
 953 }
 954
 955 sub read_config {
 956     my ($config) = @_;
 957
 958     my $test_case;
 959     my $test_num = 0;
 960
 961     $test_case = __read_config $config, \$test_num;
 962
 963     # make sure we have all mandatory configs
 964     get_ktest_configs;
 965
 966     # was a test specified?
 967     if (!$test_case) {
 968         print "No test case specified.\n";
 969         get_test_case;
 970     }
 971
 972     # set any defaults
 973
 974     foreach my $default (keys %default) {
 975         if (!defined($opt{$default})) {
 976             $opt{$default} = $default{$default};
 977         }
 978     }
 979
 980     if ($opt{"IGNORE_UNUSED"} == 1) {
 981         return;
 982     }
 983
 984     my %not_used;
 985
 986     # check if there are any stragglers (typos?)
 987     foreach my $option (keys %opt) {
 988         my $op = $option;
 989         # remove per test labels.
 990         $op =~ s/\[.*\]//;
 991         if (!exists($option_map{$op}) &&
 992             !exists($default{$op}) &&
 993             !exists($used_options{$op})) {
 994             $not_used{$op} = 1;
 995         }
 996     }
 997
 998     if (%not_used) {
 999         my $s = "s are";
1000         $s = " is" if (keys %not_used == 1);
1001         print "The following option$s not used; could be a typo:\n";
1002         foreach my $option (keys %not_used) {
1003             print "$option\n";
1004         }
1005         print "Set IGRNORE_UNUSED = 1 to have ktest ignore unused variables\n";
1006         if (!read_yn "Do you want to continue?") {
1007             exit -1;
1008         }
1009     }
1010 }
1011
1012 sub __eval_option {
1013     my ($option, $i) = @_;
1014
1015     # Add space to evaluate the character before $
1016     $option = " $option";
1017     my $retval = "";
1018     my $repeated = 0;
1019     my $parent = 0;
1020
1021     foreach my $test (keys %repeat_tests) {
1022         if ($i >= $test &&
1023             $i < $test + $repeat_tests{$test}) {
1024
1025             $repeated = 1;
1026             $parent = $test;
1027             last;
1028         }
1029     }
1030
1031     while ($option =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
1032         my $start = $1;
1033         my $var = $2;
1034         my $end = $3;
1035
1036         # Append beginning of line
1037         $retval = "$retval$start";
1038
1039         # If the iteration option OPT[$i] exists, then use that.
1040         # otherwise see if the default OPT (without [$i]) exists.
1041
1042         my $o = "$var\[$i\]";
1043         my $parento = "$var\[$parent\]";
1044
1045         if (defined($opt{$o})) {
1046             $o = $opt{$o};
1047             $retval = "$retval$o";
1048         } elsif ($repeated && defined($opt{$parento})) {
1049             $o = $opt{$parento};
1050             $retval = "$retval$o";
1051         } elsif (defined($opt{$var})) {
1052             $o = $opt{$var};
1053             $retval = "$retval$o";
1054         } else {
1055             $retval = "$retval\$\{$var\}";
1056         }
1057
1058         $option = $end;
1059     }
1060
1061     $retval = "$retval$option";
1062
1063     $retval =~ s/^ //;
1064
1065     return $retval;
1066 }
1067
1068 sub eval_option {
1069     my ($option, $i) = @_;
1070
1071     my $prev = "";
1072
1073     # Since an option can evaluate to another option,
1074     # keep iterating until we do not evaluate any more
1075     # options.
1076     my $r = 0;
1077     while ($prev ne $option) {
1078         # Check for recursive evaluations.
1079         # 100 deep should be more than enough.
1080         if ($r++ > 100) {
1081             die "Over 100 evaluations accurred with $option\n" .
1082                 "Check for recursive variables\n";
1083         }
1084         $prev = $option;
1085         $option = __eval_option($option, $i);
1086     }
1087
1088     return $option;
1089 }
1090
1091 sub _logit {
1092     if (defined($opt{"LOG_FILE"})) {
1093         open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
1094         print OUT @_;
1095         close(OUT);
1096     }
1097 }
1098
1099 sub logit {
1100     if (defined($opt{"LOG_FILE"})) {
1101         _logit @_;
1102     } else {
1103         print @_;
1104     }
1105 }
1106
1107 sub doprint {
1108     print @_;
1109     _logit @_;
1110 }
1111
1112 sub run_command;
1113 sub start_monitor;
1114 sub end_monitor;
1115 sub wait_for_monitor;
1116
1117 sub reboot {
1118     my ($time) = @_;
1119
1120     if (defined($time)) {
1121         start_monitor;
1122         # flush out current monitor
1123         # May contain the reboot success line
1124         wait_for_monitor 1;
1125     }
1126
1127     # try to reboot normally
1128     if (run_command $reboot) {
1129         if (defined($powercycle_after_reboot)) {
1130             sleep $powercycle_after_reboot;
1131             run_command "$power_cycle";
1132         }
1133     } else {
1134         # nope? power cycle it.
1135         run_command "$power_cycle";
1136     }
1137
1138     if (defined($time)) {
1139         if (wait_for_monitor($time, $reboot_success_line)) {
1140             # reboot got stuck?
1141             doprint "Reboot did not finish. Forcing power cycle\n";
1142             run_command "$power_cycle";
1143         }
1144         end_monitor;
1145     }
1146 }
1147
1148 sub reboot_to_good {
1149     my ($time) = @_;
1150
1151     if (defined($switch_to_good)) {
1152         run_command $switch_to_good;
1153     }
1154
1155     reboot $time;
1156 }
1157
1158 sub do_not_reboot {
1159     my $i = $iteration;
1160
1161     return $test_type eq "build" || $no_reboot ||
1162         ($test_type eq "patchcheck" && $opt{"PATCHCHECK_TYPE[$i]"} eq "build") ||
1163         ($test_type eq "bisect" && $opt{"BISECT_TYPE[$i]"} eq "build");
1164 }
1165
1166 sub dodie {
1167     doprint "CRITICAL FAILURE... ", @_, "\n";
1168
1169     my $i = $iteration;
1170
1171     if ($reboot_on_error && !do_not_reboot) {
1172
1173         doprint "REBOOTING\n";
1174         reboot_to_good;
1175
1176     } elsif ($poweroff_on_error && defined($power_off)) {
1177         doprint "POWERING OFF\n";
1178         `$power_off`;
1179     }
1180
1181     if (defined($opt{"LOG_FILE"})) {
1182         print " See $opt{LOG_FILE} for more info.\n";
1183     }
1184
1185     die @_, "\n";
1186 }
1187
1188 sub open_console {
1189     my ($fp) = @_;
1190
1191     my $flags;
1192
1193     my $pid = open($fp, "$console|") or
1194         dodie "Can't open console $console";
1195
1196     $flags = fcntl($fp, F_GETFL, 0) or
1197         dodie "Can't get flags for the socket: $!";
1198     $flags = fcntl($fp, F_SETFL, $flags | O_NONBLOCK) or
1199         dodie "Can't set flags for the socket: $!";
1200
1201     return $pid;
1202 }
1203
1204 sub close_console {
1205     my ($fp, $pid) = @_;
1206
1207     doprint "kill child process $pid\n";
1208     kill 2, $pid;
1209
1210     print "closing!\n";
1211     close($fp);
1212 }
1213
1214 sub start_monitor {
1215     if ($monitor_cnt++) {
1216         return;
1217     }
1218     $monitor_fp = \*MONFD;
1219     $monitor_pid = open_console $monitor_fp;
1220
1221     return;
1222
1223     open(MONFD, "Stop perl from warning about single use of MONFD");
1224 }
1225
1226 sub end_monitor {
1227     if (--$monitor_cnt) {
1228         return;
1229     }
1230     close_console($monitor_fp, $monitor_pid);
1231 }
1232
1233 sub wait_for_monitor {
1234     my ($time, $stop) = @_;
1235     my $full_line = "";
1236     my $line;
1237     my $booted = 0;
1238     my $start_time = time;
1239     my $skip_call_trace = 0;
1240     my $bug = 0;
1241     my $bug_ignored = 0;
1242     my $now;
1243
1244     doprint "** Wait for monitor to settle down **\n";
1245
1246     # read the monitor and wait for the system to calm down
1247     while (!$booted) {
1248         $line = wait_for_input($monitor_fp, $time);
1249         last if (!defined($line));
1250         print "$line";
1251         $full_line .= $line;
1252
1253         if (defined($stop) && $full_line =~ /$stop/) {
1254             doprint "wait for monitor detected $stop\n";
1255             $booted = 1;
1256         }
1257
1258         if ($full_line =~ /\[ backtrace testing \]/) {
1259             $skip_call_trace = 1;
1260         }
1261
1262         if ($full_line =~ /call trace:/i) {
1263             if (!$bug && !$skip_call_trace) {
1264                 if ($ignore_errors) {
1265                     $bug_ignored = 1;
1266                 } else {
1267                     $bug = 1;
1268                 }
1269             }
1270         }
1271
1272         if ($full_line =~ /\[ end of backtrace testing \]/) {
1273             $skip_call_trace = 0;
1274         }
1275
1276         if ($full_line =~ /Kernel panic -/) {
1277             $bug = 1;
1278         }
1279
1280         if ($line =~ /\n/) {
1281             $full_line = "";
1282         }
1283         $now = time;
1284         if ($now - $start_time >= $max_monitor_wait) {
1285             doprint "Exiting monitor flush due to hitting MAX_MONITOR_WAIT\n";
1286             return 1;
1287         }
1288     }
1289     print "** Monitor flushed **\n";
1290     return $bug;
1291 }
1292
1293 sub save_logs {
1294         my ($result, $basedir) = @_;
1295         my @t = localtime;
1296         my $date = sprintf "%04d%02d%02d%02d%02d%02d",
1297                 1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
1298
1299         my $type = $build_type;
1300         if ($type =~ /useconfig/) {
1301             $type = "useconfig";
1302         }
1303
1304         my $dir = "$machine-$test_type-$type-$result-$date";
1305
1306         $dir = "$basedir/$dir";
1307
1308         if (!-d $dir) {
1309             mkpath($dir) or
1310                 die "can't create $dir";
1311         }
1312
1313         my %files = (
1314                 "config" => $output_config,
1315                 "buildlog" => $buildlog,
1316                 "dmesg" => $dmesg,
1317                 "testlog" => $testlog,
1318         );
1319
1320         while (my ($name, $source) = each(%files)) {
1321                 if (-f "$source") {
1322                         cp "$source", "$dir/$name" or
1323                                 die "failed to copy $source";
1324                 }
1325         }
1326
1327         doprint "*** Saved info to $dir ***\n";
1328 }
1329
1330 sub fail {
1331
1332         if (defined($post_test)) {
1333                 run_command $post_test;
1334         }
1335
1336         if ($die_on_failure) {
1337                 dodie @_;
1338         }
1339
1340         doprint "FAILED\n";
1341
1342         my $i = $iteration;
1343
1344         # no need to reboot for just building.
1345         if (!do_not_reboot) {
1346             doprint "REBOOTING\n";
1347             reboot_to_good $sleep_time;
1348         }
1349
1350         my $name = "";
1351
1352         if (defined($test_name)) {
1353             $name = " ($test_name)";
1354         }
1355
1356         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1357         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1358         doprint "KTEST RESULT: TEST $i$name Failed: ", @_, "\n";
1359         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1360         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1361
1362         if (defined($store_failures)) {
1363             save_logs "fail", $store_failures;
1364         }
1365
1366         return 1;
1367 }
1368
1369 sub run_command {
1370     my ($command) = @_;
1371     my $dolog = 0;
1372     my $dord = 0;
1373     my $pid;
1374
1375     $command =~ s/\$SSH_USER/$ssh_user/g;
1376     $command =~ s/\$MACHINE/$machine/g;
1377
1378     doprint("$command ... ");
1379
1380     $pid = open(CMD, "$command 2>&1 |") or
1381         (fail "unable to exec $command" and return 0);
1382
1383     if (defined($opt{"LOG_FILE"})) {
1384         open(LOG, ">>$opt{LOG_FILE}") or
1385             dodie "failed to write to log";
1386         $dolog = 1;
1387     }
1388
1389     if (defined($redirect)) {
1390         open (RD, ">$redirect") or
1391             dodie "failed to write to redirect $redirect";
1392         $dord = 1;
1393     }
1394
1395     while (<CMD>) {
1396         print LOG if ($dolog);
1397         print RD  if ($dord);
1398     }
1399
1400     waitpid($pid, 0);
1401     my $failed = $?;
1402
1403     close(CMD);
1404     close(LOG) if ($dolog);
1405     close(RD)  if ($dord);
1406
1407     if ($failed) {
1408         doprint "FAILED!\n";
1409     } else {
1410         doprint "SUCCESS\n";
1411     }
1412
1413     return !$failed;
1414 }
1415
1416 sub run_ssh {
1417     my ($cmd) = @_;
1418     my $cp_exec = $ssh_exec;
1419
1420     $cp_exec =~ s/\$SSH_COMMAND/$cmd/g;
1421     return run_command "$cp_exec";
1422 }
1423
1424 sub run_scp {
1425     my ($src, $dst, $cp_scp) = @_;
1426
1427     $cp_scp =~ s/\$SRC_FILE/$src/g;
1428     $cp_scp =~ s/\$DST_FILE/$dst/g;
1429
1430     return run_command "$cp_scp";
1431 }
1432
1433 sub run_scp_install {
1434     my ($src, $dst) = @_;
1435
1436     my $cp_scp = $scp_to_target_install;
1437
1438     return run_scp($src, $dst, $cp_scp);
1439 }
1440
1441 sub run_scp_mod {
1442     my ($src, $dst) = @_;
1443
1444     my $cp_scp = $scp_to_target;
1445
1446     return run_scp($src, $dst, $cp_scp);
1447 }
1448
1449 sub get_grub_index {
1450
1451     if ($reboot_type ne "grub") {
1452         return;
1453     }
1454     return if (defined($grub_number));
1455
1456     doprint "Find grub menu ... ";
1457     $grub_number = -1;
1458
1459     my $ssh_grub = $ssh_exec;
1460     $ssh_grub =~ s,\$SSH_COMMAND,cat /boot/grub/menu.lst,g;
1461
1462     open(IN, "$ssh_grub |")
1463         or die "unable to get menu.lst";
1464
1465     my $found = 0;
1466
1467     while (<IN>) {
1468         if (/^\s*title\s+$grub_menu\s*$/) {
1469             $grub_number++;
1470             $found = 1;
1471             last;
1472         } elsif (/^\s*title\s/) {
1473             $grub_number++;
1474         }
1475     }
1476     close(IN);
1477
1478     die "Could not find '$grub_menu' in /boot/grub/menu on $machine"
1479         if (!$found);
1480     doprint "$grub_number\n";
1481 }
1482
1483 sub wait_for_input
1484 {
1485     my ($fp, $time) = @_;
1486     my $rin;
1487     my $ready;
1488     my $line;
1489     my $ch;
1490
1491     if (!defined($time)) {
1492         $time = $timeout;
1493     }
1494
1495     $rin = '';
1496     vec($rin, fileno($fp), 1) = 1;
1497     $ready = select($rin, undef, undef, $time);
1498
1499     $line = "";
1500
1501     # try to read one char at a time
1502     while (sysread $fp, $ch, 1) {
1503         $line .= $ch;
1504         last if ($ch eq "\n");
1505     }
1506
1507     if (!length($line)) {
1508         return undef;
1509     }
1510
1511     return $line;
1512 }
1513
1514 sub reboot_to {
1515     if (defined($switch_to_test)) {
1516         run_command $switch_to_test;
1517     }
1518
1519     if ($reboot_type eq "grub") {
1520         run_ssh "'(echo \"savedefault --default=$grub_number --once\" | grub --batch)'";
1521     } elsif (defined $reboot_script) {
1522         run_command "$reboot_script";
1523     }
1524     reboot;
1525 }
1526
1527 sub get_sha1 {
1528     my ($commit) = @_;
1529
1530     doprint "git rev-list --max-count=1 $commit ... ";
1531     my $sha1 = `git rev-list --max-count=1 $commit`;
1532     my $ret = $?;
1533
1534     logit $sha1;
1535
1536     if ($ret) {
1537         doprint "FAILED\n";
1538         dodie "Failed to get git $commit";
1539     }
1540
1541     print "SUCCESS\n";
1542
1543     chomp $sha1;
1544
1545     return $sha1;
1546 }
1547
1548 sub monitor {
1549     my $booted = 0;
1550     my $bug = 0;
1551     my $bug_ignored = 0;
1552     my $skip_call_trace = 0;
1553     my $loops;
1554
1555     wait_for_monitor 5;
1556
1557     my $line;
1558     my $full_line = "";
1559
1560     open(DMESG, "> $dmesg") or
1561         die "unable to write to $dmesg";
1562
1563     reboot_to;
1564
1565     my $success_start;
1566     my $failure_start;
1567     my $monitor_start = time;
1568     my $done = 0;
1569     my $version_found = 0;
1570
1571     while (!$done) {
1572
1573         if ($bug && defined($stop_after_failure) &&
1574             $stop_after_failure >= 0) {
1575             my $time = $stop_after_failure - (time - $failure_start);
1576             $line = wait_for_input($monitor_fp, $time);
1577             if (!defined($line)) {
1578                 doprint "bug timed out after $booted_timeout seconds\n";
1579                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
1580                 last;
1581             }
1582         } elsif ($booted) {
1583             $line = wait_for_input($monitor_fp, $booted_timeout);
1584             if (!defined($line)) {
1585                 my $s = $booted_timeout == 1 ? "" : "s";
1586                 doprint "Successful boot found: break after $booted_timeout second$s\n";
1587                 last;
1588             }
1589         } else {
1590             $line = wait_for_input($monitor_fp);
1591             if (!defined($line)) {
1592                 my $s = $timeout == 1 ? "" : "s";
1593                 doprint "Timed out after $timeout second$s\n";
1594                 last;
1595             }
1596         }
1597
1598         doprint $line;
1599         print DMESG $line;
1600
1601         # we are not guaranteed to get a full line
1602         $full_line .= $line;
1603
1604         if ($full_line =~ /$success_line/) {
1605             $booted = 1;
1606             $success_start = time;
1607         }
1608
1609         if ($booted && defined($stop_after_success) &&
1610             $stop_after_success >= 0) {
1611             my $now = time;
1612             if ($now - $success_start >= $stop_after_success) {
1613                 doprint "Test forced to stop after $stop_after_success seconds after success\n";
1614                 last;
1615             }
1616         }
1617
1618         if ($full_line =~ /\[ backtrace testing \]/) {
1619             $skip_call_trace = 1;
1620         }
1621
1622         if ($full_line =~ /call trace:/i) {
1623             if (!$bug && !$skip_call_trace) {
1624                 if ($ignore_errors) {
1625                     $bug_ignored = 1;
1626                 } else {
1627                     $bug = 1;
1628                     $failure_start = time;
1629                 }
1630             }
1631         }
1632
1633         if ($bug && defined($stop_after_failure) &&
1634             $stop_after_failure >= 0) {
1635             my $now = time;
1636             if ($now - $failure_start >= $stop_after_failure) {
1637                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
1638                 last;
1639             }
1640         }
1641
1642         if ($full_line =~ /\[ end of backtrace testing \]/) {
1643             $skip_call_trace = 0;
1644         }
1645
1646         if ($full_line =~ /Kernel panic -/) {
1647             $failure_start = time;
1648             $bug = 1;
1649         }
1650
1651         # Detect triple faults by testing the banner
1652         if ($full_line =~ /\bLinux version (\S+).*\n/) {
1653             if ($1 eq $version) {
1654                 $version_found = 1;
1655             } elsif ($version_found && $detect_triplefault) {
1656                 # We already booted into the kernel we are testing,
1657                 # but now we booted into another kernel?
1658                 # Consider this a triple fault.
1659                 doprint "Aleady booted in Linux kernel $version, but now\n";
1660                 doprint "we booted into Linux kernel $1.\n";
1661                 doprint "Assuming that this is a triple fault.\n";
1662                 doprint "To disable this: set DETECT_TRIPLE_FAULT to 0\n";
1663                 last;
1664             }
1665         }
1666
1667         if ($line =~ /\n/) {
1668             $full_line = "";
1669         }
1670
1671         if ($stop_test_after > 0 && !$booted && !$bug) {
1672             if (time - $monitor_start > $stop_test_after) {
1673                 doprint "STOP_TEST_AFTER ($stop_test_after seconds) timed out\n";
1674                 $done = 1;
1675             }
1676         }
1677     }
1678
1679     close(DMESG);
1680
1681     if ($bug) {
1682         return 0 if ($in_bisect);
1683         fail "failed - got a bug report" and return 0;
1684     }
1685
1686     if (!$booted) {
1687         return 0 if ($in_bisect);
1688         fail "failed - never got a boot prompt." and return 0;
1689     }
1690
1691     if ($bug_ignored) {
1692         doprint "WARNING: Call Trace detected but ignored due to IGNORE_ERRORS=1\n";
1693     }
1694
1695     return 1;
1696 }
1697
1698 sub eval_kernel_version {
1699     my ($option) = @_;
1700
1701     $option =~ s/\$KERNEL_VERSION/$version/g;
1702
1703     return $option;
1704 }
1705
1706 sub do_post_install {
1707
1708     return if (!defined($post_install));
1709
1710     my $cp_post_install = eval_kernel_version $post_install;
1711     run_command "$cp_post_install" or
1712         dodie "Failed to run post install";
1713 }
1714
1715 sub install {
1716
1717     return if ($no_install);
1718
1719     if (defined($pre_install)) {
1720         my $cp_pre_install = eval_kernel_version $pre_install;
1721         run_command "$cp_pre_install" or
1722             dodie "Failed to run pre install";
1723     }
1724
1725     my $cp_target = eval_kernel_version $target_image;
1726
1727     run_scp_install "$outputdir/$build_target", "$cp_target" or
1728         dodie "failed to copy image";
1729
1730     my $install_mods = 0;
1731
1732     # should we process modules?
1733     $install_mods = 0;
1734     open(IN, "$output_config") or dodie("Can't read config file");
1735     while (<IN>) {
1736         if (/CONFIG_MODULES(=y)?/) {
1737             $install_mods = 1 if (defined($1));
1738             last;
1739         }
1740     }
1741     close(IN);
1742
1743     if (!$install_mods) {
1744         do_post_install;
1745         doprint "No modules needed\n";
1746         return;
1747     }
1748
1749     run_command "$make INSTALL_MOD_STRIP=1 INSTALL_MOD_PATH=$tmpdir modules_install" or
1750         dodie "Failed to install modules";
1751
1752     my $modlib = "/lib/modules/$version";
1753     my $modtar = "ktest-mods.tar.bz2";
1754
1755     run_ssh "rm -rf $modlib" or
1756         dodie "failed to remove old mods: $modlib";
1757
1758     # would be nice if scp -r did not follow symbolic links
1759     run_command "cd $tmpdir && tar -cjf $modtar lib/modules/$version" or
1760         dodie "making tarball";
1761
1762     run_scp_mod "$tmpdir/$modtar", "/tmp" or
1763         dodie "failed to copy modules";
1764
1765     unlink "$tmpdir/$modtar";
1766
1767     run_ssh "'(cd / && tar xjf /tmp/$modtar)'" or
1768         dodie "failed to tar modules";
1769
1770     run_ssh "rm -f /tmp/$modtar";
1771
1772     do_post_install;
1773 }
1774
1775 sub get_version {
1776     # get the release name
1777     return if ($have_version);
1778     doprint "$make kernelrelease ... ";
1779     $version = `$make kernelrelease | tail -1`;
1780     chomp($version);
1781     doprint "$version\n";
1782     $have_version = 1;
1783 }
1784
1785 sub start_monitor_and_boot {
1786     # Make sure the stable kernel has finished booting
1787     start_monitor;
1788     wait_for_monitor 5;
1789     end_monitor;
1790
1791     get_grub_index;
1792     get_version;
1793     install;
1794
1795     start_monitor;
1796     return monitor;
1797 }
1798
1799 sub check_buildlog {
1800     my ($patch) = @_;
1801
1802     my @files = `git show $patch | diffstat -l`;
1803
1804     open(IN, "git show $patch |") or
1805         dodie "failed to show $patch";
1806     while (<IN>) {
1807         if (m,^--- a/(.*),) {
1808             chomp $1;
1809             $files[$#files] = $1;
1810         }
1811     }
1812     close(IN);
1813
1814     open(IN, $buildlog) or dodie "Can't open $buildlog";
1815     while (<IN>) {
1816         if (/^\s*(.*?):.*(warning|error)/) {
1817             my $err = $1;
1818             foreach my $file (@files) {
1819                 my $fullpath = "$builddir/$file";
1820                 if ($file eq $err || $fullpath eq $err) {
1821                     fail "$file built with warnings" and return 0;
1822                 }
1823             }
1824         }
1825     }
1826     close(IN);
1827
1828     return 1;
1829 }
1830
1831 sub apply_min_config {
1832     my $outconfig = "$output_config.new";
1833
1834     # Read the config file and remove anything that
1835     # is in the force_config hash (from minconfig and others)
1836     # then add the force config back.
1837
1838     doprint "Applying minimum configurations into $output_config.new\n";
1839
1840     open (OUT, ">$outconfig") or
1841         dodie "Can't create $outconfig";
1842
1843     if (-f $output_config) {
1844         open (IN, $output_config) or
1845             dodie "Failed to open $output_config";
1846         while (<IN>) {
1847             if (/^(# )?(CONFIG_[^\s=]*)/) {
1848                 next if (defined($force_config{$2}));
1849             }
1850             print OUT;
1851         }
1852         close IN;
1853     }
1854     foreach my $config (keys %force_config) {
1855         print OUT "$force_config{$config}\n";
1856     }
1857     close OUT;
1858
1859     run_command "mv $outconfig $output_config";
1860 }
1861
1862 sub make_oldconfig {
1863
1864     my @force_list = keys %force_config;
1865
1866     if ($#force_list >= 0) {
1867         apply_min_config;
1868     }
1869
1870     if (!run_command "$make oldnoconfig") {
1871         # Perhaps oldnoconfig doesn't exist in this version of the kernel
1872         # try a yes '' | oldconfig
1873         doprint "oldnoconfig failed, trying yes '' | make oldconfig\n";
1874         run_command "yes '' | $make oldconfig" or
1875             dodie "failed make config oldconfig";
1876     }
1877 }
1878
1879 # read a config file and use this to force new configs.
1880 sub load_force_config {
1881     my ($config) = @_;
1882
1883     doprint "Loading force configs from $config\n";
1884     open(IN, $config) or
1885         dodie "failed to read $config";
1886     while (<IN>) {
1887         chomp;
1888         if (/^(CONFIG[^\s=]*)(\s*=.*)/) {
1889             $force_config{$1} = $_;
1890         } elsif (/^# (CONFIG_\S*) is not set/) {
1891             $force_config{$1} = $_;
1892         }
1893     }
1894     close IN;
1895 }
1896
1897 sub build {
1898     my ($type) = @_;
1899
1900     unlink $buildlog;
1901
1902     # Failed builds should not reboot the target
1903     my $save_no_reboot = $no_reboot;
1904     $no_reboot = 1;
1905
1906     # Calculate a new version from here.
1907     $have_version = 0;
1908
1909     if (defined($pre_build)) {
1910         my $ret = run_command $pre_build;
1911         if (!$ret && defined($pre_build_die) &&
1912             $pre_build_die) {
1913             dodie "failed to pre_build\n";
1914         }
1915     }
1916
1917     if ($type =~ /^useconfig:(.*)/) {
1918         run_command "cp $1 $output_config" or
1919             dodie "could not copy $1 to .config";
1920
1921         $type = "oldconfig";
1922     }
1923
1924     # old config can ask questions
1925     if ($type eq "oldconfig") {
1926         $type = "oldnoconfig";
1927
1928         # allow for empty configs
1929         run_command "touch $output_config";
1930
1931         if (!$noclean) {
1932             run_command "mv $output_config $outputdir/config_temp" or
1933                 dodie "moving .config";
1934
1935             run_command "$make mrproper" or dodie "make mrproper";
1936
1937             run_command "mv $outputdir/config_temp $output_config" or
1938                 dodie "moving config_temp";
1939         }
1940
1941     } elsif (!$noclean) {
1942         unlink "$output_config";
1943         run_command "$make mrproper" or
1944             dodie "make mrproper";
1945     }
1946
1947     # add something to distinguish this build
1948     open(OUT, "> $outputdir/localversion") or dodie("Can't make localversion file");
1949     print OUT "$localversion\n";
1950     close(OUT);
1951
1952     if (defined($minconfig)) {
1953         load_force_config($minconfig);
1954     }
1955
1956     if ($type ne "oldnoconfig") {
1957         run_command "$make $type" or
1958             dodie "failed make config";
1959     }
1960     # Run old config regardless, to enforce min configurations
1961     make_oldconfig;
1962
1963     $redirect = "$buildlog";
1964     my $build_ret = run_command "$make $build_options";
1965     undef $redirect;
1966
1967     if (defined($post_build)) {
1968         # Because a post build may change the kernel version
1969         # do it now.
1970         get_version;
1971         my $ret = run_command $post_build;
1972         if (!$ret && defined($post_build_die) &&
1973             $post_build_die) {
1974             dodie "failed to post_build\n";
1975         }
1976     }
1977
1978     if (!$build_ret) {
1979         # bisect may need this to pass
1980         if ($in_bisect) {
1981             $no_reboot = $save_no_reboot;
1982             return 0;
1983         }
1984         fail "failed build" and return 0;
1985     }
1986
1987     $no_reboot = $save_no_reboot;
1988
1989     return 1;
1990 }
1991
1992 sub halt {
1993     if (!run_ssh "halt" or defined($power_off)) {
1994         if (defined($poweroff_after_halt)) {
1995             sleep $poweroff_after_halt;
1996             run_command "$power_off";
1997         }
1998     } else {
1999         # nope? the zap it!
2000         run_command "$power_off";
2001     }
2002 }
2003
2004 sub success {
2005     my ($i) = @_;
2006
2007     if (defined($post_test)) {
2008         run_command $post_test;
2009     }
2010
2011     $successes++;
2012
2013     my $name = "";
2014
2015     if (defined($test_name)) {
2016         $name = " ($test_name)";
2017     }
2018
2019     doprint "\n\n*******************************************\n";
2020     doprint     "*******************************************\n";
2021     doprint     "KTEST RESULT: TEST $i$name SUCCESS!!!!         **\n";
2022     doprint     "*******************************************\n";
2023     doprint     "*******************************************\n";
2024
2025     if (defined($store_successes)) {
2026         save_logs "success", $store_successes;
2027     }
2028
2029     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
2030         doprint "Reboot and wait $sleep_time seconds\n";
2031         reboot_to_good $sleep_time;
2032     }
2033 }
2034
2035 sub answer_bisect {
2036     for (;;) {
2037         doprint "Pass or fail? [p/f]";
2038         my $ans = <STDIN>;
2039         chomp $ans;
2040         if ($ans eq "p" || $ans eq "P") {
2041             return 1;
2042         } elsif ($ans eq "f" || $ans eq "F") {
2043             return 0;
2044         } else {
2045             print "Please answer 'P' or 'F'\n";
2046         }
2047     }
2048 }
2049
2050 sub child_run_test {
2051     my $failed = 0;
2052
2053     # child should have no power
2054     $reboot_on_error = 0;
2055     $poweroff_on_error = 0;
2056     $die_on_failure = 1;
2057
2058     $redirect = "$testlog";
2059     run_command $run_test or $failed = 1;
2060     undef $redirect;
2061
2062     exit $failed;
2063 }
2064
2065 my $child_done;
2066
2067 sub child_finished {
2068     $child_done = 1;
2069 }
2070
2071 sub do_run_test {
2072     my $child_pid;
2073     my $child_exit;
2074     my $line;
2075     my $full_line;
2076     my $bug = 0;
2077
2078     wait_for_monitor 1;
2079
2080     doprint "run test $run_test\n";
2081
2082     $child_done = 0;
2083
2084     $SIG{CHLD} = qw(child_finished);
2085
2086     $child_pid = fork;
2087
2088     child_run_test if (!$child_pid);
2089
2090     $full_line = "";
2091
2092     do {
2093         $line = wait_for_input($monitor_fp, 1);
2094         if (defined($line)) {
2095
2096             # we are not guaranteed to get a full line
2097             $full_line .= $line;
2098             doprint $line;
2099
2100             if ($full_line =~ /call trace:/i) {
2101                 $bug = 1;
2102             }
2103
2104             if ($full_line =~ /Kernel panic -/) {
2105                 $bug = 1;
2106             }
2107
2108             if ($line =~ /\n/) {
2109                 $full_line = "";
2110             }
2111         }
2112     } while (!$child_done && !$bug);
2113
2114     if ($bug) {
2115         my $failure_start = time;
2116         my $now;
2117         do {
2118             $line = wait_for_input($monitor_fp, 1);
2119             if (defined($line)) {
2120                 doprint $line;
2121             }
2122             $now = time;
2123             if ($now - $failure_start >= $stop_after_failure) {
2124                 last;
2125             }
2126         } while (defined($line));
2127
2128         doprint "Detected kernel crash!\n";
2129         # kill the child with extreme prejudice
2130         kill 9, $child_pid;
2131     }
2132
2133     waitpid $child_pid, 0;
2134     $child_exit = $?;
2135
2136     if (!$bug && $in_bisect) {
2137         if (defined($bisect_ret_good)) {
2138             if ($child_exit == $bisect_ret_good) {
2139                 return 1;
2140             }
2141         }
2142         if (defined($bisect_ret_skip)) {
2143             if ($child_exit == $bisect_ret_skip) {
2144                 return -1;
2145             }
2146         }
2147         if (defined($bisect_ret_abort)) {
2148             if ($child_exit == $bisect_ret_abort) {
2149                 fail "test abort" and return -2;
2150             }
2151         }
2152         if (defined($bisect_ret_bad)) {
2153             if ($child_exit == $bisect_ret_skip) {
2154                 return 0;
2155             }
2156         }
2157         if (defined($bisect_ret_default)) {
2158             if ($bisect_ret_default eq "good") {
2159                 return 1;
2160             } elsif ($bisect_ret_default eq "bad") {
2161                 return 0;
2162             } elsif ($bisect_ret_default eq "skip") {
2163                 return -1;
2164             } elsif ($bisect_ret_default eq "abort") {
2165                 return -2;
2166             } else {
2167                 fail "unknown default action: $bisect_ret_default"
2168                     and return -2;
2169             }
2170         }
2171     }
2172
2173     if ($bug || $child_exit) {
2174         return 0 if $in_bisect;
2175         fail "test failed" and return 0;
2176     }
2177     return 1;
2178 }
2179
2180 sub run_git_bisect {
2181     my ($command) = @_;
2182
2183     doprint "$command ... ";
2184
2185     my $output = `$command 2>&1`;
2186     my $ret = $?;
2187
2188     logit $output;
2189
2190     if ($ret) {
2191         doprint "FAILED\n";
2192         dodie "Failed to git bisect";
2193     }
2194
2195     doprint "SUCCESS\n";
2196     if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
2197         doprint "$1 [$2]\n";
2198     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
2199         $bisect_bad_commit = $1;
2200         doprint "Found bad commit... $1\n";
2201         return 0;
2202     } else {
2203         # we already logged it, just print it now.
2204         print $output;
2205     }
2206
2207     return 1;
2208 }
2209
2210 sub bisect_reboot {
2211     doprint "Reboot and sleep $bisect_sleep_time seconds\n";
2212     reboot_to_good $bisect_sleep_time;
2213 }
2214
2215 # returns 1 on success, 0 on failure, -1 on skip
2216 sub run_bisect_test {
2217     my ($type, $buildtype) = @_;
2218
2219     my $failed = 0;
2220     my $result;
2221     my $output;
2222     my $ret;
2223
2224     $in_bisect = 1;
2225
2226     build $buildtype or $failed = 1;
2227
2228     if ($type ne "build") {
2229         if ($failed && $bisect_skip) {
2230             $in_bisect = 0;
2231             return -1;
2232         }
2233         dodie "Failed on build" if $failed;
2234
2235         # Now boot the box
2236         start_monitor_and_boot or $failed = 1;
2237
2238         if ($type ne "boot") {
2239             if ($failed && $bisect_skip) {
2240                 end_monitor;
2241                 bisect_reboot;
2242                 $in_bisect = 0;
2243                 return -1;
2244             }
2245             dodie "Failed on boot" if $failed;
2246
2247             do_run_test or $failed = 1;
2248         }
2249         end_monitor;
2250     }
2251
2252     if ($failed) {
2253         $result = 0;
2254     } else {
2255         $result = 1;
2256     }
2257
2258     # reboot the box to a kernel we can ssh to
2259     if ($type ne "build") {
2260         bisect_reboot;
2261     }
2262     $in_bisect = 0;
2263
2264     return $result;
2265 }
2266
2267 sub run_bisect {
2268     my ($type) = @_;
2269     my $buildtype = "oldconfig";
2270
2271     # We should have a minconfig to use?
2272     if (defined($minconfig)) {
2273         $buildtype = "useconfig:$minconfig";
2274     }
2275
2276     my $ret = run_bisect_test $type, $buildtype;
2277
2278     if ($bisect_manual) {
2279         $ret = answer_bisect;
2280     }
2281
2282     # Are we looking for where it worked, not failed?
2283     if ($reverse_bisect && $ret >= 0) {
2284         $ret = !$ret;
2285     }
2286
2287     if ($ret > 0) {
2288         return "good";
2289     } elsif ($ret == 0) {
2290         return  "bad";
2291     } elsif ($bisect_skip) {
2292         doprint "HIT A BAD COMMIT ... SKIPPING\n";
2293         return "skip";
2294     }
2295 }
2296
2297 sub update_bisect_replay {
2298     my $tmp_log = "$tmpdir/ktest_bisect_log";
2299     run_command "git bisect log > $tmp_log" or
2300         die "can't create bisect log";
2301     return $tmp_log;
2302 }
2303
2304 sub bisect {
2305     my ($i) = @_;
2306
2307     my $result;
2308
2309     die "BISECT_GOOD[$i] not defined\n" if (!defined($bisect_good));
2310     die "BISECT_BAD[$i] not defined\n"  if (!defined($bisect_bad));
2311     die "BISECT_TYPE[$i] not defined\n" if (!defined($bisect_type));
2312
2313     my $good = $bisect_good;
2314     my $bad = $bisect_bad;
2315     my $type = $bisect_type;
2316     my $start = $bisect_start;
2317     my $replay = $bisect_replay;
2318     my $start_files = $bisect_files;
2319
2320     if (defined($start_files)) {
2321         $start_files = " -- " . $start_files;
2322     } else {
2323         $start_files = "";
2324     }
2325
2326     # convert to true sha1's
2327     $good = get_sha1($good);
2328     $bad = get_sha1($bad);
2329
2330     if (defined($bisect_reverse) && $bisect_reverse == 1) {
2331         doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
2332         $reverse_bisect = 1;
2333     } else {
2334         $reverse_bisect = 0;
2335     }
2336
2337     # Can't have a test without having a test to run
2338     if ($type eq "test" && !defined($run_test)) {
2339         $type = "boot";
2340     }
2341
2342     # Check if a bisect was running
2343     my $bisect_start_file = "$builddir/.git/BISECT_START";
2344
2345     my $check = $bisect_check;
2346     my $do_check = defined($check) && $check ne "0";
2347
2348     if ( -f $bisect_start_file ) {
2349         print "Bisect in progress found\n";
2350         if ($do_check) {
2351             print " If you say yes, then no checks of good or bad will be done\n";
2352         }
2353         if (defined($replay)) {
2354             print "** BISECT_REPLAY is defined in config file **";
2355             print " Ignore config option and perform new git bisect log?\n";
2356             if (read_ync " (yes, no, or cancel) ") {
2357                 $replay = update_bisect_replay;
2358                 $do_check = 0;
2359             }
2360         } elsif (read_yn "read git log and continue?") {
2361             $replay = update_bisect_replay;
2362             $do_check = 0;
2363         }
2364     }
2365
2366     if ($do_check) {
2367
2368         # get current HEAD
2369         my $head = get_sha1("HEAD");
2370
2371         if ($check ne "good") {
2372             doprint "TESTING BISECT BAD [$bad]\n";
2373             run_command "git checkout $bad" or
2374                 die "Failed to checkout $bad";
2375
2376             $result = run_bisect $type;
2377
2378             if ($result ne "bad") {
2379                 fail "Tested BISECT_BAD [$bad] and it succeeded" and return 0;
2380             }
2381         }
2382
2383         if ($check ne "bad") {
2384             doprint "TESTING BISECT GOOD [$good]\n";
2385             run_command "git checkout $good" or
2386                 die "Failed to checkout $good";
2387
2388             $result = run_bisect $type;
2389
2390             if ($result ne "good") {
2391                 fail "Tested BISECT_GOOD [$good] and it failed" and return 0;
2392             }
2393         }
2394
2395         # checkout where we started
2396         run_command "git checkout $head" or
2397             die "Failed to checkout $head";
2398     }
2399
2400     run_command "git bisect start$start_files" or
2401         dodie "could not start bisect";
2402
2403     run_command "git bisect good $good" or
2404         dodie "could not set bisect good to $good";
2405
2406     run_git_bisect "git bisect bad $bad" or
2407         dodie "could not set bisect bad to $bad";
2408
2409     if (defined($replay)) {
2410         run_command "git bisect replay $replay" or
2411             dodie "failed to run replay";
2412     }
2413
2414     if (defined($start)) {
2415         run_command "git checkout $start" or
2416             dodie "failed to checkout $start";
2417     }
2418
2419     my $test;
2420     do {
2421         $result = run_bisect $type;
2422         $test = run_git_bisect "git bisect $result";
2423     } while ($test);
2424
2425     run_command "git bisect log" or
2426         dodie "could not capture git bisect log";
2427
2428     run_command "git bisect reset" or
2429         dodie "could not reset git bisect";
2430
2431     doprint "Bad commit was [$bisect_bad_commit]\n";
2432
2433     success $i;
2434 }
2435
2436 # config_ignore holds the configs that were set (or unset) for
2437 # a good config and we will ignore these configs for the rest
2438 # of a config bisect. These configs stay as they were.
2439 my %config_ignore;
2440
2441 # config_set holds what all configs were set as.
2442 my %config_set;
2443
2444 # config_off holds the set of configs that the bad config had disabled.
2445 # We need to record them and set them in the .config when running
2446 # oldnoconfig, because oldnoconfig does not turn off new symbols, but
2447 # instead just keeps the defaults.
2448 my %config_off;
2449
2450 # config_off_tmp holds a set of configs to turn off for now
2451 my @config_off_tmp;
2452
2453 # config_list is the set of configs that are being tested
2454 my %config_list;
2455 my %null_config;
2456
2457 my %dependency;
2458
2459 sub assign_configs {
2460     my ($hash, $config) = @_;
2461
2462     open (IN, $config)
2463         or dodie "Failed to read $config";
2464
2465     while (<IN>) {
2466         if (/^((CONFIG\S*)=.*)/) {
2467             ${$hash}{$2} = $1;
2468         }
2469     }
2470
2471     close(IN);
2472 }
2473
2474 sub process_config_ignore {
2475     my ($config) = @_;
2476
2477     assign_configs \%config_ignore, $config;
2478 }
2479
2480 sub read_current_config {
2481     my ($config_ref) = @_;
2482
2483     %{$config_ref} = ();
2484     undef %{$config_ref};
2485
2486     my @key = keys %{$config_ref};
2487     if ($#key >= 0) {
2488         print "did not delete!\n";
2489         exit;
2490     }
2491     open (IN, "$output_config");
2492
2493     while (<IN>) {
2494         if (/^(CONFIG\S+)=(.*)/) {
2495             ${$config_ref}{$1} = $2;
2496         }
2497     }
2498     close(IN);
2499 }
2500
2501 sub get_dependencies {
2502     my ($config) = @_;
2503
2504     my $arr = $dependency{$config};
2505     if (!defined($arr)) {
2506         return ();
2507     }
2508
2509     my @deps = @{$arr};
2510
2511     foreach my $dep (@{$arr}) {
2512         print "ADD DEP $dep\n";
2513         @deps = (@deps, get_dependencies $dep);
2514     }
2515
2516     return @deps;
2517 }
2518
2519 sub create_config {
2520     my @configs = @_;
2521
2522     open(OUT, ">$output_config") or dodie "Can not write to $output_config";
2523
2524     foreach my $config (@configs) {
2525         print OUT "$config_set{$config}\n";
2526         my @deps = get_dependencies $config;
2527         foreach my $dep (@deps) {
2528             print OUT "$config_set{$dep}\n";
2529         }
2530     }
2531
2532     # turn off configs to keep off
2533     foreach my $config (keys %config_off) {
2534         print OUT "# $config is not set\n";
2535     }
2536
2537     # turn off configs that should be off for now
2538     foreach my $config (@config_off_tmp) {
2539         print OUT "# $config is not set\n";
2540     }
2541
2542     foreach my $config (keys %config_ignore) {
2543         print OUT "$config_ignore{$config}\n";
2544     }
2545     close(OUT);
2546
2547     make_oldconfig;
2548 }
2549
2550 sub compare_configs {
2551     my (%a, %b) = @_;
2552
2553     foreach my $item (keys %a) {
2554         if (!defined($b{$item})) {
2555             print "diff $item\n";
2556             return 1;
2557         }
2558         delete $b{$item};
2559     }
2560
2561     my @keys = keys %b;
2562     if ($#keys) {
2563         print "diff2 $keys[0]\n";
2564     }
2565     return -1 if ($#keys >= 0);
2566
2567     return 0;
2568 }
2569
2570 sub run_config_bisect_test {
2571     my ($type) = @_;
2572
2573     return run_bisect_test $type, "oldconfig";
2574 }
2575
2576 sub process_passed {
2577     my (%configs) = @_;
2578
2579     doprint "These configs had no failure: (Enabling them for further compiles)\n";
2580     # Passed! All these configs are part of a good compile.
2581     # Add them to the min options.
2582     foreach my $config (keys %configs) {
2583         if (defined($config_list{$config})) {
2584             doprint " removing $config\n";
2585             $config_ignore{$config} = $config_list{$config};
2586             delete $config_list{$config};
2587         }
2588     }
2589     doprint "config copied to $outputdir/config_good\n";
2590     run_command "cp -f $output_config $outputdir/config_good";
2591 }
2592
2593 sub process_failed {
2594     my ($config) = @_;
2595
2596     doprint "\n\n***************************************\n";
2597     doprint "Found bad config: $config\n";
2598     doprint "***************************************\n\n";
2599 }
2600
2601 sub run_config_bisect {
2602
2603     my @start_list = keys %config_list;
2604
2605     if ($#start_list < 0) {
2606         doprint "No more configs to test!!!\n";
2607         return -1;
2608     }
2609
2610     doprint "***** RUN TEST ***\n";
2611     my $type = $config_bisect_type;
2612     my $ret;
2613     my %current_config;
2614
2615     my $count = $#start_list + 1;
2616     doprint "  $count configs to test\n";
2617
2618     my $half = int($#start_list / 2);
2619
2620     do {
2621         my @tophalf = @start_list[0 .. $half];
2622
2623         # keep the bottom half off
2624         if ($half < $#start_list) {
2625             @config_off_tmp = @start_list[$half + 1 .. $#start_list];
2626         } else {
2627             @config_off_tmp = ();
2628         }
2629
2630         create_config @tophalf;
2631         read_current_config \%current_config;
2632
2633         $count = $#tophalf + 1;
2634         doprint "Testing $count configs\n";
2635         my $found = 0;
2636         # make sure we test something
2637         foreach my $config (@tophalf) {
2638             if (defined($current_config{$config})) {
2639                 logit " $config\n";
2640                 $found = 1;
2641             }
2642         }
2643         if (!$found) {
2644             # try the other half
2645             doprint "Top half produced no set configs, trying bottom half\n";
2646
2647             # keep the top half off
2648             @config_off_tmp = @tophalf;
2649             @tophalf = @start_list[$half + 1 .. $#start_list];
2650
2651             create_config @tophalf;
2652             read_current_config \%current_config;
2653             foreach my $config (@tophalf) {
2654                 if (defined($current_config{$config})) {
2655                     logit " $config\n";
2656                     $found = 1;
2657                 }
2658             }
2659             if (!$found) {
2660                 doprint "Failed: Can't make new config with current configs\n";
2661                 foreach my $config (@start_list) {
2662                     doprint "  CONFIG: $config\n";
2663                 }
2664                 return -1;
2665             }
2666             $count = $#tophalf + 1;
2667             doprint "Testing $count configs\n";
2668         }
2669
2670         $ret = run_config_bisect_test $type;
2671         if ($bisect_manual) {
2672             $ret = answer_bisect;
2673         }
2674         if ($ret) {
2675             process_passed %current_config;
2676             return 0;
2677         }
2678
2679         doprint "This config had a failure.\n";
2680         doprint "Removing these configs that were not set in this config:\n";
2681         doprint "config copied to $outputdir/config_bad\n";
2682         run_command "cp -f $output_config $outputdir/config_bad";
2683
2684         # A config exists in this group that was bad.
2685         foreach my $config (keys %config_list) {
2686             if (!defined($current_config{$config})) {
2687                 doprint " removing $config\n";
2688                 delete $config_list{$config};
2689             }
2690         }
2691
2692         @start_list = @tophalf;
2693
2694         if ($#start_list == 0) {
2695             process_failed $start_list[0];
2696             return 1;
2697         }
2698
2699         # remove half the configs we are looking at and see if
2700         # they are good.
2701         $half = int($#start_list / 2);
2702     } while ($#start_list > 0);
2703
2704     # we found a single config, try it again unless we are running manually
2705
2706     if ($bisect_manual) {
2707         process_failed $start_list[0];
2708         return 1;
2709     }
2710
2711     my @tophalf = @start_list[0 .. 0];
2712
2713     $ret = run_config_bisect_test $type;
2714     if ($ret) {
2715         process_passed %current_config;
2716         return 0;
2717     }
2718
2719     process_failed $start_list[0];
2720     return 1;
2721 }
2722
2723 sub config_bisect {
2724     my ($i) = @_;
2725
2726     my $start_config = $config_bisect;
2727
2728     my $tmpconfig = "$tmpdir/use_config";
2729
2730     if (defined($config_bisect_good)) {
2731         process_config_ignore $config_bisect_good;
2732     }
2733
2734     # Make the file with the bad config and the min config
2735     if (defined($minconfig)) {
2736         # read the min config for things to ignore
2737         run_command "cp $minconfig $tmpconfig" or
2738             dodie "failed to copy $minconfig to $tmpconfig";
2739     } else {
2740         unlink $tmpconfig;
2741     }
2742
2743     if (-f $tmpconfig) {
2744         load_force_config($tmpconfig);
2745         process_config_ignore $tmpconfig;
2746     }
2747
2748     # now process the start config
2749     run_command "cp $start_config $output_config" or
2750         dodie "failed to copy $start_config to $output_config";
2751
2752     # read directly what we want to check
2753     my %config_check;
2754     open (IN, $output_config)
2755         or dodie "failed to open $output_config";
2756
2757     while (<IN>) {
2758         if (/^((CONFIG\S*)=.*)/) {
2759             $config_check{$2} = $1;
2760         }
2761     }
2762     close(IN);
2763
2764     # Now run oldconfig with the minconfig
2765     make_oldconfig;
2766
2767     # check to see what we lost (or gained)
2768     open (IN, $output_config)
2769         or dodie "Failed to read $start_config";
2770
2771     my %removed_configs;
2772     my %added_configs;
2773
2774     while (<IN>) {
2775         if (/^((CONFIG\S*)=.*)/) {
2776             # save off all options
2777             $config_set{$2} = $1;
2778             if (defined($config_check{$2})) {
2779                 if (defined($config_ignore{$2})) {
2780                     $removed_configs{$2} = $1;
2781                 } else {
2782                     $config_list{$2} = $1;
2783                 }
2784             } elsif (!defined($config_ignore{$2})) {
2785                 $added_configs{$2} = $1;
2786                 $config_list{$2} = $1;
2787             }
2788         } elsif (/^# ((CONFIG\S*).*)/) {
2789             # Keep these configs disabled
2790             $config_set{$2} = $1;
2791             $config_off{$2} = $1;
2792         }
2793     }
2794     close(IN);
2795
2796     my @confs = keys %removed_configs;
2797     if ($#confs >= 0) {
2798         doprint "Configs overridden by default configs and removed from check:\n";
2799         foreach my $config (@confs) {
2800             doprint " $config\n";
2801         }
2802     }
2803     @confs = keys %added_configs;
2804     if ($#confs >= 0) {
2805         doprint "Configs appearing in make oldconfig and added:\n";
2806         foreach my $config (@confs) {
2807             doprint " $config\n";
2808         }
2809     }
2810
2811     my %config_test;
2812     my $once = 0;
2813
2814     @config_off_tmp = ();
2815
2816     # Sometimes kconfig does weird things. We must make sure
2817     # that the config we autocreate has everything we need
2818     # to test, otherwise we may miss testing configs, or
2819     # may not be able to create a new config.
2820     # Here we create a config with everything set.
2821     create_config (keys %config_list);
2822     read_current_config \%config_test;
2823     foreach my $config (keys %config_list) {
2824         if (!defined($config_test{$config})) {
2825             if (!$once) {
2826                 $once = 1;
2827                 doprint "Configs not produced by kconfig (will not be checked):\n";
2828             }
2829             doprint "  $config\n";
2830             delete $config_list{$config};
2831         }
2832     }
2833     my $ret;
2834
2835     if (defined($config_bisect_check) && $config_bisect_check) {
2836         doprint " Checking to make sure bad config with min config fails\n";
2837         create_config keys %config_list;
2838         $ret = run_config_bisect_test $config_bisect_type;
2839         if ($ret) {
2840             doprint " FAILED! Bad config with min config boots fine\n";
2841             return -1;
2842         }
2843         doprint " Bad config with min config fails as expected\n";
2844     }
2845
2846     do {
2847         $ret = run_config_bisect;
2848     } while (!$ret);
2849
2850     return $ret if ($ret < 0);
2851
2852     success $i;
2853 }
2854
2855 sub patchcheck_reboot {
2856     doprint "Reboot and sleep $patchcheck_sleep_time seconds\n";
2857     reboot_to_good $patchcheck_sleep_time;
2858 }
2859
2860 sub patchcheck {
2861     my ($i) = @_;
2862
2863     die "PATCHCHECK_START[$i] not defined\n"
2864         if (!defined($patchcheck_start));
2865     die "PATCHCHECK_TYPE[$i] not defined\n"
2866         if (!defined($patchcheck_type));
2867
2868     my $start = $patchcheck_start;
2869
2870     my $end = "HEAD";
2871     if (defined($patchcheck_end)) {
2872         $end = $patchcheck_end;
2873     }
2874
2875     # Get the true sha1's since we can use things like HEAD~3
2876     $start = get_sha1($start);
2877     $end = get_sha1($end);
2878
2879     my $type = $patchcheck_type;
2880
2881     # Can't have a test without having a test to run
2882     if ($type eq "test" && !defined($run_test)) {
2883         $type = "boot";
2884     }
2885
2886     open (IN, "git log --pretty=oneline $end|") or
2887         dodie "could not get git list";
2888
2889     my @list;
2890
2891     while (<IN>) {
2892         chomp;
2893         $list[$#list+1] = $_;
2894         last if (/^$start/);
2895     }
2896     close(IN);
2897
2898     if ($list[$#list] !~ /^$start/) {
2899         fail "SHA1 $start not found";
2900     }
2901
2902     # go backwards in the list
2903     @list = reverse @list;
2904
2905     my $save_clean = $noclean;
2906     my %ignored_warnings;
2907
2908     if (defined($ignore_warnings)) {
2909         foreach my $sha1 (split /\s+/, $ignore_warnings) {
2910             $ignored_warnings{$sha1} = 1;
2911         }
2912     }
2913
2914     $in_patchcheck = 1;
2915     foreach my $item (@list) {
2916         my $sha1 = $item;
2917         $sha1 =~ s/^([[:xdigit:]]+).*/$1/;
2918
2919         doprint "\nProcessing commit $item\n\n";
2920
2921         run_command "git checkout $sha1" or
2922             die "Failed to checkout $sha1";
2923
2924         # only clean on the first and last patch
2925         if ($item eq $list[0] ||
2926             $item eq $list[$#list]) {
2927             $noclean = $save_clean;
2928         } else {
2929             $noclean = 1;
2930         }
2931
2932         if (defined($minconfig)) {
2933             build "useconfig:$minconfig" or return 0;
2934         } else {
2935             # ?? no config to use?
2936             build "oldconfig" or return 0;
2937         }
2938
2939
2940         if (!defined($ignored_warnings{$sha1})) {
2941             check_buildlog $sha1 or return 0;
2942         }
2943
2944         next if ($type eq "build");
2945
2946         my $failed = 0;
2947
2948         start_monitor_and_boot or $failed = 1;
2949
2950         if (!$failed && $type ne "boot"){
2951             do_run_test or $failed = 1;
2952         }
2953         end_monitor;
2954         return 0 if ($failed);
2955
2956         patchcheck_reboot;
2957
2958     }
2959     $in_patchcheck = 0;
2960     success $i;
2961
2962     return 1;
2963 }
2964
2965 my %depends;
2966 my %depcount;
2967 my $iflevel = 0;
2968 my @ifdeps;
2969
2970 # prevent recursion
2971 my %read_kconfigs;
2972
2973 sub add_dep {
2974     # $config depends on $dep
2975     my ($config, $dep) = @_;
2976
2977     if (defined($depends{$config})) {
2978         $depends{$config} .= " " . $dep;
2979     } else {
2980         $depends{$config} = $dep;
2981     }
2982
2983     # record the number of configs depending on $dep
2984     if (defined $depcount{$dep}) {
2985         $depcount{$dep}++;
2986     } else {
2987         $depcount{$dep} = 1;
2988     }
2989 }
2990
2991 # taken from streamline_config.pl
2992 sub read_kconfig {
2993     my ($kconfig) = @_;
2994
2995     my $state = "NONE";
2996     my $config;
2997     my @kconfigs;
2998
2999     my $cont = 0;
3000     my $line;
3001
3002
3003     if (! -f $kconfig) {
3004         doprint "file $kconfig does not exist, skipping\n";
3005         return;
3006     }
3007
3008     open(KIN, "$kconfig")
3009         or die "Can't open $kconfig";
3010     while (<KIN>) {
3011         chomp;
3012
3013         # Make sure that lines ending with \ continue
3014         if ($cont) {
3015             $_ = $line . " " . $_;
3016         }
3017
3018         if (s/\\$//) {
3019             $cont = 1;
3020             $line = $_;
3021             next;
3022         }
3023
3024         $cont = 0;
3025
3026         # collect any Kconfig sources
3027         if (/^source\s*"(.*)"/) {
3028             $kconfigs[$#kconfigs+1] = $1;
3029         }
3030
3031         # configs found
3032         if (/^\s*(menu)?config\s+(\S+)\s*$/) {
3033             $state = "NEW";
3034             $config = $2;
3035
3036             for (my $i = 0; $i < $iflevel; $i++) {
3037                 add_dep $config, $ifdeps[$i];
3038             }
3039
3040         # collect the depends for the config
3041         } elsif ($state eq "NEW" && /^\s*depends\s+on\s+(.*)$/) {
3042
3043             add_dep $config, $1;
3044
3045         # Get the configs that select this config
3046         } elsif ($state eq "NEW" && /^\s*select\s+(\S+)/) {
3047
3048             # selected by depends on config
3049             add_dep $1, $config;
3050
3051         # Check for if statements
3052         } elsif (/^if\s+(.*\S)\s*$/) {
3053             my $deps = $1;
3054             # remove beginning and ending non text
3055             $deps =~ s/^[^a-zA-Z0-9_]*//;
3056             $deps =~ s/[^a-zA-Z0-9_]*$//;
3057
3058             my @deps = split /[^a-zA-Z0-9_]+/, $deps;
3059
3060             $ifdeps[$iflevel++] = join ':', @deps;
3061
3062         } elsif (/^endif/) {
3063
3064             $iflevel-- if ($iflevel);
3065
3066         # stop on "help"
3067         } elsif (/^\s*help\s*$/) {
3068             $state = "NONE";
3069         }
3070     }
3071     close(KIN);
3072
3073     # read in any configs that were found.
3074     foreach $kconfig (@kconfigs) {
3075         if (!defined($read_kconfigs{$kconfig})) {
3076             $read_kconfigs{$kconfig} = 1;
3077             read_kconfig("$builddir/$kconfig");
3078         }
3079     }
3080 }
3081
3082 sub read_depends {
3083     # find out which arch this is by the kconfig file
3084     open (IN, $output_config)
3085         or dodie "Failed to read $output_config";
3086     my $arch;
3087     while (<IN>) {
3088         if (m,Linux/(\S+)\s+\S+\s+Kernel Configuration,) {
3089             $arch = $1;
3090             last;
3091         }
3092     }
3093     close IN;
3094
3095     if (!defined($arch)) {
3096         doprint "Could not find arch from config file\n";
3097         doprint "no dependencies used\n";
3098         return;
3099     }
3100
3101     # arch is really the subarch, we need to know
3102     # what directory to look at.
3103     if ($arch eq "i386" || $arch eq "x86_64") {
3104         $arch = "x86";
3105     } elsif ($arch =~ /^tile/) {
3106         $arch = "tile";
3107     }
3108
3109     my $kconfig = "$builddir/arch/$arch/Kconfig";
3110
3111     if (! -f $kconfig && $arch =~ /\d$/) {
3112         my $orig = $arch;
3113         # some subarchs have numbers, truncate them
3114         $arch =~ s/\d*$//;
3115         $kconfig = "$builddir/arch/$arch/Kconfig";
3116         if (! -f $kconfig) {
3117             doprint "No idea what arch dir $orig is for\n";
3118             doprint "no dependencies used\n";
3119             return;
3120         }
3121     }
3122
3123     read_kconfig($kconfig);
3124 }
3125
3126 sub read_config_list {
3127     my ($config) = @_;
3128
3129     open (IN, $config)
3130         or dodie "Failed to read $config";
3131
3132     while (<IN>) {
3133         if (/^((CONFIG\S*)=.*)/) {
3134             if (!defined($config_ignore{$2})) {
3135                 $config_list{$2} = $1;
3136             }
3137         }
3138     }
3139
3140     close(IN);
3141 }
3142
3143 sub read_output_config {
3144     my ($config) = @_;
3145
3146     assign_configs \%config_ignore, $config;
3147 }
3148
3149 sub make_new_config {
3150     my @configs = @_;
3151
3152     open (OUT, ">$output_config")
3153         or dodie "Failed to write $output_config";
3154
3155     foreach my $config (@configs) {
3156         print OUT "$config\n";
3157     }
3158     close OUT;
3159 }
3160
3161 sub chomp_config {
3162     my ($config) = @_;
3163
3164     $config =~ s/CONFIG_//;
3165
3166     return $config;
3167 }
3168
3169 sub get_depends {
3170     my ($dep) = @_;
3171
3172     my $kconfig = chomp_config $dep;
3173
3174     $dep = $depends{"$kconfig"};
3175
3176     # the dep string we have saves the dependencies as they
3177     # were found, including expressions like ! && ||. We
3178     # want to split this out into just an array of configs.
3179
3180     my $valid = "A-Za-z_0-9";
3181
3182     my @configs;
3183
3184     while ($dep =~ /[$valid]/) {
3185
3186         if ($dep =~ /^[^$valid]*([$valid]+)/) {
3187             my $conf = "CONFIG_" . $1;
3188
3189             $configs[$#configs + 1] = $conf;
3190
3191             $dep =~ s/^[^$valid]*[$valid]+//;
3192         } else {
3193             die "this should never happen";
3194         }
3195     }
3196
3197     return @configs;
3198 }
3199
3200 my %min_configs;
3201 my %keep_configs;
3202 my %save_configs;
3203 my %processed_configs;
3204 my %nochange_config;
3205
3206 sub test_this_config {
3207     my ($config) = @_;
3208
3209     my $found;
3210
3211     # if we already processed this config, skip it
3212     if (defined($processed_configs{$config})) {
3213         return undef;
3214     }
3215     $processed_configs{$config} = 1;
3216
3217     # if this config failed during this round, skip it
3218     if (defined($nochange_config{$config})) {
3219         return undef;
3220     }
3221
3222     my $kconfig = chomp_config $config;
3223
3224     # Test dependencies first
3225     if (defined($depends{"$kconfig"})) {
3226         my @parents = get_depends $config;
3227         foreach my $parent (@parents) {
3228             # if the parent is in the min config, check it first
3229             next if (!defined($min_configs{$parent}));
3230             $found = test_this_config($parent);
3231             if (defined($found)) {
3232                 return $found;
3233             }
3234         }
3235     }
3236
3237     # Remove this config from the list of configs
3238     # do a make oldnoconfig and then read the resulting
3239     # .config to make sure it is missing the config that
3240     # we had before
3241     my %configs = %min_configs;
3242     delete $configs{$config};
3243     make_new_config ((values %configs), (values %keep_configs));
3244     make_oldconfig;
3245     undef %configs;
3246     assign_configs \%configs, $output_config;
3247
3248     return $config if (!defined($configs{$config}));
3249
3250     doprint "disabling config $config did not change .config\n";
3251
3252     $nochange_config{$config} = 1;
3253
3254     return undef;
3255 }
3256
3257 sub make_min_config {
3258     my ($i) = @_;
3259
3260     my $type = $minconfig_type;
3261     if ($type ne "boot" && $type ne "test") {
3262         fail "Invalid MIN_CONFIG_TYPE '$minconfig_type'\n" .
3263             " make_min_config works only with 'boot' and 'test'\n" and return;
3264     }
3265
3266     if (!defined($output_minconfig)) {
3267         fail "OUTPUT_MIN_CONFIG not defined" and return;
3268     }
3269
3270     # If output_minconfig exists, and the start_minconfig
3271     # came from min_config, than ask if we should use
3272     # that instead.
3273     if (-f $output_minconfig && !$start_minconfig_defined) {
3274         print "$output_minconfig exists\n";
3275         if (!defined($use_output_minconfig)) {
3276             if (read_yn " Use it as minconfig?") {
3277                 $start_minconfig = $output_minconfig;
3278             }
3279         } elsif ($use_output_minconfig > 0) {
3280             doprint "Using $output_minconfig as MIN_CONFIG\n";
3281             $start_minconfig = $output_minconfig;
3282         } else {
3283             doprint "Set to still use MIN_CONFIG as starting point\n";
3284         }
3285     }
3286
3287     if (!defined($start_minconfig)) {
3288         fail "START_MIN_CONFIG or MIN_CONFIG not defined" and return;
3289     }
3290
3291     my $temp_config = "$tmpdir/temp_config";
3292
3293     # First things first. We build an allnoconfig to find
3294     # out what the defaults are that we can't touch.
3295     # Some are selections, but we really can't handle selections.
3296
3297     my $save_minconfig = $minconfig;
3298     undef $minconfig;
3299
3300     run_command "$make allnoconfig" or return 0;
3301
3302     read_depends;
3303
3304     process_config_ignore $output_config;
3305
3306     undef %save_configs;
3307     undef %min_configs;
3308
3309     if (defined($ignore_config)) {
3310         # make sure the file exists
3311         `touch $ignore_config`;
3312         assign_configs \%save_configs, $ignore_config;
3313     }
3314
3315     %keep_configs = %save_configs;
3316
3317     doprint "Load initial configs from $start_minconfig\n";
3318
3319     # Look at the current min configs, and save off all the
3320     # ones that were set via the allnoconfig
3321     assign_configs \%min_configs, $start_minconfig;
3322
3323     my @config_keys = keys %min_configs;
3324
3325     # All configs need a depcount
3326     foreach my $config (@config_keys) {
3327         my $kconfig = chomp_config $config;
3328         if (!defined $depcount{$kconfig}) {
3329                 $depcount{$kconfig} = 0;
3330         }
3331     }
3332
3333     # Remove anything that was set by the make allnoconfig
3334     # we shouldn't need them as they get set for us anyway.
3335     foreach my $config (@config_keys) {
3336         # Remove anything in the ignore_config
3337         if (defined($keep_configs{$config})) {
3338             my $file = $ignore_config;
3339             $file =~ s,.*/(.*?)$,$1,;
3340             doprint "$config set by $file ... ignored\n";
3341             delete $min_configs{$config};
3342             next;
3343         }
3344         # But make sure the settings are the same. If a min config
3345         # sets a selection, we do not want to get rid of it if
3346         # it is not the same as what we have. Just move it into
3347         # the keep configs.
3348         if (defined($config_ignore{$config})) {
3349             if ($config_ignore{$config} ne $min_configs{$config}) {
3350                 doprint "$config is in allnoconfig as '$config_ignore{$config}'";
3351                 doprint " but it is '$min_configs{$config}' in minconfig .. keeping\n";
3352                 $keep_configs{$config} = $min_configs{$config};
3353             } else {
3354                 doprint "$config set by allnoconfig ... ignored\n";
3355             }
3356             delete $min_configs{$config};
3357         }
3358     }
3359
3360     my $done = 0;
3361     my $take_two = 0;
3362
3363     while (!$done) {
3364
3365         my $config;
3366         my $found;
3367
3368         # Now disable each config one by one and do a make oldconfig
3369         # till we find a config that changes our list.
3370
3371         my @test_configs = keys %min_configs;
3372
3373         # Sort keys by who is most dependent on
3374         @test_configs = sort  { $depcount{chomp_config($b)} <=> $depcount{chomp_config($a)} }
3375                           @test_configs ;
3376
3377         # Put configs that did not modify the config at the end.
3378         my $reset = 1;
3379         for (my $i = 0; $i < $#test_configs; $i++) {
3380             if (!defined($nochange_config{$test_configs[0]})) {
3381                 $reset = 0;
3382                 last;
3383             }
3384             # This config didn't change the .config last time.
3385             # Place it at the end
3386             my $config = shift @test_configs;
3387             push @test_configs, $config;
3388         }
3389
3390         # if every test config has failed to modify the .config file
3391         # in the past, then reset and start over.
3392         if ($reset) {
3393             undef %nochange_config;
3394         }
3395
3396         undef %processed_configs;
3397
3398         foreach my $config (@test_configs) {
3399
3400             $found = test_this_config $config;
3401
3402             last if (defined($found));
3403
3404             # oh well, try another config
3405         }
3406
3407         if (!defined($found)) {
3408             # we could have failed due to the nochange_config hash
3409             # reset and try again
3410             if (!$take_two) {
3411                 undef %nochange_config;
3412                 $take_two = 1;
3413                 next;
3414             }
3415             doprint "No more configs found that we can disable\n";
3416             $done = 1;
3417             last;
3418         }
3419         $take_two = 0;
3420
3421         $config = $found;
3422
3423         doprint "Test with $config disabled\n";
3424
3425         # set in_bisect to keep build and monitor from dieing
3426         $in_bisect = 1;
3427
3428         my $failed = 0;
3429         build "oldconfig" or $failed = 1;
3430         if (!$failed) {
3431                 start_monitor_and_boot or $failed = 1;
3432
3433                 if ($type eq "test" && !$failed) {
3434                     do_run_test or $failed = 1;
3435                 }
3436
3437                 end_monitor;
3438         }
3439
3440         $in_bisect = 0;
3441
3442         if ($failed) {
3443             doprint "$min_configs{$config} is needed to boot the box... keeping\n";
3444             # this config is needed, add it to the ignore list.
3445             $keep_configs{$config} = $min_configs{$config};
3446             $save_configs{$config} = $min_configs{$config};
3447             delete $min_configs{$config};
3448
3449             # update new ignore configs
3450             if (defined($ignore_config)) {
3451                 open (OUT, ">$temp_config")
3452                     or die "Can't write to $temp_config";
3453                 foreach my $config (keys %save_configs) {
3454                     print OUT "$save_configs{$config}\n";
3455                 }
3456                 close OUT;
3457                 run_command "mv $temp_config $ignore_config" or
3458                     dodie "failed to copy update to $ignore_config";
3459             }
3460
3461         } else {
3462             # We booted without this config, remove it from the minconfigs.
3463             doprint "$config is not needed, disabling\n";
3464
3465             delete $min_configs{$config};
3466
3467             # Also disable anything that is not enabled in this config
3468             my %configs;
3469             assign_configs \%configs, $output_config;
3470             my @config_keys = keys %min_configs;
3471             foreach my $config (@config_keys) {
3472                 if (!defined($configs{$config})) {
3473                     doprint "$config is not set, disabling\n";
3474                     delete $min_configs{$config};
3475                 }
3476             }
3477
3478             # Save off all the current mandidory configs
3479             open (OUT, ">$temp_config")
3480                 or die "Can't write to $temp_config";
3481             foreach my $config (keys %keep_configs) {
3482                 print OUT "$keep_configs{$config}\n";
3483             }
3484             foreach my $config (keys %min_configs) {
3485                 print OUT "$min_configs{$config}\n";
3486             }
3487             close OUT;
3488
3489             run_command "mv $temp_config $output_minconfig" or
3490                 dodie "failed to copy update to $output_minconfig";
3491         }
3492
3493         doprint "Reboot and wait $sleep_time seconds\n";
3494         reboot_to_good $sleep_time;
3495     }
3496
3497     success $i;
3498     return 1;
3499 }
3500
3501 $#ARGV < 1 or die "ktest.pl version: $VERSION\n   usage: ktest.pl config-file\n";
3502
3503 if ($#ARGV == 0) {
3504     $ktest_config = $ARGV[0];
3505     if (! -f $ktest_config) {
3506         print "$ktest_config does not exist.\n";
3507         if (!read_yn "Create it?") {
3508             exit 0;
3509         }
3510     }
3511 } else {
3512     $ktest_config = "ktest.conf";
3513 }
3514
3515 if (! -f $ktest_config) {
3516     $newconfig = 1;
3517     get_test_case;
3518     open(OUT, ">$ktest_config") or die "Can not create $ktest_config";
3519     print OUT << "EOF"
3520 # Generated by ktest.pl
3521 #
3522
3523 # PWD is a ktest.pl variable that will result in the process working
3524 # directory that ktest.pl is executed in.
3525
3526 # THIS_DIR is automatically assigned the PWD of the path that generated
3527 # the config file. It is best to use this variable when assigning other
3528 # directory paths within this directory. This allows you to easily
3529 # move the test cases to other locations or to other machines.
3530 #
3531 THIS_DIR := $variable{"PWD"}
3532
3533 # Define each test with TEST_START
3534 # The config options below it will override the defaults
3535 TEST_START
3536 TEST_TYPE = $default{"TEST_TYPE"}
3537
3538 DEFAULTS
3539 EOF
3540 ;
3541     close(OUT);
3542 }
3543 read_config $ktest_config;
3544
3545 if (defined($opt{"LOG_FILE"})) {
3546     $opt{"LOG_FILE"} = eval_option($opt{"LOG_FILE"}, -1);
3547 }
3548
3549 # Append any configs entered in manually to the config file.
3550 my @new_configs = keys %entered_configs;
3551 if ($#new_configs >= 0) {
3552     print "\nAppending entered in configs to $ktest_config\n";
3553     open(OUT, ">>$ktest_config") or die "Can not append to $ktest_config";
3554     foreach my $config (@new_configs) {
3555         print OUT "$config = $entered_configs{$config}\n";
3556         $opt{$config} = process_variables($entered_configs{$config});
3557     }
3558 }
3559
3560 if ($opt{"CLEAR_LOG"} && defined($opt{"LOG_FILE"})) {
3561     unlink $opt{"LOG_FILE"};
3562 }
3563
3564 doprint "\n\nSTARTING AUTOMATED TESTS\n\n";
3565
3566 for (my $i = 0, my $repeat = 1; $i <= $opt{"NUM_TESTS"}; $i += $repeat) {
3567
3568     if (!$i) {
3569         doprint "DEFAULT OPTIONS:\n";
3570     } else {
3571         doprint "\nTEST $i OPTIONS";
3572         if (defined($repeat_tests{$i})) {
3573             $repeat = $repeat_tests{$i};
3574             doprint " ITERATE $repeat";
3575         }
3576         doprint "\n";
3577     }
3578
3579     foreach my $option (sort keys %opt) {
3580
3581         if ($option =~ /\[(\d+)\]$/) {
3582             next if ($i != $1);
3583         } else {
3584             next if ($i);
3585         }
3586
3587         doprint "$option = $opt{$option}\n";
3588     }
3589 }
3590
3591 sub __set_test_option {
3592     my ($name, $i) = @_;
3593
3594     my $option = "$name\[$i\]";
3595
3596     if (defined($opt{$option})) {
3597         return $opt{$option};
3598     }
3599
3600     foreach my $test (keys %repeat_tests) {
3601         if ($i >= $test &&
3602             $i < $test + $repeat_tests{$test}) {
3603             $option = "$name\[$test\]";
3604             if (defined($opt{$option})) {
3605                 return $opt{$option};
3606             }
3607         }
3608     }
3609
3610     if (defined($opt{$name})) {
3611         return $opt{$name};
3612     }
3613
3614     return undef;
3615 }
3616
3617 sub set_test_option {
3618     my ($name, $i) = @_;
3619
3620     my $option = __set_test_option($name, $i);
3621     return $option if (!defined($option));
3622
3623     return eval_option($option, $i);
3624 }
3625
3626 # First we need to do is the builds
3627 for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
3628
3629     # Do not reboot on failing test options
3630     $no_reboot = 1;
3631     $reboot_success = 0;
3632
3633     $have_version = 0;
3634
3635     $iteration = $i;
3636
3637     undef %force_config;
3638
3639     my $makecmd = set_test_option("MAKE_CMD", $i);
3640
3641     # Load all the options into their mapped variable names
3642     foreach my $opt (keys %option_map) {
3643         ${$option_map{$opt}} = set_test_option($opt, $i);
3644     }
3645
3646     $start_minconfig_defined = 1;
3647
3648     # The first test may override the PRE_KTEST option
3649     if (defined($pre_ktest) && $i == 1) {
3650         doprint "\n";
3651         run_command $pre_ktest;
3652     }
3653
3654     # Any test can override the POST_KTEST option
3655     # The last test takes precedence.
3656     if (defined($post_ktest)) {
3657         $final_post_ktest = $post_ktest;
3658     }
3659
3660     if (!defined($start_minconfig)) {
3661         $start_minconfig_defined = 0;
3662         $start_minconfig = $minconfig;
3663     }
3664
3665     chdir $builddir || die "can't change directory to $builddir";
3666
3667     foreach my $dir ($tmpdir, $outputdir) {
3668         if (!-d $dir) {
3669             mkpath($dir) or
3670                 die "can't create $dir";
3671         }
3672     }
3673
3674     $ENV{"SSH_USER"} = $ssh_user;
3675     $ENV{"MACHINE"} = $machine;
3676
3677     $buildlog = "$tmpdir/buildlog-$machine";
3678     $testlog = "$tmpdir/testlog-$machine";
3679     $dmesg = "$tmpdir/dmesg-$machine";
3680     $make = "$makecmd O=$outputdir";
3681     $output_config = "$outputdir/.config";
3682
3683     if (!$buildonly) {
3684         $target = "$ssh_user\@$machine";
3685         if ($reboot_type eq "grub") {
3686             dodie "GRUB_MENU not defined" if (!defined($grub_menu));
3687         }
3688     }
3689
3690     my $run_type = $build_type;
3691     if ($test_type eq "patchcheck") {
3692         $run_type = $patchcheck_type;
3693     } elsif ($test_type eq "bisect") {
3694         $run_type = $bisect_type;
3695     } elsif ($test_type eq "config_bisect") {
3696         $run_type = $config_bisect_type;
3697     }
3698
3699     if ($test_type eq "make_min_config") {
3700         $run_type = "";
3701     }
3702
3703     # mistake in config file?
3704     if (!defined($run_type)) {
3705         $run_type = "ERROR";
3706     }
3707
3708     my $installme = "";
3709     $installme = " no_install" if ($no_install);
3710
3711     doprint "\n\n";
3712     doprint "RUNNING TEST $i of $opt{NUM_TESTS} with option $test_type $run_type$installme\n\n";
3713
3714     if (defined($pre_test)) {
3715         run_command $pre_test;
3716     }
3717
3718     unlink $dmesg;
3719     unlink $buildlog;
3720     unlink $testlog;
3721
3722     if (defined($addconfig)) {
3723         my $min = $minconfig;
3724         if (!defined($minconfig)) {
3725             $min = "";
3726         }
3727         run_command "cat $addconfig $min > $tmpdir/add_config" or
3728             dodie "Failed to create temp config";
3729         $minconfig = "$tmpdir/add_config";
3730     }
3731
3732     if (defined($checkout)) {
3733         run_command "git checkout $checkout" or
3734             die "failed to checkout $checkout";
3735     }
3736
3737     $no_reboot = 0;
3738
3739     # A test may opt to not reboot the box
3740     if ($reboot_on_success) {
3741         $reboot_success = 1;
3742     }
3743
3744     if ($test_type eq "bisect") {
3745         bisect $i;
3746         next;
3747     } elsif ($test_type eq "config_bisect") {
3748         config_bisect $i;
3749         next;
3750     } elsif ($test_type eq "patchcheck") {
3751         patchcheck $i;
3752         next;
3753     } elsif ($test_type eq "make_min_config") {
3754         make_min_config $i;
3755         next;
3756     }
3757
3758     if ($build_type ne "nobuild") {
3759         build $build_type or next;
3760     }
3761
3762     if ($test_type eq "install") {
3763         get_version;
3764         install;
3765         success $i;
3766         next;
3767     }
3768
3769     if ($test_type ne "build") {
3770         my $failed = 0;
3771         start_monitor_and_boot or $failed = 1;
3772
3773         if (!$failed && $test_type ne "boot" && defined($run_test)) {
3774             do_run_test or $failed = 1;
3775         }
3776         end_monitor;
3777         next if ($failed);
3778     }
3779
3780     success $i;
3781 }
3782
3783 if (defined($final_post_ktest)) {
3784     run_command $final_post_ktest;
3785 }
3786
3787 if ($opt{"POWEROFF_ON_SUCCESS"}) {
3788     halt;
3789 } elsif ($opt{"REBOOT_ON_SUCCESS"} && !do_not_reboot && $reboot_success) {
3790     reboot_to_good;
3791 } elsif (defined($switch_to_good)) {
3792     # still need to get to the good kernel
3793     run_command $switch_to_good;
3794 }
3795
3796
3797 doprint "\n    $successes of $opt{NUM_TESTS} tests were successful\n\n";
3798
3799 exit 0;