tools/testing/ktest/ktest.pl

   1 #!/usr/bin/perl -w
   2 #
   3 # Copywrite 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
   4 # Licensed under the terms of the GNU GPL License version 2
   5 #
   6
   7 use strict;
   8 use IPC::Open2;
   9 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
  10 use File::Path qw(mkpath);
  11 use File::Copy qw(cp);
  12 use FileHandle;
  13
  14 my $VERSION = "0.2";
  15
  16 $| = 1;
  17
  18 my %opt;
  19 my %repeat_tests;
  20 my %repeats;
  21 my %default;
  22
  23 #default opts
  24 $default{"NUM_TESTS"}           = 1;
  25 $default{"REBOOT_TYPE"}         = "grub";
  26 $default{"TEST_TYPE"}           = "test";
  27 $default{"BUILD_TYPE"}          = "randconfig";
  28 $default{"MAKE_CMD"}            = "make";
  29 $default{"TIMEOUT"}             = 120;
  30 $default{"TMP_DIR"}             = "/tmp/ktest";
  31 $default{"SLEEP_TIME"}          = 60;   # sleep time between tests
  32 $default{"BUILD_NOCLEAN"}       = 0;
  33 $default{"REBOOT_ON_ERROR"}     = 0;
  34 $default{"POWEROFF_ON_ERROR"}   = 0;
  35 $default{"REBOOT_ON_SUCCESS"}   = 1;
  36 $default{"POWEROFF_ON_SUCCESS"} = 0;
  37 $default{"BUILD_OPTIONS"}       = "";
  38 $default{"BISECT_SLEEP_TIME"}   = 60;   # sleep time between bisects
  39 $default{"CLEAR_LOG"}           = 0;
  40 $default{"BISECT_MANUAL"}       = 0;
  41 $default{"SUCCESS_LINE"}        = "login:";
  42 $default{"BOOTED_TIMEOUT"}      = 1;
  43 $default{"DIE_ON_FAILURE"}      = 1;
  44 $default{"SSH_EXEC"}            = "ssh \$SSH_USER\@\$MACHINE \$SSH_COMMAND";
  45 $default{"SCP_TO_TARGET"}       = "scp \$SRC_FILE \$SSH_USER\@\$MACHINE:\$DST_FILE";
  46 $default{"REBOOT"}              = "ssh \$SSH_USER\@\$MACHINE reboot";
  47 $default{"STOP_AFTER_SUCCESS"}  = 10;
  48 $default{"STOP_AFTER_FAILURE"}  = 60;
  49 $default{"LOCALVERSION"}        = "-test";
  50
  51 my $ktest_config;
  52 my $version;
  53 my $machine;
  54 my $ssh_user;
  55 my $tmpdir;
  56 my $builddir;
  57 my $outputdir;
  58 my $output_config;
  59 my $test_type;
  60 my $build_type;
  61 my $build_options;
  62 my $reboot_type;
  63 my $reboot_script;
  64 my $power_cycle;
  65 my $reboot;
  66 my $reboot_on_error;
  67 my $poweroff_on_error;
  68 my $die_on_failure;
  69 my $powercycle_after_reboot;
  70 my $poweroff_after_halt;
  71 my $ssh_exec;
  72 my $scp_to_target;
  73 my $power_off;
  74 my $grub_menu;
  75 my $grub_number;
  76 my $target;
  77 my $make;
  78 my $post_install;
  79 my $noclean;
  80 my $minconfig;
  81 my $addconfig;
  82 my $in_bisect = 0;
  83 my $bisect_bad = "";
  84 my $reverse_bisect;
  85 my $bisect_manual;
  86 my $in_patchcheck = 0;
  87 my $run_test;
  88 my $redirect;
  89 my $buildlog;
  90 my $dmesg;
  91 my $monitor_fp;
  92 my $monitor_pid;
  93 my $monitor_cnt = 0;
  94 my $sleep_time;
  95 my $bisect_sleep_time;
  96 my $store_failures;
  97 my $timeout;
  98 my $booted_timeout;
  99 my $console;
 100 my $success_line;
 101 my $stop_after_success;
 102 my $stop_after_failure;
 103 my $build_target;
 104 my $target_image;
 105 my $localversion;
 106 my $iteration = 0;
 107 my $successes = 0;
 108
 109 my %entered_configs;
 110 my %config_help;
 111
 112 $config_help{"MACHINE"} = << "EOF"
 113  The machine hostname that you will test.
 114 EOF
 115     ;
 116 $config_help{"SSH_USER"} = << "EOF"
 117  The box is expected to have ssh on normal bootup, provide the user
 118   (most likely root, since you need privileged operations)
 119 EOF
 120     ;
 121 $config_help{"BUILD_DIR"} = << "EOF"
 122  The directory that contains the Linux source code (full path).
 123 EOF
 124     ;
 125 $config_help{"OUTPUT_DIR"} = << "EOF"
 126  The directory that the objects will be built (full path).
 127  (can not be same as BUILD_DIR)
 128 EOF
 129     ;
 130 $config_help{"BUILD_TARGET"} = << "EOF"
 131  The location of the compiled file to copy to the target.
 132  (relative to OUTPUT_DIR)
 133 EOF
 134     ;
 135 $config_help{"TARGET_IMAGE"} = << "EOF"
 136  The place to put your image on the test machine.
 137 EOF
 138     ;
 139 $config_help{"POWER_CYCLE"} = << "EOF"
 140  A script or command to reboot the box.
 141
 142  Here is a digital loggers power switch example
 143  POWER_CYCLE = wget --no-proxy -O /dev/null -q  --auth-no-challenge 'http://admin:admin\@power/outlet?5=CCL'
 144
 145  Here is an example to reboot a virtual box on the current host
 146  with the name "Guest".
 147  POWER_CYCLE = virsh destroy Guest; sleep 5; virsh start Guest
 148 EOF
 149     ;
 150 $config_help{"CONSOLE"} = << "EOF"
 151  The script or command that reads the console
 152
 153   If you use ttywatch server, something like the following would work.
 154 CONSOLE = nc -d localhost 3001
 155
 156  For a virtual machine with guest name "Guest".
 157 CONSOLE =  virsh console Guest
 158 EOF
 159     ;
 160 $config_help{"LOCALVERSION"} = << "EOF"
 161  Required version ending to differentiate the test
 162  from other linux builds on the system.
 163 EOF
 164     ;
 165 $config_help{"REBOOT_TYPE"} = << "EOF"
 166  Way to reboot the box to the test kernel.
 167  Only valid options so far are "grub" and "script".
 168
 169  If you specify grub, it will assume grub version 1
 170  and will search in /boot/grub/menu.lst for the title \$GRUB_MENU
 171  and select that target to reboot to the kernel. If this is not
 172  your setup, then specify "script" and have a command or script
 173  specified in REBOOT_SCRIPT to boot to the target.
 174
 175  The entry in /boot/grub/menu.lst must be entered in manually.
 176  The test will not modify that file.
 177 EOF
 178     ;
 179 $config_help{"GRUB_MENU"} = << "EOF"
 180  The grub title name for the test kernel to boot
 181  (Only mandatory if REBOOT_TYPE = grub)
 182
 183  Note, ktest.pl will not update the grub menu.lst, you need to
 184  manually add an option for the test. ktest.pl will search
 185  the grub menu.lst for this option to find what kernel to
 186  reboot into.
 187
 188  For example, if in the /boot/grub/menu.lst the test kernel title has:
 189  title Test Kernel
 190  kernel vmlinuz-test
 191  GRUB_MENU = Test Kernel
 192 EOF
 193     ;
 194 $config_help{"REBOOT_SCRIPT"} = << "EOF"
 195  A script to reboot the target into the test kernel
 196  (Only mandatory if REBOOT_TYPE = script)
 197 EOF
 198     ;
 199
 200
 201 sub get_ktest_config {
 202     my ($config) = @_;
 203
 204     return if (defined($opt{$config}));
 205
 206     if (defined($config_help{$config})) {
 207         print "\n";
 208         print $config_help{$config};
 209     }
 210
 211     for (;;) {
 212         print "$config = ";
 213         if (defined($default{$config})) {
 214             print "\[$default{$config}\] ";
 215         }
 216         $entered_configs{$config} = <STDIN>;
 217         $entered_configs{$config} =~ s/^\s*(.*\S)\s*$/$1/;
 218         if ($entered_configs{$config} =~ /^\s*$/) {
 219             if ($default{$config}) {
 220                 $entered_configs{$config} = $default{$config};
 221             } else {
 222                 print "Your answer can not be blank\n";
 223                 next;
 224             }
 225         }
 226         last;
 227     }
 228 }
 229
 230 sub get_ktest_configs {
 231     get_ktest_config("MACHINE");
 232     get_ktest_config("SSH_USER");
 233     get_ktest_config("BUILD_DIR");
 234     get_ktest_config("OUTPUT_DIR");
 235     get_ktest_config("BUILD_TARGET");
 236     get_ktest_config("TARGET_IMAGE");
 237     get_ktest_config("POWER_CYCLE");
 238     get_ktest_config("CONSOLE");
 239     get_ktest_config("LOCALVERSION");
 240
 241     my $rtype = $opt{"REBOOT_TYPE"};
 242
 243     if (!defined($rtype)) {
 244         if (!defined($opt{"GRUB_MENU"})) {
 245             get_ktest_config("REBOOT_TYPE");
 246             $rtype = $entered_configs{"REBOOT_TYPE"};
 247         } else {
 248             $rtype = "grub";
 249         }
 250     }
 251
 252     if ($rtype eq "grub") {
 253         get_ktest_config("GRUB_MENU");
 254     } else {
 255         get_ktest_config("REBOOT_SCRIPT");
 256     }
 257 }
 258
 259 sub set_value {
 260     my ($lvalue, $rvalue) = @_;
 261
 262     if (defined($opt{$lvalue})) {
 263         die "Error: Option $lvalue defined more than once!\n";
 264     }
 265     if ($rvalue =~ /^\s*$/) {
 266         delete $opt{$lvalue};
 267     } else {
 268         $opt{$lvalue} = $rvalue;
 269     }
 270 }
 271
 272 sub read_config {
 273     my ($config) = @_;
 274
 275     open(IN, $config) || die "can't read file $config";
 276
 277     my $name = $config;
 278     $name =~ s,.*/(.*),$1,;
 279
 280     my $test_num = 0;
 281     my $default = 1;
 282     my $repeat = 1;
 283     my $num_tests_set = 0;
 284     my $skip = 0;
 285     my $rest;
 286
 287     while (<IN>) {
 288
 289         # ignore blank lines and comments
 290         next if (/^\s*$/ || /\s*\#/);
 291
 292         if (/^\s*TEST_START(.*)/) {
 293
 294             $rest = $1;
 295
 296             if ($num_tests_set) {
 297                 die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 298             }
 299
 300             my $old_test_num = $test_num;
 301             my $old_repeat = $repeat;
 302
 303             $test_num += $repeat;
 304             $default = 0;
 305             $repeat = 1;
 306
 307             if ($rest =~ /\s+SKIP(.*)/) {
 308                 $rest = $1;
 309                 $skip = 1;
 310             } else {
 311                 $skip = 0;
 312             }
 313
 314             if ($rest =~ /\s+ITERATE\s+(\d+)(.*)$/) {
 315                 $repeat = $1;
 316                 $rest = $2;
 317                 $repeat_tests{"$test_num"} = $repeat;
 318             }
 319
 320             if ($rest =~ /\s+SKIP(.*)/) {
 321                 $rest = $1;
 322                 $skip = 1;
 323             }
 324
 325             if ($rest !~ /^\s*$/) {
 326                 die "$name: $.: Gargbage found after TEST_START\n$_";
 327             }
 328
 329             if ($skip) {
 330                 $test_num = $old_test_num;
 331                 $repeat = $old_repeat;
 332             }
 333
 334         } elsif (/^\s*DEFAULTS(.*)$/) {
 335             $default = 1;
 336
 337             $rest = $1;
 338
 339             if ($rest =~ /\s+SKIP(.*)/) {
 340                 $rest = $1;
 341                 $skip = 1;
 342             } else {
 343                 $skip = 0;
 344             }
 345
 346             if ($rest !~ /^\s*$/) {
 347                 die "$name: $.: Gargbage found after DEFAULTS\n$_";
 348             }
 349
 350         } elsif (/^\s*([A-Z_\[\]\d]+)\s*=\s*(.*?)\s*$/) {
 351
 352             next if ($skip);
 353
 354             my $lvalue = $1;
 355             my $rvalue = $2;
 356
 357             if (!$default &&
 358                 ($lvalue eq "NUM_TESTS" ||
 359                  $lvalue eq "LOG_FILE" ||
 360                  $lvalue eq "CLEAR_LOG")) {
 361                 die "$name: $.: $lvalue must be set in DEFAULTS section\n";
 362             }
 363
 364             if ($lvalue eq "NUM_TESTS") {
 365                 if ($test_num) {
 366                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 367                 }
 368                 if (!$default) {
 369                     die "$name: $.: NUM_TESTS must be set in default section\n";
 370                 }
 371                 $num_tests_set = 1;
 372             }
 373
 374             if ($default || $lvalue =~ /\[\d+\]$/) {
 375                 set_value($lvalue, $rvalue);
 376             } else {
 377                 my $val = "$lvalue\[$test_num\]";
 378                 set_value($val, $rvalue);
 379
 380                 if ($repeat > 1) {
 381                     $repeats{$val} = $repeat;
 382                 }
 383             }
 384         } else {
 385             die "$name: $.: Garbage found in config\n$_";
 386         }
 387     }
 388
 389     close(IN);
 390
 391     if ($test_num) {
 392         $test_num += $repeat - 1;
 393         $opt{"NUM_TESTS"} = $test_num;
 394     }
 395
 396     # make sure we have all mandatory configs
 397     get_ktest_configs;
 398
 399     # set any defaults
 400
 401     foreach my $default (keys %default) {
 402         if (!defined($opt{$default})) {
 403             $opt{$default} = $default{$default};
 404         }
 405     }
 406 }
 407
 408 sub _logit {
 409     if (defined($opt{"LOG_FILE"})) {
 410         open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
 411         print OUT @_;
 412         close(OUT);
 413     }
 414 }
 415
 416 sub logit {
 417     if (defined($opt{"LOG_FILE"})) {
 418         _logit @_;
 419     } else {
 420         print @_;
 421     }
 422 }
 423
 424 sub doprint {
 425     print @_;
 426     _logit @_;
 427 }
 428
 429 sub run_command;
 430
 431 sub reboot {
 432     # try to reboot normally
 433     if (run_command $reboot) {
 434         if (defined($powercycle_after_reboot)) {
 435             sleep $powercycle_after_reboot;
 436             run_command "$power_cycle";
 437         }
 438     } else {
 439         # nope? power cycle it.
 440         run_command "$power_cycle";
 441     }
 442 }
 443
 444 sub do_not_reboot {
 445     my $i = $iteration;
 446
 447     return $test_type eq "build" ||
 448         ($test_type eq "patchcheck" && $opt{"PATCHCHECK_TYPE[$i]"} eq "build") ||
 449         ($test_type eq "bisect" && $opt{"BISECT_TYPE[$i]"} eq "build");
 450 }
 451
 452 sub dodie {
 453     doprint "CRITICAL FAILURE... ", @_, "\n";
 454
 455     my $i = $iteration;
 456
 457     if ($reboot_on_error && !do_not_reboot) {
 458
 459         doprint "REBOOTING\n";
 460         reboot;
 461
 462     } elsif ($poweroff_on_error && defined($power_off)) {
 463         doprint "POWERING OFF\n";
 464         `$power_off`;
 465     }
 466
 467     if (defined($opt{"LOG_FILE"})) {
 468         print " See $opt{LOG_FILE} for more info.\n";
 469     }
 470
 471     die @_, "\n";
 472 }
 473
 474 sub open_console {
 475     my ($fp) = @_;
 476
 477     my $flags;
 478
 479     my $pid = open($fp, "$console|") or
 480         dodie "Can't open console $console";
 481
 482     $flags = fcntl($fp, F_GETFL, 0) or
 483         dodie "Can't get flags for the socket: $!";
 484     $flags = fcntl($fp, F_SETFL, $flags | O_NONBLOCK) or
 485         dodie "Can't set flags for the socket: $!";
 486
 487     return $pid;
 488 }
 489
 490 sub close_console {
 491     my ($fp, $pid) = @_;
 492
 493     doprint "kill child process $pid\n";
 494     kill 2, $pid;
 495
 496     print "closing!\n";
 497     close($fp);
 498 }
 499
 500 sub start_monitor {
 501     if ($monitor_cnt++) {
 502         return;
 503     }
 504     $monitor_fp = \*MONFD;
 505     $monitor_pid = open_console $monitor_fp;
 506
 507     return;
 508
 509     open(MONFD, "Stop perl from warning about single use of MONFD");
 510 }
 511
 512 sub end_monitor {
 513     if (--$monitor_cnt) {
 514         return;
 515     }
 516     close_console($monitor_fp, $monitor_pid);
 517 }
 518
 519 sub wait_for_monitor {
 520     my ($time) = @_;
 521     my $line;
 522
 523     doprint "** Wait for monitor to settle down **\n";
 524
 525     # read the monitor and wait for the system to calm down
 526     do {
 527         $line = wait_for_input($monitor_fp, $time);
 528         print "$line" if (defined($line));
 529     } while (defined($line));
 530     print "** Monitor flushed **\n";
 531 }
 532
 533 sub fail {
 534
 535         if ($die_on_failure) {
 536                 dodie @_;
 537         }
 538
 539         doprint "FAILED\n";
 540
 541         my $i = $iteration;
 542
 543         # no need to reboot for just building.
 544         if (!do_not_reboot) {
 545             doprint "REBOOTING\n";
 546             reboot;
 547             start_monitor;
 548             wait_for_monitor $sleep_time;
 549             end_monitor;
 550         }
 551
 552         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 553         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 554         doprint "KTEST RESULT: TEST $i Failed: ", @_, "\n";
 555         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 556         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 557
 558         return 1 if (!defined($store_failures));
 559
 560         my @t = localtime;
 561         my $date = sprintf "%04d%02d%02d%02d%02d%02d",
 562                 1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
 563
 564         my $type = $build_type;
 565         if ($type =~ /useconfig/) {
 566             $type = "useconfig";
 567         }
 568
 569         my $dir = "$machine-$test_type-$type-fail-$date";
 570         my $faildir = "$store_failures/$dir";
 571
 572         if (!-d $faildir) {
 573             mkpath($faildir) or
 574                 die "can't create $faildir";
 575         }
 576         if (-f "$output_config") {
 577             cp "$output_config", "$faildir/config" or
 578                 die "failed to copy .config";
 579         }
 580         if (-f $buildlog) {
 581             cp $buildlog, "$faildir/buildlog" or
 582                 die "failed to move $buildlog";
 583         }
 584         if (-f $dmesg) {
 585             cp $dmesg, "$faildir/dmesg" or
 586                 die "failed to move $dmesg";
 587         }
 588
 589         doprint "*** Saved info to $faildir ***\n";
 590
 591         return 1;
 592 }
 593
 594 sub run_command {
 595     my ($command) = @_;
 596     my $dolog = 0;
 597     my $dord = 0;
 598     my $pid;
 599
 600     $command =~ s/\$SSH_USER/$ssh_user/g;
 601     $command =~ s/\$MACHINE/$machine/g;
 602
 603     doprint("$command ... ");
 604
 605     $pid = open(CMD, "$command 2>&1 |") or
 606         (fail "unable to exec $command" and return 0);
 607
 608     if (defined($opt{"LOG_FILE"})) {
 609         open(LOG, ">>$opt{LOG_FILE}") or
 610             dodie "failed to write to log";
 611         $dolog = 1;
 612     }
 613
 614     if (defined($redirect)) {
 615         open (RD, ">$redirect") or
 616             dodie "failed to write to redirect $redirect";
 617         $dord = 1;
 618     }
 619
 620     while (<CMD>) {
 621         print LOG if ($dolog);
 622         print RD  if ($dord);
 623     }
 624
 625     waitpid($pid, 0);
 626     my $failed = $?;
 627
 628     close(CMD);
 629     close(LOG) if ($dolog);
 630     close(RD)  if ($dord);
 631
 632     if ($failed) {
 633         doprint "FAILED!\n";
 634     } else {
 635         doprint "SUCCESS\n";
 636     }
 637
 638     return !$failed;
 639 }
 640
 641 sub run_ssh {
 642     my ($cmd) = @_;
 643     my $cp_exec = $ssh_exec;
 644
 645     $cp_exec =~ s/\$SSH_COMMAND/$cmd/g;
 646     return run_command "$cp_exec";
 647 }
 648
 649 sub run_scp {
 650     my ($src, $dst) = @_;
 651     my $cp_scp = $scp_to_target;
 652
 653     $cp_scp =~ s/\$SRC_FILE/$src/g;
 654     $cp_scp =~ s/\$DST_FILE/$dst/g;
 655
 656     return run_command "$cp_scp";
 657 }
 658
 659 sub get_grub_index {
 660
 661     if ($reboot_type ne "grub") {
 662         return;
 663     }
 664     return if (defined($grub_number));
 665
 666     doprint "Find grub menu ... ";
 667     $grub_number = -1;
 668
 669     my $ssh_grub = $ssh_exec;
 670     $ssh_grub =~ s,\$SSH_COMMAND,cat /boot/grub/menu.lst,g;
 671
 672     open(IN, "$ssh_grub |")
 673         or die "unable to get menu.lst";
 674
 675     while (<IN>) {
 676         if (/^\s*title\s+$grub_menu\s*$/) {
 677             $grub_number++;
 678             last;
 679         } elsif (/^\s*title\s/) {
 680             $grub_number++;
 681         }
 682     }
 683     close(IN);
 684
 685     die "Could not find '$grub_menu' in /boot/grub/menu on $machine"
 686         if ($grub_number < 0);
 687     doprint "$grub_number\n";
 688 }
 689
 690 sub wait_for_input
 691 {
 692     my ($fp, $time) = @_;
 693     my $rin;
 694     my $ready;
 695     my $line;
 696     my $ch;
 697
 698     if (!defined($time)) {
 699         $time = $timeout;
 700     }
 701
 702     $rin = '';
 703     vec($rin, fileno($fp), 1) = 1;
 704     $ready = select($rin, undef, undef, $time);
 705
 706     $line = "";
 707
 708     # try to read one char at a time
 709     while (sysread $fp, $ch, 1) {
 710         $line .= $ch;
 711         last if ($ch eq "\n");
 712     }
 713
 714     if (!length($line)) {
 715         return undef;
 716     }
 717
 718     return $line;
 719 }
 720
 721 sub reboot_to {
 722     if ($reboot_type eq "grub") {
 723         run_ssh "'(echo \"savedefault --default=$grub_number --once\" | grub --batch; reboot)'";
 724         return;
 725     }
 726
 727     run_command "$reboot_script";
 728 }
 729
 730 sub get_sha1 {
 731     my ($commit) = @_;
 732
 733     doprint "git rev-list --max-count=1 $commit ... ";
 734     my $sha1 = `git rev-list --max-count=1 $commit`;
 735     my $ret = $?;
 736
 737     logit $sha1;
 738
 739     if ($ret) {
 740         doprint "FAILED\n";
 741         dodie "Failed to get git $commit";
 742     }
 743
 744     print "SUCCESS\n";
 745
 746     chomp $sha1;
 747
 748     return $sha1;
 749 }
 750
 751 sub monitor {
 752     my $booted = 0;
 753     my $bug = 0;
 754     my $skip_call_trace = 0;
 755     my $loops;
 756
 757     wait_for_monitor 5;
 758
 759     my $line;
 760     my $full_line = "";
 761
 762     open(DMESG, "> $dmesg") or
 763         die "unable to write to $dmesg";
 764
 765     reboot_to;
 766
 767     my $success_start;
 768     my $failure_start;
 769
 770     for (;;) {
 771
 772         if ($booted) {
 773             $line = wait_for_input($monitor_fp, $booted_timeout);
 774         } else {
 775             $line = wait_for_input($monitor_fp);
 776         }
 777
 778         last if (!defined($line));
 779
 780         doprint $line;
 781         print DMESG $line;
 782
 783         # we are not guaranteed to get a full line
 784         $full_line .= $line;
 785
 786         if ($full_line =~ /$success_line/) {
 787             $booted = 1;
 788             $success_start = time;
 789         }
 790
 791         if ($booted && defined($stop_after_success) &&
 792             $stop_after_success >= 0) {
 793             my $now = time;
 794             if ($now - $success_start >= $stop_after_success) {
 795                 doprint "Test forced to stop after $stop_after_success seconds after success\n";
 796                 last;
 797             }
 798         }
 799
 800         if ($full_line =~ /\[ backtrace testing \]/) {
 801             $skip_call_trace = 1;
 802         }
 803
 804         if ($full_line =~ /call trace:/i) {
 805             if (!$skip_call_trace) {
 806                 $bug = 1;
 807                 $failure_start = time;
 808             }
 809         }
 810
 811         if ($bug && defined($stop_after_failure) &&
 812             $stop_after_failure >= 0) {
 813             my $now = time;
 814             if ($now - $failure_start >= $stop_after_failure) {
 815                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
 816                 last;
 817             }
 818         }
 819
 820         if ($full_line =~ /\[ end of backtrace testing \]/) {
 821             $skip_call_trace = 0;
 822         }
 823
 824         if ($full_line =~ /Kernel panic -/) {
 825             $failure_start = time;
 826             $bug = 1;
 827         }
 828
 829         if ($line =~ /\n/) {
 830             $full_line = "";
 831         }
 832     }
 833
 834     close(DMESG);
 835
 836     if ($bug) {
 837         return 0 if ($in_bisect);
 838         fail "failed - got a bug report" and return 0;
 839     }
 840
 841     if (!$booted) {
 842         return 0 if ($in_bisect);
 843         fail "failed - never got a boot prompt." and return 0;
 844     }
 845
 846     return 1;
 847 }
 848
 849 sub install {
 850
 851     run_scp "$outputdir/$build_target", "$target_image" or
 852         dodie "failed to copy image";
 853
 854     my $install_mods = 0;
 855
 856     # should we process modules?
 857     $install_mods = 0;
 858     open(IN, "$output_config") or dodie("Can't read config file");
 859     while (<IN>) {
 860         if (/CONFIG_MODULES(=y)?/) {
 861             $install_mods = 1 if (defined($1));
 862             last;
 863         }
 864     }
 865     close(IN);
 866
 867     if (!$install_mods) {
 868         doprint "No modules needed\n";
 869         return;
 870     }
 871
 872     run_command "$make INSTALL_MOD_PATH=$tmpdir modules_install" or
 873         dodie "Failed to install modules";
 874
 875     my $modlib = "/lib/modules/$version";
 876     my $modtar = "ktest-mods.tar.bz2";
 877
 878     run_ssh "rm -rf $modlib" or
 879         dodie "failed to remove old mods: $modlib";
 880
 881     # would be nice if scp -r did not follow symbolic links
 882     run_command "cd $tmpdir && tar -cjf $modtar lib/modules/$version" or
 883         dodie "making tarball";
 884
 885     run_scp "$tmpdir/$modtar", "/tmp" or
 886         dodie "failed to copy modules";
 887
 888     unlink "$tmpdir/$modtar";
 889
 890     run_ssh "'(cd / && tar xf /tmp/$modtar)'" or
 891         dodie "failed to tar modules";
 892
 893     run_ssh "rm -f /tmp/$modtar";
 894
 895     return if (!defined($post_install));
 896
 897     my $cp_post_install = $post_install;
 898     $cp_post_install = s/\$KERNEL_VERSION/$version/g;
 899     run_command "$cp_post_install" or
 900         dodie "Failed to run post install";
 901 }
 902
 903 sub check_buildlog {
 904     my ($patch) = @_;
 905
 906     my @files = `git show $patch | diffstat -l`;
 907
 908     open(IN, "git show $patch |") or
 909         dodie "failed to show $patch";
 910     while (<IN>) {
 911         if (m,^--- a/(.*),) {
 912             chomp $1;
 913             $files[$#files] = $1;
 914         }
 915     }
 916     close(IN);
 917
 918     open(IN, $buildlog) or dodie "Can't open $buildlog";
 919     while (<IN>) {
 920         if (/^\s*(.*?):.*(warning|error)/) {
 921             my $err = $1;
 922             foreach my $file (@files) {
 923                 my $fullpath = "$builddir/$file";
 924                 if ($file eq $err || $fullpath eq $err) {
 925                     fail "$file built with warnings" and return 0;
 926                 }
 927             }
 928         }
 929     }
 930     close(IN);
 931
 932     return 1;
 933 }
 934
 935 sub make_oldconfig {
 936     my ($defconfig) = @_;
 937
 938     if (!run_command "$defconfig $make oldnoconfig") {
 939         # Perhaps oldnoconfig doesn't exist in this version of the kernel
 940         # try a yes '' | oldconfig
 941         doprint "oldnoconfig failed, trying yes '' | make oldconfig\n";
 942         run_command "yes '' | $defconfig $make oldconfig" or
 943             dodie "failed make config oldconfig";
 944     }
 945 }
 946
 947 sub build {
 948     my ($type) = @_;
 949     my $defconfig = "";
 950
 951     unlink $buildlog;
 952
 953     if ($type =~ /^useconfig:(.*)/) {
 954         run_command "cp $1 $output_config" or
 955             dodie "could not copy $1 to .config";
 956
 957         $type = "oldconfig";
 958     }
 959
 960     # old config can ask questions
 961     if ($type eq "oldconfig") {
 962         $type = "oldnoconfig";
 963
 964         # allow for empty configs
 965         run_command "touch $output_config";
 966
 967         run_command "mv $output_config $outputdir/config_temp" or
 968             dodie "moving .config";
 969
 970         if (!$noclean && !run_command "$make mrproper") {
 971             dodie "make mrproper";
 972         }
 973
 974         run_command "mv $outputdir/config_temp $output_config" or
 975             dodie "moving config_temp";
 976
 977     } elsif (!$noclean) {
 978         unlink "$output_config";
 979         run_command "$make mrproper" or
 980             dodie "make mrproper";
 981     }
 982
 983     # add something to distinguish this build
 984     open(OUT, "> $outputdir/localversion") or dodie("Can't make localversion file");
 985     print OUT "$localversion\n";
 986     close(OUT);
 987
 988     if (defined($minconfig)) {
 989         $defconfig = "KCONFIG_ALLCONFIG=$minconfig";
 990     }
 991
 992     if ($type eq "oldnoconfig") {
 993         make_oldconfig $defconfig;
 994     } else {
 995         run_command "$defconfig $make $type" or
 996             dodie "failed make config";
 997     }
 998
 999     $redirect = "$buildlog";
1000     if (!run_command "$make $build_options") {
1001         undef $redirect;
1002         # bisect may need this to pass
1003         return 0 if ($in_bisect);
1004         fail "failed build" and return 0;
1005     }
1006     undef $redirect;
1007
1008     return 1;
1009 }
1010
1011 sub halt {
1012     if (!run_ssh "halt" or defined($power_off)) {
1013         if (defined($poweroff_after_halt)) {
1014             sleep $poweroff_after_halt;
1015             run_command "$power_off";
1016         }
1017     } else {
1018         # nope? the zap it!
1019         run_command "$power_off";
1020     }
1021 }
1022
1023 sub success {
1024     my ($i) = @_;
1025
1026     $successes++;
1027
1028     doprint "\n\n*******************************************\n";
1029     doprint     "*******************************************\n";
1030     doprint     "KTEST RESULT: TEST $i SUCCESS!!!!         **\n";
1031     doprint     "*******************************************\n";
1032     doprint     "*******************************************\n";
1033
1034     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
1035         doprint "Reboot and wait $sleep_time seconds\n";
1036         reboot;
1037         start_monitor;
1038         wait_for_monitor $sleep_time;
1039         end_monitor;
1040     }
1041 }
1042
1043 sub get_version {
1044     # get the release name
1045     doprint "$make kernelrelease ... ";
1046     $version = `$make kernelrelease | tail -1`;
1047     chomp($version);
1048     doprint "$version\n";
1049 }
1050
1051 sub answer_bisect {
1052     for (;;) {
1053         doprint "Pass or fail? [p/f]";
1054         my $ans = <STDIN>;
1055         chomp $ans;
1056         if ($ans eq "p" || $ans eq "P") {
1057             return 1;
1058         } elsif ($ans eq "f" || $ans eq "F") {
1059             return 0;
1060         } else {
1061             print "Please answer 'P' or 'F'\n";
1062         }
1063     }
1064 }
1065
1066 sub child_run_test {
1067     my $failed = 0;
1068
1069     # child should have no power
1070     $reboot_on_error = 0;
1071     $poweroff_on_error = 0;
1072     $die_on_failure = 1;
1073
1074     run_command $run_test or $failed = 1;
1075     exit $failed;
1076 }
1077
1078 my $child_done;
1079
1080 sub child_finished {
1081     $child_done = 1;
1082 }
1083
1084 sub do_run_test {
1085     my $child_pid;
1086     my $child_exit;
1087     my $line;
1088     my $full_line;
1089     my $bug = 0;
1090
1091     wait_for_monitor 1;
1092
1093     doprint "run test $run_test\n";
1094
1095     $child_done = 0;
1096
1097     $SIG{CHLD} = qw(child_finished);
1098
1099     $child_pid = fork;
1100
1101     child_run_test if (!$child_pid);
1102
1103     $full_line = "";
1104
1105     do {
1106         $line = wait_for_input($monitor_fp, 1);
1107         if (defined($line)) {
1108
1109             # we are not guaranteed to get a full line
1110             $full_line .= $line;
1111
1112             if ($full_line =~ /call trace:/i) {
1113                 $bug = 1;
1114             }
1115
1116             if ($full_line =~ /Kernel panic -/) {
1117                 $bug = 1;
1118             }
1119
1120             if ($line =~ /\n/) {
1121                 $full_line = "";
1122             }
1123         }
1124     } while (!$child_done && !$bug);
1125
1126     if ($bug) {
1127         doprint "Detected kernel crash!\n";
1128         # kill the child with extreme prejudice
1129         kill 9, $child_pid;
1130     }
1131
1132     waitpid $child_pid, 0;
1133     $child_exit = $?;
1134
1135     if ($bug || $child_exit) {
1136         return 0 if $in_bisect;
1137         fail "test failed" and return 0;
1138     }
1139     return 1;
1140 }
1141
1142 sub run_git_bisect {
1143     my ($command) = @_;
1144
1145     doprint "$command ... ";
1146
1147     my $output = `$command 2>&1`;
1148     my $ret = $?;
1149
1150     logit $output;
1151
1152     if ($ret) {
1153         doprint "FAILED\n";
1154         dodie "Failed to git bisect";
1155     }
1156
1157     doprint "SUCCESS\n";
1158     if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
1159         doprint "$1 [$2]\n";
1160     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
1161         $bisect_bad = $1;
1162         doprint "Found bad commit... $1\n";
1163         return 0;
1164     } else {
1165         # we already logged it, just print it now.
1166         print $output;
1167     }
1168
1169     return 1;
1170 }
1171
1172 # returns 1 on success, 0 on failure
1173 sub run_bisect_test {
1174     my ($type, $buildtype) = @_;
1175
1176     my $failed = 0;
1177     my $result;
1178     my $output;
1179     my $ret;
1180
1181     $in_bisect = 1;
1182
1183     build $buildtype or $failed = 1;
1184
1185     if ($type ne "build") {
1186         dodie "Failed on build" if $failed;
1187
1188         # Now boot the box
1189         get_grub_index;
1190         get_version;
1191         install;
1192
1193         start_monitor;
1194         monitor or $failed = 1;
1195
1196         if ($type ne "boot") {
1197             dodie "Failed on boot" if $failed;
1198
1199             do_run_test or $failed = 1;
1200         }
1201         end_monitor;
1202     }
1203
1204     if ($failed) {
1205         $result = 0;
1206
1207         # reboot the box to a good kernel
1208         if ($type ne "build") {
1209             doprint "Reboot and sleep $bisect_sleep_time seconds\n";
1210             reboot;
1211             start_monitor;
1212             wait_for_monitor $bisect_sleep_time;
1213             end_monitor;
1214         }
1215     } else {
1216         $result = 1;
1217     }
1218     $in_bisect = 0;
1219
1220     return $result;
1221 }
1222
1223 sub run_bisect {
1224     my ($type) = @_;
1225     my $buildtype = "oldconfig";
1226
1227     # We should have a minconfig to use?
1228     if (defined($minconfig)) {
1229         $buildtype = "useconfig:$minconfig";
1230     }
1231
1232     my $ret = run_bisect_test $type, $buildtype;
1233
1234     if ($bisect_manual) {
1235         $ret = answer_bisect;
1236     }
1237
1238     # Are we looking for where it worked, not failed?
1239     if ($reverse_bisect) {
1240         $ret = !$ret;
1241     }
1242
1243     if ($ret) {
1244         return "good";
1245     } else {
1246         return  "bad";
1247     }
1248 }
1249
1250 sub bisect {
1251     my ($i) = @_;
1252
1253     my $result;
1254
1255     die "BISECT_GOOD[$i] not defined\n" if (!defined($opt{"BISECT_GOOD[$i]"}));
1256     die "BISECT_BAD[$i] not defined\n"  if (!defined($opt{"BISECT_BAD[$i]"}));
1257     die "BISECT_TYPE[$i] not defined\n" if (!defined($opt{"BISECT_TYPE[$i]"}));
1258
1259     my $good = $opt{"BISECT_GOOD[$i]"};
1260     my $bad = $opt{"BISECT_BAD[$i]"};
1261     my $type = $opt{"BISECT_TYPE[$i]"};
1262     my $start = $opt{"BISECT_START[$i]"};
1263     my $replay = $opt{"BISECT_REPLAY[$i]"};
1264
1265     # convert to true sha1's
1266     $good = get_sha1($good);
1267     $bad = get_sha1($bad);
1268
1269     if (defined($opt{"BISECT_REVERSE[$i]"}) &&
1270         $opt{"BISECT_REVERSE[$i]"} == 1) {
1271         doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
1272         $reverse_bisect = 1;
1273     } else {
1274         $reverse_bisect = 0;
1275     }
1276
1277     # Can't have a test without having a test to run
1278     if ($type eq "test" && !defined($run_test)) {
1279         $type = "boot";
1280     }
1281
1282     my $check = $opt{"BISECT_CHECK[$i]"};
1283     if (defined($check) && $check ne "0") {
1284
1285         # get current HEAD
1286         my $head = get_sha1("HEAD");
1287
1288         if ($check ne "good") {
1289             doprint "TESTING BISECT BAD [$bad]\n";
1290             run_command "git checkout $bad" or
1291                 die "Failed to checkout $bad";
1292
1293             $result = run_bisect $type;
1294
1295             if ($result ne "bad") {
1296                 fail "Tested BISECT_BAD [$bad] and it succeeded" and return 0;
1297             }
1298         }
1299
1300         if ($check ne "bad") {
1301             doprint "TESTING BISECT GOOD [$good]\n";
1302             run_command "git checkout $good" or
1303                 die "Failed to checkout $good";
1304
1305             $result = run_bisect $type;
1306
1307             if ($result ne "good") {
1308                 fail "Tested BISECT_GOOD [$good] and it failed" and return 0;
1309             }
1310         }
1311
1312         # checkout where we started
1313         run_command "git checkout $head" or
1314             die "Failed to checkout $head";
1315     }
1316
1317     run_command "git bisect start" or
1318         dodie "could not start bisect";
1319
1320     run_command "git bisect good $good" or
1321         dodie "could not set bisect good to $good";
1322
1323     run_git_bisect "git bisect bad $bad" or
1324         dodie "could not set bisect bad to $bad";
1325
1326     if (defined($replay)) {
1327         run_command "git bisect replay $replay" or
1328             dodie "failed to run replay";
1329     }
1330
1331     if (defined($start)) {
1332         run_command "git checkout $start" or
1333             dodie "failed to checkout $start";
1334     }
1335
1336     my $test;
1337     do {
1338         $result = run_bisect $type;
1339         $test = run_git_bisect "git bisect $result";
1340     } while ($test);
1341
1342     run_command "git bisect log" or
1343         dodie "could not capture git bisect log";
1344
1345     run_command "git bisect reset" or
1346         dodie "could not reset git bisect";
1347
1348     doprint "Bad commit was [$bisect_bad]\n";
1349
1350     success $i;
1351 }
1352
1353 my %config_ignore;
1354 my %config_set;
1355
1356 my %config_list;
1357 my %null_config;
1358
1359 my %dependency;
1360
1361 sub process_config_ignore {
1362     my ($config) = @_;
1363
1364     open (IN, $config)
1365         or dodie "Failed to read $config";
1366
1367     while (<IN>) {
1368         if (/^(.*?(CONFIG\S*)(=.*| is not set))/) {
1369             $config_ignore{$2} = $1;
1370         }
1371     }
1372
1373     close(IN);
1374 }
1375
1376 sub read_current_config {
1377     my ($config_ref) = @_;
1378
1379     %{$config_ref} = ();
1380     undef %{$config_ref};
1381
1382     my @key = keys %{$config_ref};
1383     if ($#key >= 0) {
1384         print "did not delete!\n";
1385         exit;
1386     }
1387     open (IN, "$output_config");
1388
1389     while (<IN>) {
1390         if (/^(CONFIG\S+)=(.*)/) {
1391             ${$config_ref}{$1} = $2;
1392         }
1393     }
1394     close(IN);
1395 }
1396
1397 sub get_dependencies {
1398     my ($config) = @_;
1399
1400     my $arr = $dependency{$config};
1401     if (!defined($arr)) {
1402         return ();
1403     }
1404
1405     my @deps = @{$arr};
1406
1407     foreach my $dep (@{$arr}) {
1408         print "ADD DEP $dep\n";
1409         @deps = (@deps, get_dependencies $dep);
1410     }
1411
1412     return @deps;
1413 }
1414
1415 sub create_config {
1416     my @configs = @_;
1417
1418     open(OUT, ">$output_config") or dodie "Can not write to $output_config";
1419
1420     foreach my $config (@configs) {
1421         print OUT "$config_set{$config}\n";
1422         my @deps = get_dependencies $config;
1423         foreach my $dep (@deps) {
1424             print OUT "$config_set{$dep}\n";
1425         }
1426     }
1427
1428     foreach my $config (keys %config_ignore) {
1429         print OUT "$config_ignore{$config}\n";
1430     }
1431     close(OUT);
1432
1433 #    exit;
1434     make_oldconfig "";
1435 }
1436
1437 sub compare_configs {
1438     my (%a, %b) = @_;
1439
1440     foreach my $item (keys %a) {
1441         if (!defined($b{$item})) {
1442             print "diff $item\n";
1443             return 1;
1444         }
1445         delete $b{$item};
1446     }
1447
1448     my @keys = keys %b;
1449     if ($#keys) {
1450         print "diff2 $keys[0]\n";
1451     }
1452     return -1 if ($#keys >= 0);
1453
1454     return 0;
1455 }
1456
1457 sub run_config_bisect_test {
1458     my ($type) = @_;
1459
1460     return run_bisect_test $type, "oldconfig";
1461 }
1462
1463 sub process_passed {
1464     my (%configs) = @_;
1465
1466     doprint "These configs had no failure: (Enabling them for further compiles)\n";
1467     # Passed! All these configs are part of a good compile.
1468     # Add them to the min options.
1469     foreach my $config (keys %configs) {
1470         if (defined($config_list{$config})) {
1471             doprint " removing $config\n";
1472             $config_ignore{$config} = $config_list{$config};
1473             delete $config_list{$config};
1474         }
1475     }
1476     doprint "config copied to $outputdir/config_good\n";
1477     run_command "cp -f $output_config $outputdir/config_good";
1478 }
1479
1480 sub process_failed {
1481     my ($config) = @_;
1482
1483     doprint "\n\n***************************************\n";
1484     doprint "Found bad config: $config\n";
1485     doprint "***************************************\n\n";
1486 }
1487
1488 sub run_config_bisect {
1489
1490     my @start_list = keys %config_list;
1491
1492     if ($#start_list < 0) {
1493         doprint "No more configs to test!!!\n";
1494         return -1;
1495     }
1496
1497     doprint "***** RUN TEST ***\n";
1498     my $type = $opt{"CONFIG_BISECT_TYPE[$iteration]"};
1499     my $ret;
1500     my %current_config;
1501
1502     my $count = $#start_list + 1;
1503     doprint "  $count configs to test\n";
1504
1505     my $half = int($#start_list / 2);
1506
1507     do {
1508         my @tophalf = @start_list[0 .. $half];
1509
1510         create_config @tophalf;
1511         read_current_config \%current_config;
1512
1513         $count = $#tophalf + 1;
1514         doprint "Testing $count configs\n";
1515         my $found = 0;
1516         # make sure we test something
1517         foreach my $config (@tophalf) {
1518             if (defined($current_config{$config})) {
1519                 logit " $config\n";
1520                 $found = 1;
1521             }
1522         }
1523         if (!$found) {
1524             # try the other half
1525             doprint "Top half produced no set configs, trying bottom half\n";
1526             @tophalf = @start_list[$half .. $#start_list];
1527             create_config @tophalf;
1528             read_current_config \%current_config;
1529             foreach my $config (@tophalf) {
1530                 if (defined($current_config{$config})) {
1531                     logit " $config\n";
1532                     $found = 1;
1533                 }
1534             }
1535             if (!$found) {
1536                 doprint "Failed: Can't make new config with current configs\n";
1537                 foreach my $config (@start_list) {
1538                     doprint "  CONFIG: $config\n";
1539                 }
1540                 return -1;
1541             }
1542             $count = $#tophalf + 1;
1543             doprint "Testing $count configs\n";
1544         }
1545
1546         $ret = run_config_bisect_test $type;
1547         if ($bisect_manual) {
1548             $ret = answer_bisect;
1549         }
1550         if ($ret) {
1551             process_passed %current_config;
1552             return 0;
1553         }
1554
1555         doprint "This config had a failure.\n";
1556         doprint "Removing these configs that were not set in this config:\n";
1557         doprint "config copied to $outputdir/config_bad\n";
1558         run_command "cp -f $output_config $outputdir/config_bad";
1559
1560         # A config exists in this group that was bad.
1561         foreach my $config (keys %config_list) {
1562             if (!defined($current_config{$config})) {
1563                 doprint " removing $config\n";
1564                 delete $config_list{$config};
1565             }
1566         }
1567
1568         @start_list = @tophalf;
1569
1570         if ($#start_list == 0) {
1571             process_failed $start_list[0];
1572             return 1;
1573         }
1574
1575         # remove half the configs we are looking at and see if
1576         # they are good.
1577         $half = int($#start_list / 2);
1578     } while ($half > 0);
1579
1580     # we found a single config, try it again unless we are running manually
1581
1582     if ($bisect_manual) {
1583         process_failed $start_list[0];
1584         return 1;
1585     }
1586
1587     my @tophalf = @start_list[0 .. 0];
1588
1589     $ret = run_config_bisect_test $type;
1590     if ($ret) {
1591         process_passed %current_config;
1592         return 0;
1593     }
1594
1595     process_failed $start_list[0];
1596     return 1;
1597 }
1598
1599 sub config_bisect {
1600     my ($i) = @_;
1601
1602     my $start_config = $opt{"CONFIG_BISECT[$i]"};
1603
1604     my $tmpconfig = "$tmpdir/use_config";
1605
1606     # Make the file with the bad config and the min config
1607     if (defined($minconfig)) {
1608         # read the min config for things to ignore
1609         run_command "cp $minconfig $tmpconfig" or
1610             dodie "failed to copy $minconfig to $tmpconfig";
1611     } else {
1612         unlink $tmpconfig;
1613     }
1614
1615     # Add other configs
1616     if (defined($addconfig)) {
1617         run_command "cat $addconfig >> $tmpconfig" or
1618             dodie "failed to append $addconfig";
1619     }
1620
1621     my $defconfig = "";
1622     if (-f $tmpconfig) {
1623         $defconfig = "KCONFIG_ALLCONFIG=$tmpconfig";
1624         process_config_ignore $tmpconfig;
1625     }
1626
1627     # now process the start config
1628     run_command "cp $start_config $output_config" or
1629         dodie "failed to copy $start_config to $output_config";
1630
1631     # read directly what we want to check
1632     my %config_check;
1633     open (IN, $output_config)
1634         or dodie "faied to open $output_config";
1635
1636     while (<IN>) {
1637         if (/^((CONFIG\S*)=.*)/) {
1638             $config_check{$2} = $1;
1639         }
1640     }
1641     close(IN);
1642
1643     # Now run oldconfig with the minconfig (and addconfigs)
1644     make_oldconfig $defconfig;
1645
1646     # check to see what we lost (or gained)
1647     open (IN, $output_config)
1648         or dodie "Failed to read $start_config";
1649
1650     my %removed_configs;
1651     my %added_configs;
1652
1653     while (<IN>) {
1654         if (/^((CONFIG\S*)=.*)/) {
1655             # save off all options
1656             $config_set{$2} = $1;
1657             if (defined($config_check{$2})) {
1658                 if (defined($config_ignore{$2})) {
1659                     $removed_configs{$2} = $1;
1660                 } else {
1661                     $config_list{$2} = $1;
1662                 }
1663             } elsif (!defined($config_ignore{$2})) {
1664                 $added_configs{$2} = $1;
1665                 $config_list{$2} = $1;
1666             }
1667         }
1668     }
1669     close(IN);
1670
1671     my @confs = keys %removed_configs;
1672     if ($#confs >= 0) {
1673         doprint "Configs overridden by default configs and removed from check:\n";
1674         foreach my $config (@confs) {
1675             doprint " $config\n";
1676         }
1677     }
1678     @confs = keys %added_configs;
1679     if ($#confs >= 0) {
1680         doprint "Configs appearing in make oldconfig and added:\n";
1681         foreach my $config (@confs) {
1682             doprint " $config\n";
1683         }
1684     }
1685
1686     my %config_test;
1687     my $once = 0;
1688
1689     # Sometimes kconfig does weird things. We must make sure
1690     # that the config we autocreate has everything we need
1691     # to test, otherwise we may miss testing configs, or
1692     # may not be able to create a new config.
1693     # Here we create a config with everything set.
1694     create_config (keys %config_list);
1695     read_current_config \%config_test;
1696     foreach my $config (keys %config_list) {
1697         if (!defined($config_test{$config})) {
1698             if (!$once) {
1699                 $once = 1;
1700                 doprint "Configs not produced by kconfig (will not be checked):\n";
1701             }
1702             doprint "  $config\n";
1703             delete $config_list{$config};
1704         }
1705     }
1706     my $ret;
1707     do {
1708         $ret = run_config_bisect;
1709     } while (!$ret);
1710
1711     return $ret if ($ret < 0);
1712
1713     success $i;
1714 }
1715
1716 sub patchcheck {
1717     my ($i) = @_;
1718
1719     die "PATCHCHECK_START[$i] not defined\n"
1720         if (!defined($opt{"PATCHCHECK_START[$i]"}));
1721     die "PATCHCHECK_TYPE[$i] not defined\n"
1722         if (!defined($opt{"PATCHCHECK_TYPE[$i]"}));
1723
1724     my $start = $opt{"PATCHCHECK_START[$i]"};
1725
1726     my $end = "HEAD";
1727     if (defined($opt{"PATCHCHECK_END[$i]"})) {
1728         $end = $opt{"PATCHCHECK_END[$i]"};
1729     }
1730
1731     # Get the true sha1's since we can use things like HEAD~3
1732     $start = get_sha1($start);
1733     $end = get_sha1($end);
1734
1735     my $type = $opt{"PATCHCHECK_TYPE[$i]"};
1736
1737     # Can't have a test without having a test to run
1738     if ($type eq "test" && !defined($run_test)) {
1739         $type = "boot";
1740     }
1741
1742     open (IN, "git log --pretty=oneline $end|") or
1743         dodie "could not get git list";
1744
1745     my @list;
1746
1747     while (<IN>) {
1748         chomp;
1749         $list[$#list+1] = $_;
1750         last if (/^$start/);
1751     }
1752     close(IN);
1753
1754     if ($list[$#list] !~ /^$start/) {
1755         fail "SHA1 $start not found";
1756     }
1757
1758     # go backwards in the list
1759     @list = reverse @list;
1760
1761     my $save_clean = $noclean;
1762
1763     $in_patchcheck = 1;
1764     foreach my $item (@list) {
1765         my $sha1 = $item;
1766         $sha1 =~ s/^([[:xdigit:]]+).*/$1/;
1767
1768         doprint "\nProcessing commit $item\n\n";
1769
1770         run_command "git checkout $sha1" or
1771             die "Failed to checkout $sha1";
1772
1773         # only clean on the first and last patch
1774         if ($item eq $list[0] ||
1775             $item eq $list[$#list]) {
1776             $noclean = $save_clean;
1777         } else {
1778             $noclean = 1;
1779         }
1780
1781         if (defined($minconfig)) {
1782             build "useconfig:$minconfig" or return 0;
1783         } else {
1784             # ?? no config to use?
1785             build "oldconfig" or return 0;
1786         }
1787
1788         check_buildlog $sha1 or return 0;
1789
1790         next if ($type eq "build");
1791
1792         get_grub_index;
1793         get_version;
1794         install;
1795
1796         my $failed = 0;
1797
1798         start_monitor;
1799         monitor or $failed = 1;
1800
1801         if (!$failed && $type ne "boot"){
1802             do_run_test or $failed = 1;
1803         }
1804         end_monitor;
1805         return 0 if ($failed);
1806
1807     }
1808     $in_patchcheck = 0;
1809     success $i;
1810
1811     return 1;
1812 }
1813
1814 $#ARGV < 1 or die "ktest.pl version: $VERSION\n   usage: ktest.pl config-file\n";
1815
1816 if ($#ARGV == 0) {
1817     $ktest_config = $ARGV[0];
1818     if (! -f $ktest_config) {
1819         print "$ktest_config does not exist.\n";
1820         my $ans;
1821         for (;;) {
1822             print "Create it? [Y/n] ";
1823             $ans = <STDIN>;
1824             chomp $ans;
1825             if ($ans =~ /^\s*$/) {
1826                 $ans = "y";
1827             }
1828             last if ($ans =~ /^y$/i || $ans =~ /^n$/i);
1829             print "Please answer either 'y' or 'n'.\n";
1830         }
1831         if ($ans !~ /^y$/i) {
1832             exit 0;
1833         }
1834     }
1835 } else {
1836     $ktest_config = "ktest.conf";
1837 }
1838
1839 if (! -f $ktest_config) {
1840     open(OUT, ">$ktest_config") or die "Can not create $ktest_config";
1841     print OUT << "EOF"
1842 # Generated by ktest.pl
1843 #
1844 # Define each test with TEST_START
1845 # The config options below it will override the defaults
1846 TEST_START
1847
1848 DEFAULTS
1849 EOF
1850 ;
1851     close(OUT);
1852 }
1853 read_config $ktest_config;
1854
1855 # Append any configs entered in manually to the config file.
1856 my @new_configs = keys %entered_configs;
1857 if ($#new_configs >= 0) {
1858     print "\nAppending entered in configs to $ktest_config\n";
1859     open(OUT, ">>$ktest_config") or die "Can not append to $ktest_config";
1860     foreach my $config (@new_configs) {
1861         print OUT "$config = $entered_configs{$config}\n";
1862         $opt{$config} = $entered_configs{$config};
1863     }
1864 }
1865
1866 if ($opt{"CLEAR_LOG"} && defined($opt{"LOG_FILE"})) {
1867     unlink $opt{"LOG_FILE"};
1868 }
1869
1870 doprint "\n\nSTARTING AUTOMATED TESTS\n\n";
1871
1872 for (my $i = 0, my $repeat = 1; $i <= $opt{"NUM_TESTS"}; $i += $repeat) {
1873
1874     if (!$i) {
1875         doprint "DEFAULT OPTIONS:\n";
1876     } else {
1877         doprint "\nTEST $i OPTIONS";
1878         if (defined($repeat_tests{$i})) {
1879             $repeat = $repeat_tests{$i};
1880             doprint " ITERATE $repeat";
1881         }
1882         doprint "\n";
1883     }
1884
1885     foreach my $option (sort keys %opt) {
1886
1887         if ($option =~ /\[(\d+)\]$/) {
1888             next if ($i != $1);
1889         } else {
1890             next if ($i);
1891         }
1892
1893         doprint "$option = $opt{$option}\n";
1894     }
1895 }
1896
1897 sub set_test_option {
1898     my ($name, $i) = @_;
1899
1900     my $option = "$name\[$i\]";
1901
1902     if (defined($opt{$option})) {
1903         return $opt{$option};
1904     }
1905
1906     foreach my $test (keys %repeat_tests) {
1907         if ($i >= $test &&
1908             $i < $test + $repeat_tests{$test}) {
1909             $option = "$name\[$test\]";
1910             if (defined($opt{$option})) {
1911                 return $opt{$option};
1912             }
1913         }
1914     }
1915
1916     if (defined($opt{$name})) {
1917         return $opt{$name};
1918     }
1919
1920     return undef;
1921 }
1922
1923 # First we need to do is the builds
1924 for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
1925
1926     $iteration = $i;
1927
1928     my $makecmd = set_test_option("MAKE_CMD", $i);
1929
1930     $machine = set_test_option("MACHINE", $i);
1931     $ssh_user = set_test_option("SSH_USER", $i);
1932     $tmpdir = set_test_option("TMP_DIR", $i);
1933     $outputdir = set_test_option("OUTPUT_DIR", $i);
1934     $builddir = set_test_option("BUILD_DIR", $i);
1935     $test_type = set_test_option("TEST_TYPE", $i);
1936     $build_type = set_test_option("BUILD_TYPE", $i);
1937     $build_options = set_test_option("BUILD_OPTIONS", $i);
1938     $power_cycle = set_test_option("POWER_CYCLE", $i);
1939     $reboot = set_test_option("REBOOT", $i);
1940     $noclean = set_test_option("BUILD_NOCLEAN", $i);
1941     $minconfig = set_test_option("MIN_CONFIG", $i);
1942     $run_test = set_test_option("TEST", $i);
1943     $addconfig = set_test_option("ADD_CONFIG", $i);
1944     $reboot_type = set_test_option("REBOOT_TYPE", $i);
1945     $grub_menu = set_test_option("GRUB_MENU", $i);
1946     $post_install = set_test_option("POST_INSTALL", $i);
1947     $reboot_script = set_test_option("REBOOT_SCRIPT", $i);
1948     $reboot_on_error = set_test_option("REBOOT_ON_ERROR", $i);
1949     $poweroff_on_error = set_test_option("POWEROFF_ON_ERROR", $i);
1950     $die_on_failure = set_test_option("DIE_ON_FAILURE", $i);
1951     $power_off = set_test_option("POWER_OFF", $i);
1952     $powercycle_after_reboot = set_test_option("POWERCYCLE_AFTER_REBOOT", $i);
1953     $poweroff_after_halt = set_test_option("POWEROFF_AFTER_HALT", $i);
1954     $sleep_time = set_test_option("SLEEP_TIME", $i);
1955     $bisect_sleep_time = set_test_option("BISECT_SLEEP_TIME", $i);
1956     $bisect_manual = set_test_option("BISECT_MANUAL", $i);
1957     $store_failures = set_test_option("STORE_FAILURES", $i);
1958     $timeout = set_test_option("TIMEOUT", $i);
1959     $booted_timeout = set_test_option("BOOTED_TIMEOUT", $i);
1960     $console = set_test_option("CONSOLE", $i);
1961     $success_line = set_test_option("SUCCESS_LINE", $i);
1962     $stop_after_success = set_test_option("STOP_AFTER_SUCCESS", $i);
1963     $stop_after_failure = set_test_option("STOP_AFTER_FAILURE", $i);
1964     $build_target = set_test_option("BUILD_TARGET", $i);
1965     $ssh_exec = set_test_option("SSH_EXEC", $i);
1966     $scp_to_target = set_test_option("SCP_TO_TARGET", $i);
1967     $target_image = set_test_option("TARGET_IMAGE", $i);
1968     $localversion = set_test_option("LOCALVERSION", $i);
1969
1970     chdir $builddir || die "can't change directory to $builddir";
1971
1972     if (!-d $tmpdir) {
1973         mkpath($tmpdir) or
1974             die "can't create $tmpdir";
1975     }
1976
1977     $ENV{"SSH_USER"} = $ssh_user;
1978     $ENV{"MACHINE"} = $machine;
1979
1980     $target = "$ssh_user\@$machine";
1981
1982     $buildlog = "$tmpdir/buildlog-$machine";
1983     $dmesg = "$tmpdir/dmesg-$machine";
1984     $make = "$makecmd O=$outputdir";
1985     $output_config = "$outputdir/.config";
1986
1987     if ($reboot_type eq "grub") {
1988         dodie "GRUB_MENU not defined" if (!defined($grub_menu));
1989     } elsif (!defined($reboot_script)) {
1990         dodie "REBOOT_SCRIPT not defined"
1991     }
1992
1993     my $run_type = $build_type;
1994     if ($test_type eq "patchcheck") {
1995         $run_type = $opt{"PATCHCHECK_TYPE[$i]"};
1996     } elsif ($test_type eq "bisect") {
1997         $run_type = $opt{"BISECT_TYPE[$i]"};
1998     } elsif ($test_type eq "config_bisect") {
1999         $run_type = $opt{"CONFIG_BISECT_TYPE[$i]"};
2000     }
2001
2002     # mistake in config file?
2003     if (!defined($run_type)) {
2004         $run_type = "ERROR";
2005     }
2006
2007     doprint "\n\n";
2008     doprint "RUNNING TEST $i of $opt{NUM_TESTS} with option $test_type $run_type\n\n";
2009
2010     unlink $dmesg;
2011     unlink $buildlog;
2012
2013     if (!defined($minconfig)) {
2014         $minconfig = $addconfig;
2015
2016     } elsif (defined($addconfig)) {
2017         run_command "cat $addconfig $minconfig > $tmpdir/add_config" or
2018             dodie "Failed to create temp config";
2019         $minconfig = "$tmpdir/add_config";
2020     }
2021
2022     my $checkout = $opt{"CHECKOUT[$i]"};
2023     if (defined($checkout)) {
2024         run_command "git checkout $checkout" or
2025             die "failed to checkout $checkout";
2026     }
2027
2028     if ($test_type eq "bisect") {
2029         bisect $i;
2030         next;
2031     } elsif ($test_type eq "config_bisect") {
2032         config_bisect $i;
2033         next;
2034     } elsif ($test_type eq "patchcheck") {
2035         patchcheck $i;
2036         next;
2037     }
2038
2039     if ($build_type ne "nobuild") {
2040         build $build_type or next;
2041     }
2042
2043     if ($test_type ne "build") {
2044         get_grub_index;
2045         get_version;
2046         install;
2047
2048         my $failed = 0;
2049         start_monitor;
2050         monitor or $failed = 1;;
2051
2052         if (!$failed && $test_type ne "boot" && defined($run_test)) {
2053             do_run_test or $failed = 1;
2054         }
2055         end_monitor;
2056         next if ($failed);
2057     }
2058
2059     success $i;
2060 }
2061
2062 if ($opt{"POWEROFF_ON_SUCCESS"}) {
2063     halt;
2064 } elsif ($opt{"REBOOT_ON_SUCCESS"} && !do_not_reboot) {
2065     reboot;
2066 }
2067
2068 doprint "\n    $successes of $opt{NUM_TESTS} tests were successful\n\n";
2069
2070 exit 0;