tools/testing/ktest/ktest.pl

   1 #!/usr/bin/perl -w
   2 #
   3 # Copywrite 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
   4 # Licensed under the terms of the GNU GPL License version 2
   5 #
   6
   7 use strict;
   8 use IPC::Open2;
   9 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
  10 use File::Path qw(mkpath);
  11 use File::Copy qw(cp);
  12 use FileHandle;
  13
  14 my $VERSION = "0.2";
  15
  16 $| = 1;
  17
  18 my %opt;
  19 my %repeat_tests;
  20 my %repeats;
  21 my %default;
  22
  23 #default opts
  24 $default{"NUM_TESTS"}           = 1;
  25 $default{"REBOOT_TYPE"}         = "grub";
  26 $default{"TEST_TYPE"}           = "test";
  27 $default{"BUILD_TYPE"}          = "randconfig";
  28 $default{"MAKE_CMD"}            = "make";
  29 $default{"TIMEOUT"}             = 120;
  30 $default{"TMP_DIR"}             = "/tmp/ktest";
  31 $default{"SLEEP_TIME"}          = 60;   # sleep time between tests
  32 $default{"BUILD_NOCLEAN"}       = 0;
  33 $default{"REBOOT_ON_ERROR"}     = 0;
  34 $default{"POWEROFF_ON_ERROR"}   = 0;
  35 $default{"REBOOT_ON_SUCCESS"}   = 1;
  36 $default{"POWEROFF_ON_SUCCESS"} = 0;
  37 $default{"BUILD_OPTIONS"}       = "";
  38 $default{"BISECT_SLEEP_TIME"}   = 60;   # sleep time between bisects
  39 $default{"CLEAR_LOG"}           = 0;
  40 $default{"SUCCESS_LINE"}        = "login:";
  41 $default{"BOOTED_TIMEOUT"}      = 1;
  42 $default{"DIE_ON_FAILURE"}      = 1;
  43 $default{"SSH_EXEC"}            = "ssh \$SSH_USER\@\$MACHINE \$SSH_COMMAND";
  44 $default{"SCP_TO_TARGET"}       = "scp \$SRC_FILE \$SSH_USER\@\$MACHINE:\$DST_FILE";
  45 $default{"REBOOT"}              = "ssh \$SSH_USER\@\$MACHINE reboot";
  46 $default{"STOP_AFTER_SUCCESS"}  = 10;
  47 $default{"STOP_AFTER_FAILURE"}  = 60;
  48 $default{"LOCALVERSION"}        = "-test";
  49
  50 my $ktest_config;
  51 my $version;
  52 my $machine;
  53 my $ssh_user;
  54 my $tmpdir;
  55 my $builddir;
  56 my $outputdir;
  57 my $output_config;
  58 my $test_type;
  59 my $build_type;
  60 my $build_options;
  61 my $reboot_type;
  62 my $reboot_script;
  63 my $power_cycle;
  64 my $reboot;
  65 my $reboot_on_error;
  66 my $poweroff_on_error;
  67 my $die_on_failure;
  68 my $powercycle_after_reboot;
  69 my $poweroff_after_halt;
  70 my $ssh_exec;
  71 my $scp_to_target;
  72 my $power_off;
  73 my $grub_menu;
  74 my $grub_number;
  75 my $target;
  76 my $make;
  77 my $post_install;
  78 my $noclean;
  79 my $minconfig;
  80 my $addconfig;
  81 my $in_bisect = 0;
  82 my $bisect_bad = "";
  83 my $reverse_bisect;
  84 my $in_patchcheck = 0;
  85 my $run_test;
  86 my $redirect;
  87 my $buildlog;
  88 my $dmesg;
  89 my $monitor_fp;
  90 my $monitor_pid;
  91 my $monitor_cnt = 0;
  92 my $sleep_time;
  93 my $bisect_sleep_time;
  94 my $store_failures;
  95 my $timeout;
  96 my $booted_timeout;
  97 my $console;
  98 my $success_line;
  99 my $stop_after_success;
 100 my $stop_after_failure;
 101 my $build_target;
 102 my $target_image;
 103 my $localversion;
 104 my $iteration = 0;
 105 my $successes = 0;
 106
 107 my %entered_configs;
 108 my %config_help;
 109
 110 $config_help{"MACHINE"} = << "EOF"
 111  The machine hostname that you will test.
 112 EOF
 113     ;
 114 $config_help{"SSH_USER"} = << "EOF"
 115  The box is expected to have ssh on normal bootup, provide the user
 116   (most likely root, since you need privileged operations)
 117 EOF
 118     ;
 119 $config_help{"BUILD_DIR"} = << "EOF"
 120  The directory that contains the Linux source code (full path).
 121 EOF
 122     ;
 123 $config_help{"OUTPUT_DIR"} = << "EOF"
 124  The directory that the objects will be built (full path).
 125  (can not be same as BUILD_DIR)
 126 EOF
 127     ;
 128 $config_help{"BUILD_TARGET"} = << "EOF"
 129  The location of the compiled file to copy to the target.
 130  (relative to OUTPUT_DIR)
 131 EOF
 132     ;
 133 $config_help{"TARGET_IMAGE"} = << "EOF"
 134  The place to put your image on the test machine.
 135 EOF
 136     ;
 137 $config_help{"POWER_CYCLE"} = << "EOF"
 138  A script or command to reboot the box.
 139
 140  Here is a digital loggers power switch example
 141  POWER_CYCLE = wget --no-proxy -O /dev/null -q  --auth-no-challenge 'http://admin:admin\@power/outlet?5=CCL'
 142
 143  Here is an example to reboot a virtual box on the current host
 144  with the name "Guest".
 145  POWER_CYCLE = virsh destroy Guest; sleep 5; virsh start Guest
 146 EOF
 147     ;
 148 $config_help{"CONSOLE"} = << "EOF"
 149  The script or command that reads the console
 150
 151   If you use ttywatch server, something like the following would work.
 152 CONSOLE = nc -d localhost 3001
 153
 154  For a virtual machine with guest name "Guest".
 155 CONSOLE =  virsh console Guest
 156 EOF
 157     ;
 158 $config_help{"LOCALVERSION"} = << "EOF"
 159  Required version ending to differentiate the test
 160  from other linux builds on the system.
 161 EOF
 162     ;
 163 $config_help{"REBOOT_TYPE"} = << "EOF"
 164  Way to reboot the box to the test kernel.
 165  Only valid options so far are "grub" and "script".
 166
 167  If you specify grub, it will assume grub version 1
 168  and will search in /boot/grub/menu.lst for the title \$GRUB_MENU
 169  and select that target to reboot to the kernel. If this is not
 170  your setup, then specify "script" and have a command or script
 171  specified in REBOOT_SCRIPT to boot to the target.
 172
 173  The entry in /boot/grub/menu.lst must be entered in manually.
 174  The test will not modify that file.
 175 EOF
 176     ;
 177 $config_help{"GRUB_MENU"} = << "EOF"
 178  The grub title name for the test kernel to boot
 179  (Only mandatory if REBOOT_TYPE = grub)
 180
 181  Note, ktest.pl will not update the grub menu.lst, you need to
 182  manually add an option for the test. ktest.pl will search
 183  the grub menu.lst for this option to find what kernel to
 184  reboot into.
 185
 186  For example, if in the /boot/grub/menu.lst the test kernel title has:
 187  title Test Kernel
 188  kernel vmlinuz-test
 189  GRUB_MENU = Test Kernel
 190 EOF
 191     ;
 192 $config_help{"REBOOT_SCRIPT"} = << "EOF"
 193  A script to reboot the target into the test kernel
 194  (Only mandatory if REBOOT_TYPE = script)
 195 EOF
 196     ;
 197
 198
 199 sub get_ktest_config {
 200     my ($config) = @_;
 201
 202     return if (defined($opt{$config}));
 203
 204     if (defined($config_help{$config})) {
 205         print "\n";
 206         print $config_help{$config};
 207     }
 208
 209     for (;;) {
 210         print "$config = ";
 211         if (defined($default{$config})) {
 212             print "\[$default{$config}\] ";
 213         }
 214         $entered_configs{$config} = <STDIN>;
 215         $entered_configs{$config} =~ s/^\s*(.*\S)\s*$/$1/;
 216         if ($entered_configs{$config} =~ /^\s*$/) {
 217             if ($default{$config}) {
 218                 $entered_configs{$config} = $default{$config};
 219             } else {
 220                 print "Your answer can not be blank\n";
 221                 next;
 222             }
 223         }
 224         last;
 225     }
 226 }
 227
 228 sub get_ktest_configs {
 229     get_ktest_config("MACHINE");
 230     get_ktest_config("SSH_USER");
 231     get_ktest_config("BUILD_DIR");
 232     get_ktest_config("OUTPUT_DIR");
 233     get_ktest_config("BUILD_TARGET");
 234     get_ktest_config("TARGET_IMAGE");
 235     get_ktest_config("POWER_CYCLE");
 236     get_ktest_config("CONSOLE");
 237     get_ktest_config("LOCALVERSION");
 238
 239     my $rtype = $opt{"REBOOT_TYPE"};
 240
 241     if (!defined($rtype)) {
 242         if (!defined($opt{"GRUB_MENU"})) {
 243             get_ktest_config("REBOOT_TYPE");
 244             $rtype = $entered_configs{"REBOOT_TYPE"};
 245         } else {
 246             $rtype = "grub";
 247         }
 248     }
 249
 250     if ($rtype eq "grub") {
 251         get_ktest_config("GRUB_MENU");
 252     } else {
 253         get_ktest_config("REBOOT_SCRIPT");
 254     }
 255 }
 256
 257 sub set_value {
 258     my ($lvalue, $rvalue) = @_;
 259
 260     if (defined($opt{$lvalue})) {
 261         die "Error: Option $lvalue defined more than once!\n";
 262     }
 263     if ($rvalue =~ /^\s*$/) {
 264         delete $opt{$lvalue};
 265     } else {
 266         $opt{$lvalue} = $rvalue;
 267     }
 268 }
 269
 270 sub read_config {
 271     my ($config) = @_;
 272
 273     open(IN, $config) || die "can't read file $config";
 274
 275     my $name = $config;
 276     $name =~ s,.*/(.*),$1,;
 277
 278     my $test_num = 0;
 279     my $default = 1;
 280     my $repeat = 1;
 281     my $num_tests_set = 0;
 282     my $skip = 0;
 283     my $rest;
 284
 285     while (<IN>) {
 286
 287         # ignore blank lines and comments
 288         next if (/^\s*$/ || /\s*\#/);
 289
 290         if (/^\s*TEST_START(.*)/) {
 291
 292             $rest = $1;
 293
 294             if ($num_tests_set) {
 295                 die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 296             }
 297
 298             my $old_test_num = $test_num;
 299             my $old_repeat = $repeat;
 300
 301             $test_num += $repeat;
 302             $default = 0;
 303             $repeat = 1;
 304
 305             if ($rest =~ /\s+SKIP(.*)/) {
 306                 $rest = $1;
 307                 $skip = 1;
 308             } else {
 309                 $skip = 0;
 310             }
 311
 312             if ($rest =~ /\s+ITERATE\s+(\d+)(.*)$/) {
 313                 $repeat = $1;
 314                 $rest = $2;
 315                 $repeat_tests{"$test_num"} = $repeat;
 316             }
 317
 318             if ($rest =~ /\s+SKIP(.*)/) {
 319                 $rest = $1;
 320                 $skip = 1;
 321             }
 322
 323             if ($rest !~ /^\s*$/) {
 324                 die "$name: $.: Gargbage found after TEST_START\n$_";
 325             }
 326
 327             if ($skip) {
 328                 $test_num = $old_test_num;
 329                 $repeat = $old_repeat;
 330             }
 331
 332         } elsif (/^\s*DEFAULTS(.*)$/) {
 333             $default = 1;
 334
 335             $rest = $1;
 336
 337             if ($rest =~ /\s+SKIP(.*)/) {
 338                 $rest = $1;
 339                 $skip = 1;
 340             } else {
 341                 $skip = 0;
 342             }
 343
 344             if ($rest !~ /^\s*$/) {
 345                 die "$name: $.: Gargbage found after DEFAULTS\n$_";
 346             }
 347
 348         } elsif (/^\s*([A-Z_\[\]\d]+)\s*=\s*(.*?)\s*$/) {
 349
 350             next if ($skip);
 351
 352             my $lvalue = $1;
 353             my $rvalue = $2;
 354
 355             if (!$default &&
 356                 ($lvalue eq "NUM_TESTS" ||
 357                  $lvalue eq "LOG_FILE" ||
 358                  $lvalue eq "CLEAR_LOG")) {
 359                 die "$name: $.: $lvalue must be set in DEFAULTS section\n";
 360             }
 361
 362             if ($lvalue eq "NUM_TESTS") {
 363                 if ($test_num) {
 364                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 365                 }
 366                 if (!$default) {
 367                     die "$name: $.: NUM_TESTS must be set in default section\n";
 368                 }
 369                 $num_tests_set = 1;
 370             }
 371
 372             if ($default || $lvalue =~ /\[\d+\]$/) {
 373                 set_value($lvalue, $rvalue);
 374             } else {
 375                 my $val = "$lvalue\[$test_num\]";
 376                 set_value($val, $rvalue);
 377
 378                 if ($repeat > 1) {
 379                     $repeats{$val} = $repeat;
 380                 }
 381             }
 382         } else {
 383             die "$name: $.: Garbage found in config\n$_";
 384         }
 385     }
 386
 387     close(IN);
 388
 389     if ($test_num) {
 390         $test_num += $repeat - 1;
 391         $opt{"NUM_TESTS"} = $test_num;
 392     }
 393
 394     # make sure we have all mandatory configs
 395     get_ktest_configs;
 396
 397     # set any defaults
 398
 399     foreach my $default (keys %default) {
 400         if (!defined($opt{$default})) {
 401             $opt{$default} = $default{$default};
 402         }
 403     }
 404 }
 405
 406 sub _logit {
 407     if (defined($opt{"LOG_FILE"})) {
 408         open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
 409         print OUT @_;
 410         close(OUT);
 411     }
 412 }
 413
 414 sub logit {
 415     if (defined($opt{"LOG_FILE"})) {
 416         _logit @_;
 417     } else {
 418         print @_;
 419     }
 420 }
 421
 422 sub doprint {
 423     print @_;
 424     _logit @_;
 425 }
 426
 427 sub run_command;
 428
 429 sub reboot {
 430     # try to reboot normally
 431     if (run_command $reboot) {
 432         if (defined($powercycle_after_reboot)) {
 433             sleep $powercycle_after_reboot;
 434             run_command "$power_cycle";
 435         }
 436     } else {
 437         # nope? power cycle it.
 438         run_command "$power_cycle";
 439     }
 440 }
 441
 442 sub do_not_reboot {
 443     my $i = $iteration;
 444
 445     return $test_type eq "build" ||
 446         ($test_type eq "patchcheck" && $opt{"PATCHCHECK_TYPE[$i]"} eq "build") ||
 447         ($test_type eq "bisect" && $opt{"BISECT_TYPE[$i]"} eq "build");
 448 }
 449
 450 sub dodie {
 451     doprint "CRITICAL FAILURE... ", @_, "\n";
 452
 453     my $i = $iteration;
 454
 455     if ($reboot_on_error && !do_not_reboot) {
 456
 457         doprint "REBOOTING\n";
 458         reboot;
 459
 460     } elsif ($poweroff_on_error && defined($power_off)) {
 461         doprint "POWERING OFF\n";
 462         `$power_off`;
 463     }
 464
 465     if (defined($opt{"LOG_FILE"})) {
 466         print " See $opt{LOG_FILE} for more info.\n";
 467     }
 468
 469     die @_, "\n";
 470 }
 471
 472 sub open_console {
 473     my ($fp) = @_;
 474
 475     my $flags;
 476
 477     my $pid = open($fp, "$console|") or
 478         dodie "Can't open console $console";
 479
 480     $flags = fcntl($fp, F_GETFL, 0) or
 481         dodie "Can't get flags for the socket: $!";
 482     $flags = fcntl($fp, F_SETFL, $flags | O_NONBLOCK) or
 483         dodie "Can't set flags for the socket: $!";
 484
 485     return $pid;
 486 }
 487
 488 sub close_console {
 489     my ($fp, $pid) = @_;
 490
 491     doprint "kill child process $pid\n";
 492     kill 2, $pid;
 493
 494     print "closing!\n";
 495     close($fp);
 496 }
 497
 498 sub start_monitor {
 499     if ($monitor_cnt++) {
 500         return;
 501     }
 502     $monitor_fp = \*MONFD;
 503     $monitor_pid = open_console $monitor_fp;
 504
 505     return;
 506
 507     open(MONFD, "Stop perl from warning about single use of MONFD");
 508 }
 509
 510 sub end_monitor {
 511     if (--$monitor_cnt) {
 512         return;
 513     }
 514     close_console($monitor_fp, $monitor_pid);
 515 }
 516
 517 sub wait_for_monitor {
 518     my ($time) = @_;
 519     my $line;
 520
 521     doprint "** Wait for monitor to settle down **\n";
 522
 523     # read the monitor and wait for the system to calm down
 524     do {
 525         $line = wait_for_input($monitor_fp, $time);
 526         print "$line" if (defined($line));
 527     } while (defined($line));
 528     print "** Monitor flushed **\n";
 529 }
 530
 531 sub fail {
 532
 533         if ($die_on_failure) {
 534                 dodie @_;
 535         }
 536
 537         doprint "FAILED\n";
 538
 539         my $i = $iteration;
 540
 541         # no need to reboot for just building.
 542         if (!do_not_reboot) {
 543             doprint "REBOOTING\n";
 544             reboot;
 545             start_monitor;
 546             wait_for_monitor $sleep_time;
 547             end_monitor;
 548         }
 549
 550         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 551         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 552         doprint "KTEST RESULT: TEST $i Failed: ", @_, "\n";
 553         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 554         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 555
 556         return 1 if (!defined($store_failures));
 557
 558         my @t = localtime;
 559         my $date = sprintf "%04d%02d%02d%02d%02d%02d",
 560                 1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
 561
 562         my $type = $build_type;
 563         if ($type =~ /useconfig/) {
 564             $type = "useconfig";
 565         }
 566
 567         my $dir = "$machine-$test_type-$type-fail-$date";
 568         my $faildir = "$store_failures/$dir";
 569
 570         if (!-d $faildir) {
 571             mkpath($faildir) or
 572                 die "can't create $faildir";
 573         }
 574         if (-f "$output_config") {
 575             cp "$output_config", "$faildir/config" or
 576                 die "failed to copy .config";
 577         }
 578         if (-f $buildlog) {
 579             cp $buildlog, "$faildir/buildlog" or
 580                 die "failed to move $buildlog";
 581         }
 582         if (-f $dmesg) {
 583             cp $dmesg, "$faildir/dmesg" or
 584                 die "failed to move $dmesg";
 585         }
 586
 587         doprint "*** Saved info to $faildir ***\n";
 588
 589         return 1;
 590 }
 591
 592 sub run_command {
 593     my ($command) = @_;
 594     my $dolog = 0;
 595     my $dord = 0;
 596     my $pid;
 597
 598     $command =~ s/\$SSH_USER/$ssh_user/g;
 599     $command =~ s/\$MACHINE/$machine/g;
 600
 601     doprint("$command ... ");
 602
 603     $pid = open(CMD, "$command 2>&1 |") or
 604         (fail "unable to exec $command" and return 0);
 605
 606     if (defined($opt{"LOG_FILE"})) {
 607         open(LOG, ">>$opt{LOG_FILE}") or
 608             dodie "failed to write to log";
 609         $dolog = 1;
 610     }
 611
 612     if (defined($redirect)) {
 613         open (RD, ">$redirect") or
 614             dodie "failed to write to redirect $redirect";
 615         $dord = 1;
 616     }
 617
 618     while (<CMD>) {
 619         print LOG if ($dolog);
 620         print RD  if ($dord);
 621     }
 622
 623     waitpid($pid, 0);
 624     my $failed = $?;
 625
 626     close(CMD);
 627     close(LOG) if ($dolog);
 628     close(RD)  if ($dord);
 629
 630     if ($failed) {
 631         doprint "FAILED!\n";
 632     } else {
 633         doprint "SUCCESS\n";
 634     }
 635
 636     return !$failed;
 637 }
 638
 639 sub run_ssh {
 640     my ($cmd) = @_;
 641     my $cp_exec = $ssh_exec;
 642
 643     $cp_exec =~ s/\$SSH_COMMAND/$cmd/g;
 644     return run_command "$cp_exec";
 645 }
 646
 647 sub run_scp {
 648     my ($src, $dst) = @_;
 649     my $cp_scp = $scp_to_target;
 650
 651     $cp_scp =~ s/\$SRC_FILE/$src/g;
 652     $cp_scp =~ s/\$DST_FILE/$dst/g;
 653
 654     return run_command "$cp_scp";
 655 }
 656
 657 sub get_grub_index {
 658
 659     if ($reboot_type ne "grub") {
 660         return;
 661     }
 662     return if (defined($grub_number));
 663
 664     doprint "Find grub menu ... ";
 665     $grub_number = -1;
 666
 667     my $ssh_grub = $ssh_exec;
 668     $ssh_grub =~ s,\$SSH_COMMAND,cat /boot/grub/menu.lst,g;
 669
 670     open(IN, "$ssh_grub |")
 671         or die "unable to get menu.lst";
 672
 673     while (<IN>) {
 674         if (/^\s*title\s+$grub_menu\s*$/) {
 675             $grub_number++;
 676             last;
 677         } elsif (/^\s*title\s/) {
 678             $grub_number++;
 679         }
 680     }
 681     close(IN);
 682
 683     die "Could not find '$grub_menu' in /boot/grub/menu on $machine"
 684         if ($grub_number < 0);
 685     doprint "$grub_number\n";
 686 }
 687
 688 sub wait_for_input
 689 {
 690     my ($fp, $time) = @_;
 691     my $rin;
 692     my $ready;
 693     my $line;
 694     my $ch;
 695
 696     if (!defined($time)) {
 697         $time = $timeout;
 698     }
 699
 700     $rin = '';
 701     vec($rin, fileno($fp), 1) = 1;
 702     $ready = select($rin, undef, undef, $time);
 703
 704     $line = "";
 705
 706     # try to read one char at a time
 707     while (sysread $fp, $ch, 1) {
 708         $line .= $ch;
 709         last if ($ch eq "\n");
 710     }
 711
 712     if (!length($line)) {
 713         return undef;
 714     }
 715
 716     return $line;
 717 }
 718
 719 sub reboot_to {
 720     if ($reboot_type eq "grub") {
 721         run_ssh "'(echo \"savedefault --default=$grub_number --once\" | grub --batch; reboot)'";
 722         return;
 723     }
 724
 725     run_command "$reboot_script";
 726 }
 727
 728 sub get_sha1 {
 729     my ($commit) = @_;
 730
 731     doprint "git rev-list --max-count=1 $commit ... ";
 732     my $sha1 = `git rev-list --max-count=1 $commit`;
 733     my $ret = $?;
 734
 735     logit $sha1;
 736
 737     if ($ret) {
 738         doprint "FAILED\n";
 739         dodie "Failed to get git $commit";
 740     }
 741
 742     print "SUCCESS\n";
 743
 744     chomp $sha1;
 745
 746     return $sha1;
 747 }
 748
 749 sub monitor {
 750     my $booted = 0;
 751     my $bug = 0;
 752     my $skip_call_trace = 0;
 753     my $loops;
 754
 755     wait_for_monitor 5;
 756
 757     my $line;
 758     my $full_line = "";
 759
 760     open(DMESG, "> $dmesg") or
 761         die "unable to write to $dmesg";
 762
 763     reboot_to;
 764
 765     my $success_start;
 766     my $failure_start;
 767
 768     for (;;) {
 769
 770         if ($booted) {
 771             $line = wait_for_input($monitor_fp, $booted_timeout);
 772         } else {
 773             $line = wait_for_input($monitor_fp);
 774         }
 775
 776         last if (!defined($line));
 777
 778         doprint $line;
 779         print DMESG $line;
 780
 781         # we are not guaranteed to get a full line
 782         $full_line .= $line;
 783
 784         if ($full_line =~ /$success_line/) {
 785             $booted = 1;
 786             $success_start = time;
 787         }
 788
 789         if ($booted && defined($stop_after_success) &&
 790             $stop_after_success >= 0) {
 791             my $now = time;
 792             if ($now - $success_start >= $stop_after_success) {
 793                 doprint "Test forced to stop after $stop_after_success seconds after success\n";
 794                 last;
 795             }
 796         }
 797
 798         if ($full_line =~ /\[ backtrace testing \]/) {
 799             $skip_call_trace = 1;
 800         }
 801
 802         if ($full_line =~ /call trace:/i) {
 803             if (!$skip_call_trace) {
 804                 $bug = 1;
 805                 $failure_start = time;
 806             }
 807         }
 808
 809         if ($bug && defined($stop_after_failure) &&
 810             $stop_after_failure >= 0) {
 811             my $now = time;
 812             if ($now - $failure_start >= $stop_after_failure) {
 813                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
 814                 last;
 815             }
 816         }
 817
 818         if ($full_line =~ /\[ end of backtrace testing \]/) {
 819             $skip_call_trace = 0;
 820         }
 821
 822         if ($full_line =~ /Kernel panic -/) {
 823             $bug = 1;
 824         }
 825
 826         if ($line =~ /\n/) {
 827             $full_line = "";
 828         }
 829     }
 830
 831     close(DMESG);
 832
 833     if ($bug) {
 834         return 0 if ($in_bisect);
 835         fail "failed - got a bug report" and return 0;
 836     }
 837
 838     if (!$booted) {
 839         return 0 if ($in_bisect);
 840         fail "failed - never got a boot prompt." and return 0;
 841     }
 842
 843     return 1;
 844 }
 845
 846 sub install {
 847
 848     run_scp "$outputdir/$build_target", "$target_image" or
 849         dodie "failed to copy image";
 850
 851     my $install_mods = 0;
 852
 853     # should we process modules?
 854     $install_mods = 0;
 855     open(IN, "$output_config") or dodie("Can't read config file");
 856     while (<IN>) {
 857         if (/CONFIG_MODULES(=y)?/) {
 858             $install_mods = 1 if (defined($1));
 859             last;
 860         }
 861     }
 862     close(IN);
 863
 864     if (!$install_mods) {
 865         doprint "No modules needed\n";
 866         return;
 867     }
 868
 869     run_command "$make INSTALL_MOD_PATH=$tmpdir modules_install" or
 870         dodie "Failed to install modules";
 871
 872     my $modlib = "/lib/modules/$version";
 873     my $modtar = "ktest-mods.tar.bz2";
 874
 875     run_ssh "rm -rf $modlib" or
 876         dodie "failed to remove old mods: $modlib";
 877
 878     # would be nice if scp -r did not follow symbolic links
 879     run_command "cd $tmpdir && tar -cjf $modtar lib/modules/$version" or
 880         dodie "making tarball";
 881
 882     run_scp "$tmpdir/$modtar", "/tmp" or
 883         dodie "failed to copy modules";
 884
 885     unlink "$tmpdir/$modtar";
 886
 887     run_ssh "'(cd / && tar xf /tmp/$modtar)'" or
 888         dodie "failed to tar modules";
 889
 890     run_ssh "rm -f /tmp/$modtar";
 891
 892     return if (!defined($post_install));
 893
 894     my $cp_post_install = $post_install;
 895     $cp_post_install = s/\$KERNEL_VERSION/$version/g;
 896     run_command "$cp_post_install" or
 897         dodie "Failed to run post install";
 898 }
 899
 900 sub check_buildlog {
 901     my ($patch) = @_;
 902
 903     my @files = `git show $patch | diffstat -l`;
 904
 905     open(IN, "git show $patch |") or
 906         dodie "failed to show $patch";
 907     while (<IN>) {
 908         if (m,^--- a/(.*),) {
 909             chomp $1;
 910             $files[$#files] = $1;
 911         }
 912     }
 913     close(IN);
 914
 915     open(IN, $buildlog) or dodie "Can't open $buildlog";
 916     while (<IN>) {
 917         if (/^\s*(.*?):.*(warning|error)/) {
 918             my $err = $1;
 919             foreach my $file (@files) {
 920                 my $fullpath = "$builddir/$file";
 921                 if ($file eq $err || $fullpath eq $err) {
 922                     fail "$file built with warnings" and return 0;
 923                 }
 924             }
 925         }
 926     }
 927     close(IN);
 928
 929     return 1;
 930 }
 931
 932 sub build {
 933     my ($type) = @_;
 934     my $defconfig = "";
 935
 936     unlink $buildlog;
 937
 938     if ($type =~ /^useconfig:(.*)/) {
 939         run_command "cp $1 $output_config" or
 940             dodie "could not copy $1 to .config";
 941
 942         $type = "oldconfig";
 943     }
 944
 945     # old config can ask questions
 946     if ($type eq "oldconfig") {
 947         $type = "oldnoconfig";
 948
 949         # allow for empty configs
 950         run_command "touch $output_config";
 951
 952         run_command "mv $output_config $outputdir/config_temp" or
 953             dodie "moving .config";
 954
 955         if (!$noclean && !run_command "$make mrproper") {
 956             dodie "make mrproper";
 957         }
 958
 959         run_command "mv $outputdir/config_temp $output_config" or
 960             dodie "moving config_temp";
 961
 962     } elsif (!$noclean) {
 963         unlink "$output_config";
 964         run_command "$make mrproper" or
 965             dodie "make mrproper";
 966     }
 967
 968     # add something to distinguish this build
 969     open(OUT, "> $outputdir/localversion") or dodie("Can't make localversion file");
 970     print OUT "$localversion\n";
 971     close(OUT);
 972
 973     if (defined($minconfig)) {
 974         $defconfig = "KCONFIG_ALLCONFIG=$minconfig";
 975     }
 976
 977     run_command "$defconfig $make $type" or
 978         dodie "failed make config";
 979
 980     $redirect = "$buildlog";
 981     if (!run_command "$make $build_options") {
 982         undef $redirect;
 983         # bisect may need this to pass
 984         return 0 if ($in_bisect);
 985         fail "failed build" and return 0;
 986     }
 987     undef $redirect;
 988
 989     return 1;
 990 }
 991
 992 sub halt {
 993     if (!run_ssh "halt" or defined($power_off)) {
 994         if (defined($poweroff_after_halt)) {
 995             sleep $poweroff_after_halt;
 996             run_command "$power_off";
 997         }
 998     } else {
 999         # nope? the zap it!
1000         run_command "$power_off";
1001     }
1002 }
1003
1004 sub success {
1005     my ($i) = @_;
1006
1007     $successes++;
1008
1009     doprint "\n\n*******************************************\n";
1010     doprint     "*******************************************\n";
1011     doprint     "KTEST RESULT: TEST $i SUCCESS!!!!         **\n";
1012     doprint     "*******************************************\n";
1013     doprint     "*******************************************\n";
1014
1015     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
1016         doprint "Reboot and wait $sleep_time seconds\n";
1017         reboot;
1018         start_monitor;
1019         wait_for_monitor $sleep_time;
1020         end_monitor;
1021     }
1022 }
1023
1024 sub get_version {
1025     # get the release name
1026     doprint "$make kernelrelease ... ";
1027     $version = `$make kernelrelease | tail -1`;
1028     chomp($version);
1029     doprint "$version\n";
1030 }
1031
1032 sub child_run_test {
1033     my $failed = 0;
1034
1035     # child should have no power
1036     $reboot_on_error = 0;
1037     $poweroff_on_error = 0;
1038     $die_on_failure = 1;
1039
1040     run_command $run_test or $failed = 1;
1041     exit $failed;
1042 }
1043
1044 my $child_done;
1045
1046 sub child_finished {
1047     $child_done = 1;
1048 }
1049
1050 sub do_run_test {
1051     my $child_pid;
1052     my $child_exit;
1053     my $line;
1054     my $full_line;
1055     my $bug = 0;
1056
1057     wait_for_monitor 1;
1058
1059     doprint "run test $run_test\n";
1060
1061     $child_done = 0;
1062
1063     $SIG{CHLD} = qw(child_finished);
1064
1065     $child_pid = fork;
1066
1067     child_run_test if (!$child_pid);
1068
1069     $full_line = "";
1070
1071     do {
1072         $line = wait_for_input($monitor_fp, 1);
1073         if (defined($line)) {
1074
1075             # we are not guaranteed to get a full line
1076             $full_line .= $line;
1077
1078             if ($full_line =~ /call trace:/i) {
1079                 $bug = 1;
1080             }
1081
1082             if ($full_line =~ /Kernel panic -/) {
1083                 $bug = 1;
1084             }
1085
1086             if ($line =~ /\n/) {
1087                 $full_line = "";
1088             }
1089         }
1090     } while (!$child_done && !$bug);
1091
1092     if ($bug) {
1093         doprint "Detected kernel crash!\n";
1094         # kill the child with extreme prejudice
1095         kill 9, $child_pid;
1096     }
1097
1098     waitpid $child_pid, 0;
1099     $child_exit = $?;
1100
1101     if ($bug || $child_exit) {
1102         return 0 if $in_bisect;
1103         fail "test failed" and return 0;
1104     }
1105     return 1;
1106 }
1107
1108 sub run_git_bisect {
1109     my ($command) = @_;
1110
1111     doprint "$command ... ";
1112
1113     my $output = `$command 2>&1`;
1114     my $ret = $?;
1115
1116     logit $output;
1117
1118     if ($ret) {
1119         doprint "FAILED\n";
1120         dodie "Failed to git bisect";
1121     }
1122
1123     doprint "SUCCESS\n";
1124     if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
1125         doprint "$1 [$2]\n";
1126     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
1127         $bisect_bad = $1;
1128         doprint "Found bad commit... $1\n";
1129         return 0;
1130     } else {
1131         # we already logged it, just print it now.
1132         print $output;
1133     }
1134
1135     return 1;
1136 }
1137
1138 # returns 1 on success, 0 on failure
1139 sub run_bisect_test {
1140     my ($type, $buildtype) = @_;
1141
1142     my $failed = 0;
1143     my $result;
1144     my $output;
1145     my $ret;
1146
1147     $in_bisect = 1;
1148
1149     build $buildtype or $failed = 1;
1150
1151     if ($type ne "build") {
1152         dodie "Failed on build" if $failed;
1153
1154         # Now boot the box
1155         get_grub_index;
1156         get_version;
1157         install;
1158
1159         start_monitor;
1160         monitor or $failed = 1;
1161
1162         if ($type ne "boot") {
1163             dodie "Failed on boot" if $failed;
1164
1165             do_run_test or $failed = 1;
1166         }
1167         end_monitor;
1168     }
1169
1170     if ($failed) {
1171         $result = 0;
1172
1173         # reboot the box to a good kernel
1174         if ($type ne "build") {
1175             doprint "Reboot and sleep $bisect_sleep_time seconds\n";
1176             reboot;
1177             start_monitor;
1178             wait_for_monitor $bisect_sleep_time;
1179             end_monitor;
1180         }
1181     } else {
1182         $result = 1;
1183     }
1184     $in_bisect = 0;
1185
1186     return $result;
1187 }
1188
1189 sub run_bisect {
1190     my ($type) = @_;
1191     my $buildtype = "oldconfig";
1192
1193     # We should have a minconfig to use?
1194     if (defined($minconfig)) {
1195         $buildtype = "useconfig:$minconfig";
1196     }
1197
1198     my $ret = run_bisect_test $type, $buildtype;
1199
1200
1201     # Are we looking for where it worked, not failed?
1202     if ($reverse_bisect) {
1203         $ret = !$ret;
1204     }
1205
1206     if ($ret) {
1207         return "good";
1208     } else {
1209         return  "bad";
1210     }
1211 }
1212
1213 sub bisect {
1214     my ($i) = @_;
1215
1216     my $result;
1217
1218     die "BISECT_GOOD[$i] not defined\n" if (!defined($opt{"BISECT_GOOD[$i]"}));
1219     die "BISECT_BAD[$i] not defined\n"  if (!defined($opt{"BISECT_BAD[$i]"}));
1220     die "BISECT_TYPE[$i] not defined\n" if (!defined($opt{"BISECT_TYPE[$i]"}));
1221
1222     my $good = $opt{"BISECT_GOOD[$i]"};
1223     my $bad = $opt{"BISECT_BAD[$i]"};
1224     my $type = $opt{"BISECT_TYPE[$i]"};
1225     my $start = $opt{"BISECT_START[$i]"};
1226     my $replay = $opt{"BISECT_REPLAY[$i]"};
1227
1228     # convert to true sha1's
1229     $good = get_sha1($good);
1230     $bad = get_sha1($bad);
1231
1232     if (defined($opt{"BISECT_REVERSE[$i]"}) &&
1233         $opt{"BISECT_REVERSE[$i]"} == 1) {
1234         doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
1235         $reverse_bisect = 1;
1236     } else {
1237         $reverse_bisect = 0;
1238     }
1239
1240     # Can't have a test without having a test to run
1241     if ($type eq "test" && !defined($run_test)) {
1242         $type = "boot";
1243     }
1244
1245     my $check = $opt{"BISECT_CHECK[$i]"};
1246     if (defined($check) && $check ne "0") {
1247
1248         # get current HEAD
1249         my $head = get_sha1("HEAD");
1250
1251         if ($check ne "good") {
1252             doprint "TESTING BISECT BAD [$bad]\n";
1253             run_command "git checkout $bad" or
1254                 die "Failed to checkout $bad";
1255
1256             $result = run_bisect $type;
1257
1258             if ($result ne "bad") {
1259                 fail "Tested BISECT_BAD [$bad] and it succeeded" and return 0;
1260             }
1261         }
1262
1263         if ($check ne "bad") {
1264             doprint "TESTING BISECT GOOD [$good]\n";
1265             run_command "git checkout $good" or
1266                 die "Failed to checkout $good";
1267
1268             $result = run_bisect $type;
1269
1270             if ($result ne "good") {
1271                 fail "Tested BISECT_GOOD [$good] and it failed" and return 0;
1272             }
1273         }
1274
1275         # checkout where we started
1276         run_command "git checkout $head" or
1277             die "Failed to checkout $head";
1278     }
1279
1280     run_command "git bisect start" or
1281         dodie "could not start bisect";
1282
1283     run_command "git bisect good $good" or
1284         dodie "could not set bisect good to $good";
1285
1286     run_git_bisect "git bisect bad $bad" or
1287         dodie "could not set bisect bad to $bad";
1288
1289     if (defined($replay)) {
1290         run_command "git bisect replay $replay" or
1291             dodie "failed to run replay";
1292     }
1293
1294     if (defined($start)) {
1295         run_command "git checkout $start" or
1296             dodie "failed to checkout $start";
1297     }
1298
1299     my $test;
1300     do {
1301         $result = run_bisect $type;
1302         $test = run_git_bisect "git bisect $result";
1303     } while ($test);
1304
1305     run_command "git bisect log" or
1306         dodie "could not capture git bisect log";
1307
1308     run_command "git bisect reset" or
1309         dodie "could not reset git bisect";
1310
1311     doprint "Bad commit was [$bisect_bad]\n";
1312
1313     success $i;
1314 }
1315
1316 my %config_ignore;
1317 my %config_set;
1318
1319 my %config_list;
1320 my %null_config;
1321
1322 my %dependency;
1323
1324 sub process_config_ignore {
1325     my ($config) = @_;
1326
1327     open (IN, $config)
1328         or dodie "Failed to read $config";
1329
1330     while (<IN>) {
1331         if (/^(.*?(CONFIG\S*)(=.*| is not set))/) {
1332             $config_ignore{$2} = $1;
1333         }
1334     }
1335
1336     close(IN);
1337 }
1338
1339 sub read_current_config {
1340     my ($config_ref) = @_;
1341
1342     %{$config_ref} = ();
1343     undef %{$config_ref};
1344
1345     my @key = keys %{$config_ref};
1346     if ($#key >= 0) {
1347         print "did not delete!\n";
1348         exit;
1349     }
1350     open (IN, "$output_config");
1351
1352     while (<IN>) {
1353         if (/^(CONFIG\S+)=(.*)/) {
1354             ${$config_ref}{$1} = $2;
1355         }
1356     }
1357     close(IN);
1358 }
1359
1360 sub get_dependencies {
1361     my ($config) = @_;
1362
1363     my $arr = $dependency{$config};
1364     if (!defined($arr)) {
1365         return ();
1366     }
1367
1368     my @deps = @{$arr};
1369
1370     foreach my $dep (@{$arr}) {
1371         print "ADD DEP $dep\n";
1372         @deps = (@deps, get_dependencies $dep);
1373     }
1374
1375     return @deps;
1376 }
1377
1378 sub create_config {
1379     my @configs = @_;
1380
1381     open(OUT, ">$output_config") or dodie "Can not write to $output_config";
1382
1383     foreach my $config (@configs) {
1384         print OUT "$config_set{$config}\n";
1385         my @deps = get_dependencies $config;
1386         foreach my $dep (@deps) {
1387             print OUT "$config_set{$dep}\n";
1388         }
1389     }
1390
1391     foreach my $config (keys %config_ignore) {
1392         print OUT "$config_ignore{$config}\n";
1393     }
1394     close(OUT);
1395
1396 #    exit;
1397     run_command "$make oldnoconfig" or
1398         dodie "failed make config oldconfig";
1399
1400 }
1401
1402 sub compare_configs {
1403     my (%a, %b) = @_;
1404
1405     foreach my $item (keys %a) {
1406         if (!defined($b{$item})) {
1407             print "diff $item\n";
1408             return 1;
1409         }
1410         delete $b{$item};
1411     }
1412
1413     my @keys = keys %b;
1414     if ($#keys) {
1415         print "diff2 $keys[0]\n";
1416     }
1417     return -1 if ($#keys >= 0);
1418
1419     return 0;
1420 }
1421
1422 sub run_config_bisect_test {
1423     my ($type) = @_;
1424
1425     return run_bisect_test $type, "oldconfig";
1426 }
1427
1428 sub process_passed {
1429     my (%configs) = @_;
1430
1431     doprint "These configs had no failure: (Enabling them for further compiles)\n";
1432     # Passed! All these configs are part of a good compile.
1433     # Add them to the min options.
1434     foreach my $config (keys %configs) {
1435         if (defined($config_list{$config})) {
1436             doprint " removing $config\n";
1437             $config_ignore{$config} = $config_list{$config};
1438             delete $config_list{$config};
1439         }
1440     }
1441     doprint "config copied to $outputdir/config_good\n";
1442     run_command "cp -f $output_config $outputdir/config_good";
1443 }
1444
1445 sub process_failed {
1446     my ($config) = @_;
1447
1448     doprint "\n\n***************************************\n";
1449     doprint "Found bad config: $config\n";
1450     doprint "***************************************\n\n";
1451 }
1452
1453 sub run_config_bisect {
1454
1455     my @start_list = keys %config_list;
1456
1457     if ($#start_list < 0) {
1458         doprint "No more configs to test!!!\n";
1459         return -1;
1460     }
1461
1462     doprint "***** RUN TEST ***\n";
1463     my $type = $opt{"CONFIG_BISECT_TYPE[$iteration]"};
1464     my $ret;
1465     my %current_config;
1466
1467     my $count = $#start_list + 1;
1468     doprint "  $count configs to test\n";
1469
1470     my $half = int($#start_list / 2);
1471
1472     do {
1473         my @tophalf = @start_list[0 .. $half];
1474
1475         create_config @tophalf;
1476         read_current_config \%current_config;
1477
1478         $count = $#tophalf + 1;
1479         doprint "Testing $count configs\n";
1480         my $found = 0;
1481         # make sure we test something
1482         foreach my $config (@tophalf) {
1483             if (defined($current_config{$config})) {
1484                 logit " $config\n";
1485                 $found = 1;
1486             }
1487         }
1488         if (!$found) {
1489             # try the other half
1490             doprint "Top half produced no set configs, trying bottom half\n";
1491             @tophalf = @start_list[$half .. $#start_list];
1492             create_config @tophalf;
1493             read_current_config \%current_config;
1494             foreach my $config (@tophalf) {
1495                 if (defined($current_config{$config})) {
1496                     logit " $config\n";
1497                     $found = 1;
1498                 }
1499             }
1500             if (!$found) {
1501                 doprint "Failed: Can't make new config with current configs\n";
1502                 foreach my $config (@start_list) {
1503                     doprint "  CONFIG: $config\n";
1504                 }
1505                 return -1;
1506             }
1507             $count = $#tophalf + 1;
1508             doprint "Testing $count configs\n";
1509         }
1510
1511         $ret = run_config_bisect_test $type;
1512
1513         if ($ret) {
1514             process_passed %current_config;
1515             return 0;
1516         }
1517
1518         doprint "This config had a failure.\n";
1519         doprint "Removing these configs that were not set in this config:\n";
1520         doprint "config copied to $outputdir/config_bad\n";
1521         run_command "cp -f $output_config $outputdir/config_bad";
1522
1523         # A config exists in this group that was bad.
1524         foreach my $config (keys %config_list) {
1525             if (!defined($current_config{$config})) {
1526                 doprint " removing $config\n";
1527                 delete $config_list{$config};
1528             }
1529         }
1530
1531         @start_list = @tophalf;
1532
1533         if ($#start_list == 0) {
1534             process_failed $start_list[0];
1535             return 1;
1536         }
1537
1538         # remove half the configs we are looking at and see if
1539         # they are good.
1540         $half = int($#start_list / 2);
1541     } while ($half > 0);
1542
1543     # we found a single config, try it again
1544     my @tophalf = @start_list[0 .. 0];
1545
1546     $ret = run_config_bisect_test $type;
1547     if ($ret) {
1548         process_passed %current_config;
1549         return 0;
1550     }
1551
1552     process_failed $start_list[0];
1553     return 1;
1554 }
1555
1556 sub config_bisect {
1557     my ($i) = @_;
1558
1559     my $start_config = $opt{"CONFIG_BISECT[$i]"};
1560
1561     my $tmpconfig = "$tmpdir/use_config";
1562
1563     # Make the file with the bad config and the min config
1564     if (defined($minconfig)) {
1565         # read the min config for things to ignore
1566         run_command "cp $minconfig $tmpconfig" or
1567             dodie "failed to copy $minconfig to $tmpconfig";
1568     } else {
1569         unlink $tmpconfig;
1570     }
1571
1572     # Add other configs
1573     if (defined($addconfig)) {
1574         run_command "cat $addconfig >> $tmpconfig" or
1575             dodie "failed to append $addconfig";
1576     }
1577
1578     my $defconfig = "";
1579     if (-f $tmpconfig) {
1580         $defconfig = "KCONFIG_ALLCONFIG=$tmpconfig";
1581         process_config_ignore $tmpconfig;
1582     }
1583
1584     # now process the start config
1585     run_command "cp $start_config $output_config" or
1586         dodie "failed to copy $start_config to $output_config";
1587
1588     # read directly what we want to check
1589     my %config_check;
1590     open (IN, $output_config)
1591         or dodie "faied to open $output_config";
1592
1593     while (<IN>) {
1594         if (/^((CONFIG\S*)=.*)/) {
1595             $config_check{$2} = $1;
1596         }
1597     }
1598     close(IN);
1599
1600     # Now run oldconfig with the minconfig (and addconfigs)
1601     run_command "$defconfig $make oldnoconfig" or
1602         dodie "failed make config oldconfig";
1603
1604     # check to see what we lost (or gained)
1605     open (IN, $output_config)
1606         or dodie "Failed to read $start_config";
1607
1608     my %removed_configs;
1609     my %added_configs;
1610
1611     while (<IN>) {
1612         if (/^((CONFIG\S*)=.*)/) {
1613             # save off all options
1614             $config_set{$2} = $1;
1615             if (defined($config_check{$2})) {
1616                 if (defined($config_ignore{$2})) {
1617                     $removed_configs{$2} = $1;
1618                 } else {
1619                     $config_list{$2} = $1;
1620                 }
1621             } elsif (!defined($config_ignore{$2})) {
1622                 $added_configs{$2} = $1;
1623                 $config_list{$2} = $1;
1624             }
1625         }
1626     }
1627     close(IN);
1628
1629     my @confs = keys %removed_configs;
1630     if ($#confs >= 0) {
1631         doprint "Configs overridden by default configs and removed from check:\n";
1632         foreach my $config (@confs) {
1633             doprint " $config\n";
1634         }
1635     }
1636     @confs = keys %added_configs;
1637     if ($#confs >= 0) {
1638         doprint "Configs appearing in make oldconfig and added:\n";
1639         foreach my $config (@confs) {
1640             doprint " $config\n";
1641         }
1642     }
1643
1644     my %config_test;
1645     my $once = 0;
1646
1647     # Sometimes kconfig does weird things. We must make sure
1648     # that the config we autocreate has everything we need
1649     # to test, otherwise we may miss testing configs, or
1650     # may not be able to create a new config.
1651     # Here we create a config with everything set.
1652     create_config (keys %config_list);
1653     read_current_config \%config_test;
1654     foreach my $config (keys %config_list) {
1655         if (!defined($config_test{$config})) {
1656             if (!$once) {
1657                 $once = 1;
1658                 doprint "Configs not produced by kconfig (will not be checked):\n";
1659             }
1660             doprint "  $config\n";
1661             delete $config_list{$config};
1662         }
1663     }
1664     my $ret;
1665     do {
1666         $ret = run_config_bisect;
1667     } while (!$ret);
1668
1669     return $ret if ($ret < 0);
1670
1671     success $i;
1672 }
1673
1674 sub patchcheck {
1675     my ($i) = @_;
1676
1677     die "PATCHCHECK_START[$i] not defined\n"
1678         if (!defined($opt{"PATCHCHECK_START[$i]"}));
1679     die "PATCHCHECK_TYPE[$i] not defined\n"
1680         if (!defined($opt{"PATCHCHECK_TYPE[$i]"}));
1681
1682     my $start = $opt{"PATCHCHECK_START[$i]"};
1683
1684     my $end = "HEAD";
1685     if (defined($opt{"PATCHCHECK_END[$i]"})) {
1686         $end = $opt{"PATCHCHECK_END[$i]"};
1687     }
1688
1689     # Get the true sha1's since we can use things like HEAD~3
1690     $start = get_sha1($start);
1691     $end = get_sha1($end);
1692
1693     my $type = $opt{"PATCHCHECK_TYPE[$i]"};
1694
1695     # Can't have a test without having a test to run
1696     if ($type eq "test" && !defined($run_test)) {
1697         $type = "boot";
1698     }
1699
1700     open (IN, "git log --pretty=oneline $end|") or
1701         dodie "could not get git list";
1702
1703     my @list;
1704
1705     while (<IN>) {
1706         chomp;
1707         $list[$#list+1] = $_;
1708         last if (/^$start/);
1709     }
1710     close(IN);
1711
1712     if ($list[$#list] !~ /^$start/) {
1713         fail "SHA1 $start not found";
1714     }
1715
1716     # go backwards in the list
1717     @list = reverse @list;
1718
1719     my $save_clean = $noclean;
1720
1721     $in_patchcheck = 1;
1722     foreach my $item (@list) {
1723         my $sha1 = $item;
1724         $sha1 =~ s/^([[:xdigit:]]+).*/$1/;
1725
1726         doprint "\nProcessing commit $item\n\n";
1727
1728         run_command "git checkout $sha1" or
1729             die "Failed to checkout $sha1";
1730
1731         # only clean on the first and last patch
1732         if ($item eq $list[0] ||
1733             $item eq $list[$#list]) {
1734             $noclean = $save_clean;
1735         } else {
1736             $noclean = 1;
1737         }
1738
1739         if (defined($minconfig)) {
1740             build "useconfig:$minconfig" or return 0;
1741         } else {
1742             # ?? no config to use?
1743             build "oldconfig" or return 0;
1744         }
1745
1746         check_buildlog $sha1 or return 0;
1747
1748         next if ($type eq "build");
1749
1750         get_grub_index;
1751         get_version;
1752         install;
1753
1754         my $failed = 0;
1755
1756         start_monitor;
1757         monitor or $failed = 1;
1758
1759         if (!$failed && $type ne "boot"){
1760             do_run_test or $failed = 1;
1761         }
1762         end_monitor;
1763         return 0 if ($failed);
1764
1765     }
1766     $in_patchcheck = 0;
1767     success $i;
1768
1769     return 1;
1770 }
1771
1772 $#ARGV < 1 or die "ktest.pl version: $VERSION\n   usage: ktest.pl config-file\n";
1773
1774 if ($#ARGV == 0) {
1775     $ktest_config = $ARGV[0];
1776     if (! -f $ktest_config) {
1777         print "$ktest_config does not exist.\n";
1778         my $ans;
1779         for (;;) {
1780             print "Create it? [Y/n] ";
1781             $ans = <STDIN>;
1782             chomp $ans;
1783             if ($ans =~ /^\s*$/) {
1784                 $ans = "y";
1785             }
1786             last if ($ans =~ /^y$/i || $ans =~ /^n$/i);
1787             print "Please answer either 'y' or 'n'.\n";
1788         }
1789         if ($ans !~ /^y$/i) {
1790             exit 0;
1791         }
1792     }
1793 } else {
1794     $ktest_config = "ktest.conf";
1795 }
1796
1797 if (! -f $ktest_config) {
1798     open(OUT, ">$ktest_config") or die "Can not create $ktest_config";
1799     print OUT << "EOF"
1800 # Generated by ktest.pl
1801 #
1802 # Define each test with TEST_START
1803 # The config options below it will override the defaults
1804 TEST_START
1805
1806 DEFAULTS
1807 EOF
1808 ;
1809     close(OUT);
1810 }
1811 read_config $ktest_config;
1812
1813 # Append any configs entered in manually to the config file.
1814 my @new_configs = keys %entered_configs;
1815 if ($#new_configs >= 0) {
1816     print "\nAppending entered in configs to $ktest_config\n";
1817     open(OUT, ">>$ktest_config") or die "Can not append to $ktest_config";
1818     foreach my $config (@new_configs) {
1819         print OUT "$config = $entered_configs{$config}\n";
1820         $opt{$config} = $entered_configs{$config};
1821     }
1822 }
1823
1824 if ($opt{"CLEAR_LOG"} && defined($opt{"LOG_FILE"})) {
1825     unlink $opt{"LOG_FILE"};
1826 }
1827
1828 doprint "\n\nSTARTING AUTOMATED TESTS\n\n";
1829
1830 for (my $i = 0, my $repeat = 1; $i <= $opt{"NUM_TESTS"}; $i += $repeat) {
1831
1832     if (!$i) {
1833         doprint "DEFAULT OPTIONS:\n";
1834     } else {
1835         doprint "\nTEST $i OPTIONS";
1836         if (defined($repeat_tests{$i})) {
1837             $repeat = $repeat_tests{$i};
1838             doprint " ITERATE $repeat";
1839         }
1840         doprint "\n";
1841     }
1842
1843     foreach my $option (sort keys %opt) {
1844
1845         if ($option =~ /\[(\d+)\]$/) {
1846             next if ($i != $1);
1847         } else {
1848             next if ($i);
1849         }
1850
1851         doprint "$option = $opt{$option}\n";
1852     }
1853 }
1854
1855 sub set_test_option {
1856     my ($name, $i) = @_;
1857
1858     my $option = "$name\[$i\]";
1859
1860     if (defined($opt{$option})) {
1861         return $opt{$option};
1862     }
1863
1864     foreach my $test (keys %repeat_tests) {
1865         if ($i >= $test &&
1866             $i < $test + $repeat_tests{$test}) {
1867             $option = "$name\[$test\]";
1868             if (defined($opt{$option})) {
1869                 return $opt{$option};
1870             }
1871         }
1872     }
1873
1874     if (defined($opt{$name})) {
1875         return $opt{$name};
1876     }
1877
1878     return undef;
1879 }
1880
1881 # First we need to do is the builds
1882 for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
1883
1884     $iteration = $i;
1885
1886     my $makecmd = set_test_option("MAKE_CMD", $i);
1887
1888     $machine = set_test_option("MACHINE", $i);
1889     $ssh_user = set_test_option("SSH_USER", $i);
1890     $tmpdir = set_test_option("TMP_DIR", $i);
1891     $outputdir = set_test_option("OUTPUT_DIR", $i);
1892     $builddir = set_test_option("BUILD_DIR", $i);
1893     $test_type = set_test_option("TEST_TYPE", $i);
1894     $build_type = set_test_option("BUILD_TYPE", $i);
1895     $build_options = set_test_option("BUILD_OPTIONS", $i);
1896     $power_cycle = set_test_option("POWER_CYCLE", $i);
1897     $reboot = set_test_option("REBOOT", $i);
1898     $noclean = set_test_option("BUILD_NOCLEAN", $i);
1899     $minconfig = set_test_option("MIN_CONFIG", $i);
1900     $run_test = set_test_option("TEST", $i);
1901     $addconfig = set_test_option("ADD_CONFIG", $i);
1902     $reboot_type = set_test_option("REBOOT_TYPE", $i);
1903     $grub_menu = set_test_option("GRUB_MENU", $i);
1904     $post_install = set_test_option("POST_INSTALL", $i);
1905     $reboot_script = set_test_option("REBOOT_SCRIPT", $i);
1906     $reboot_on_error = set_test_option("REBOOT_ON_ERROR", $i);
1907     $poweroff_on_error = set_test_option("POWEROFF_ON_ERROR", $i);
1908     $die_on_failure = set_test_option("DIE_ON_FAILURE", $i);
1909     $power_off = set_test_option("POWER_OFF", $i);
1910     $powercycle_after_reboot = set_test_option("POWERCYCLE_AFTER_REBOOT", $i);
1911     $poweroff_after_halt = set_test_option("POWEROFF_AFTER_HALT", $i);
1912     $sleep_time = set_test_option("SLEEP_TIME", $i);
1913     $bisect_sleep_time = set_test_option("BISECT_SLEEP_TIME", $i);
1914     $store_failures = set_test_option("STORE_FAILURES", $i);
1915     $timeout = set_test_option("TIMEOUT", $i);
1916     $booted_timeout = set_test_option("BOOTED_TIMEOUT", $i);
1917     $console = set_test_option("CONSOLE", $i);
1918     $success_line = set_test_option("SUCCESS_LINE", $i);
1919     $stop_after_success = set_test_option("STOP_AFTER_SUCCESS", $i);
1920     $stop_after_failure = set_test_option("STOP_AFTER_FAILURE", $i);
1921     $build_target = set_test_option("BUILD_TARGET", $i);
1922     $ssh_exec = set_test_option("SSH_EXEC", $i);
1923     $scp_to_target = set_test_option("SCP_TO_TARGET", $i);
1924     $target_image = set_test_option("TARGET_IMAGE", $i);
1925     $localversion = set_test_option("LOCALVERSION", $i);
1926
1927     chdir $builddir || die "can't change directory to $builddir";
1928
1929     if (!-d $tmpdir) {
1930         mkpath($tmpdir) or
1931             die "can't create $tmpdir";
1932     }
1933
1934     $ENV{"SSH_USER"} = $ssh_user;
1935     $ENV{"MACHINE"} = $machine;
1936
1937     $target = "$ssh_user\@$machine";
1938
1939     $buildlog = "$tmpdir/buildlog-$machine";
1940     $dmesg = "$tmpdir/dmesg-$machine";
1941     $make = "$makecmd O=$outputdir";
1942     $output_config = "$outputdir/.config";
1943
1944     if ($reboot_type eq "grub") {
1945         dodie "GRUB_MENU not defined" if (!defined($grub_menu));
1946     } elsif (!defined($reboot_script)) {
1947         dodie "REBOOT_SCRIPT not defined"
1948     }
1949
1950     my $run_type = $build_type;
1951     if ($test_type eq "patchcheck") {
1952         $run_type = $opt{"PATCHCHECK_TYPE[$i]"};
1953     } elsif ($test_type eq "bisect") {
1954         $run_type = $opt{"BISECT_TYPE[$i]"};
1955     } elsif ($test_type eq "config_bisect") {
1956         $run_type = $opt{"CONFIG_BISECT_TYPE[$i]"};
1957     }
1958
1959     # mistake in config file?
1960     if (!defined($run_type)) {
1961         $run_type = "ERROR";
1962     }
1963
1964     doprint "\n\n";
1965     doprint "RUNNING TEST $i of $opt{NUM_TESTS} with option $test_type $run_type\n\n";
1966
1967     unlink $dmesg;
1968     unlink $buildlog;
1969
1970     if (!defined($minconfig)) {
1971         $minconfig = $addconfig;
1972
1973     } elsif (defined($addconfig)) {
1974         run_command "cat $addconfig $minconfig > $tmpdir/add_config" or
1975             dodie "Failed to create temp config";
1976         $minconfig = "$tmpdir/add_config";
1977     }
1978
1979     my $checkout = $opt{"CHECKOUT[$i]"};
1980     if (defined($checkout)) {
1981         run_command "git checkout $checkout" or
1982             die "failed to checkout $checkout";
1983     }
1984
1985     if ($test_type eq "bisect") {
1986         bisect $i;
1987         next;
1988     } elsif ($test_type eq "config_bisect") {
1989         config_bisect $i;
1990         next;
1991     } elsif ($test_type eq "patchcheck") {
1992         patchcheck $i;
1993         next;
1994     }
1995
1996     if ($build_type ne "nobuild") {
1997         build $build_type or next;
1998     }
1999
2000     if ($test_type ne "build") {
2001         get_grub_index;
2002         get_version;
2003         install;
2004
2005         my $failed = 0;
2006         start_monitor;
2007         monitor or $failed = 1;;
2008
2009         if (!$failed && $test_type ne "boot" && defined($run_test)) {
2010             do_run_test or $failed = 1;
2011         }
2012         end_monitor;
2013         next if ($failed);
2014     }
2015
2016     success $i;
2017 }
2018
2019 if ($opt{"POWEROFF_ON_SUCCESS"}) {
2020     halt;
2021 } elsif ($opt{"REBOOT_ON_SUCCESS"} && !do_not_reboot) {
2022     reboot;
2023 }
2024
2025 doprint "\n    $successes of $opt{NUM_TESTS} tests were successful\n\n";
2026
2027 exit 0;