tools/testing/ktest/ktest.pl

   1 #!/usr/bin/perl -w
   2 #
   3 # Copywrite 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
   4 # Licensed under the terms of the GNU GPL License version 2
   5 #
   6
   7 use strict;
   8 use IPC::Open2;
   9 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
  10 use File::Path qw(mkpath);
  11 use File::Copy qw(cp);
  12 use FileHandle;
  13
  14 my $VERSION = "0.2";
  15
  16 $| = 1;
  17
  18 my %opt;
  19 my %repeat_tests;
  20 my %repeats;
  21 my %default;
  22
  23 #default opts
  24 $default{"NUM_TESTS"}           = 1;
  25 $default{"REBOOT_TYPE"}         = "grub";
  26 $default{"TEST_TYPE"}           = "test";
  27 $default{"BUILD_TYPE"}          = "randconfig";
  28 $default{"MAKE_CMD"}            = "make";
  29 $default{"TIMEOUT"}             = 120;
  30 $default{"TMP_DIR"}             = "/tmp/ktest";
  31 $default{"SLEEP_TIME"}          = 60;   # sleep time between tests
  32 $default{"BUILD_NOCLEAN"}       = 0;
  33 $default{"REBOOT_ON_ERROR"}     = 0;
  34 $default{"POWEROFF_ON_ERROR"}   = 0;
  35 $default{"REBOOT_ON_SUCCESS"}   = 1;
  36 $default{"POWEROFF_ON_SUCCESS"} = 0;
  37 $default{"BUILD_OPTIONS"}       = "";
  38 $default{"BISECT_SLEEP_TIME"}   = 60;   # sleep time between bisects
  39 $default{"CLEAR_LOG"}           = 0;
  40 $default{"BISECT_MANUAL"}       = 0;
  41 $default{"BISECT_SKIP"}         = 1;
  42 $default{"SUCCESS_LINE"}        = "login:";
  43 $default{"BOOTED_TIMEOUT"}      = 1;
  44 $default{"DIE_ON_FAILURE"}      = 1;
  45 $default{"SSH_EXEC"}            = "ssh \$SSH_USER\@\$MACHINE \$SSH_COMMAND";
  46 $default{"SCP_TO_TARGET"}       = "scp \$SRC_FILE \$SSH_USER\@\$MACHINE:\$DST_FILE";
  47 $default{"REBOOT"}              = "ssh \$SSH_USER\@\$MACHINE reboot";
  48 $default{"STOP_AFTER_SUCCESS"}  = 10;
  49 $default{"STOP_AFTER_FAILURE"}  = 60;
  50 $default{"LOCALVERSION"}        = "-test";
  51
  52 my $ktest_config;
  53 my $version;
  54 my $machine;
  55 my $ssh_user;
  56 my $tmpdir;
  57 my $builddir;
  58 my $outputdir;
  59 my $output_config;
  60 my $test_type;
  61 my $build_type;
  62 my $build_options;
  63 my $reboot_type;
  64 my $reboot_script;
  65 my $power_cycle;
  66 my $reboot;
  67 my $reboot_on_error;
  68 my $poweroff_on_error;
  69 my $die_on_failure;
  70 my $powercycle_after_reboot;
  71 my $poweroff_after_halt;
  72 my $ssh_exec;
  73 my $scp_to_target;
  74 my $power_off;
  75 my $grub_menu;
  76 my $grub_number;
  77 my $target;
  78 my $make;
  79 my $post_install;
  80 my $noclean;
  81 my $minconfig;
  82 my $addconfig;
  83 my $in_bisect = 0;
  84 my $bisect_bad = "";
  85 my $reverse_bisect;
  86 my $bisect_manual;
  87 my $bisect_skip;
  88 my $in_patchcheck = 0;
  89 my $run_test;
  90 my $redirect;
  91 my $buildlog;
  92 my $dmesg;
  93 my $monitor_fp;
  94 my $monitor_pid;
  95 my $monitor_cnt = 0;
  96 my $sleep_time;
  97 my $bisect_sleep_time;
  98 my $store_failures;
  99 my $timeout;
 100 my $booted_timeout;
 101 my $console;
 102 my $success_line;
 103 my $stop_after_success;
 104 my $stop_after_failure;
 105 my $build_target;
 106 my $target_image;
 107 my $localversion;
 108 my $iteration = 0;
 109 my $successes = 0;
 110
 111 my %entered_configs;
 112 my %config_help;
 113
 114 $config_help{"MACHINE"} = << "EOF"
 115  The machine hostname that you will test.
 116 EOF
 117     ;
 118 $config_help{"SSH_USER"} = << "EOF"
 119  The box is expected to have ssh on normal bootup, provide the user
 120   (most likely root, since you need privileged operations)
 121 EOF
 122     ;
 123 $config_help{"BUILD_DIR"} = << "EOF"
 124  The directory that contains the Linux source code (full path).
 125 EOF
 126     ;
 127 $config_help{"OUTPUT_DIR"} = << "EOF"
 128  The directory that the objects will be built (full path).
 129  (can not be same as BUILD_DIR)
 130 EOF
 131     ;
 132 $config_help{"BUILD_TARGET"} = << "EOF"
 133  The location of the compiled file to copy to the target.
 134  (relative to OUTPUT_DIR)
 135 EOF
 136     ;
 137 $config_help{"TARGET_IMAGE"} = << "EOF"
 138  The place to put your image on the test machine.
 139 EOF
 140     ;
 141 $config_help{"POWER_CYCLE"} = << "EOF"
 142  A script or command to reboot the box.
 143
 144  Here is a digital loggers power switch example
 145  POWER_CYCLE = wget --no-proxy -O /dev/null -q  --auth-no-challenge 'http://admin:admin\@power/outlet?5=CCL'
 146
 147  Here is an example to reboot a virtual box on the current host
 148  with the name "Guest".
 149  POWER_CYCLE = virsh destroy Guest; sleep 5; virsh start Guest
 150 EOF
 151     ;
 152 $config_help{"CONSOLE"} = << "EOF"
 153  The script or command that reads the console
 154
 155   If you use ttywatch server, something like the following would work.
 156 CONSOLE = nc -d localhost 3001
 157
 158  For a virtual machine with guest name "Guest".
 159 CONSOLE =  virsh console Guest
 160 EOF
 161     ;
 162 $config_help{"LOCALVERSION"} = << "EOF"
 163  Required version ending to differentiate the test
 164  from other linux builds on the system.
 165 EOF
 166     ;
 167 $config_help{"REBOOT_TYPE"} = << "EOF"
 168  Way to reboot the box to the test kernel.
 169  Only valid options so far are "grub" and "script".
 170
 171  If you specify grub, it will assume grub version 1
 172  and will search in /boot/grub/menu.lst for the title \$GRUB_MENU
 173  and select that target to reboot to the kernel. If this is not
 174  your setup, then specify "script" and have a command or script
 175  specified in REBOOT_SCRIPT to boot to the target.
 176
 177  The entry in /boot/grub/menu.lst must be entered in manually.
 178  The test will not modify that file.
 179 EOF
 180     ;
 181 $config_help{"GRUB_MENU"} = << "EOF"
 182  The grub title name for the test kernel to boot
 183  (Only mandatory if REBOOT_TYPE = grub)
 184
 185  Note, ktest.pl will not update the grub menu.lst, you need to
 186  manually add an option for the test. ktest.pl will search
 187  the grub menu.lst for this option to find what kernel to
 188  reboot into.
 189
 190  For example, if in the /boot/grub/menu.lst the test kernel title has:
 191  title Test Kernel
 192  kernel vmlinuz-test
 193  GRUB_MENU = Test Kernel
 194 EOF
 195     ;
 196 $config_help{"REBOOT_SCRIPT"} = << "EOF"
 197  A script to reboot the target into the test kernel
 198  (Only mandatory if REBOOT_TYPE = script)
 199 EOF
 200     ;
 201
 202
 203 sub get_ktest_config {
 204     my ($config) = @_;
 205
 206     return if (defined($opt{$config}));
 207
 208     if (defined($config_help{$config})) {
 209         print "\n";
 210         print $config_help{$config};
 211     }
 212
 213     for (;;) {
 214         print "$config = ";
 215         if (defined($default{$config})) {
 216             print "\[$default{$config}\] ";
 217         }
 218         $entered_configs{$config} = <STDIN>;
 219         $entered_configs{$config} =~ s/^\s*(.*\S)\s*$/$1/;
 220         if ($entered_configs{$config} =~ /^\s*$/) {
 221             if ($default{$config}) {
 222                 $entered_configs{$config} = $default{$config};
 223             } else {
 224                 print "Your answer can not be blank\n";
 225                 next;
 226             }
 227         }
 228         last;
 229     }
 230 }
 231
 232 sub get_ktest_configs {
 233     get_ktest_config("MACHINE");
 234     get_ktest_config("SSH_USER");
 235     get_ktest_config("BUILD_DIR");
 236     get_ktest_config("OUTPUT_DIR");
 237     get_ktest_config("BUILD_TARGET");
 238     get_ktest_config("TARGET_IMAGE");
 239     get_ktest_config("POWER_CYCLE");
 240     get_ktest_config("CONSOLE");
 241     get_ktest_config("LOCALVERSION");
 242
 243     my $rtype = $opt{"REBOOT_TYPE"};
 244
 245     if (!defined($rtype)) {
 246         if (!defined($opt{"GRUB_MENU"})) {
 247             get_ktest_config("REBOOT_TYPE");
 248             $rtype = $entered_configs{"REBOOT_TYPE"};
 249         } else {
 250             $rtype = "grub";
 251         }
 252     }
 253
 254     if ($rtype eq "grub") {
 255         get_ktest_config("GRUB_MENU");
 256     } else {
 257         get_ktest_config("REBOOT_SCRIPT");
 258     }
 259 }
 260
 261 sub set_value {
 262     my ($lvalue, $rvalue) = @_;
 263
 264     if (defined($opt{$lvalue})) {
 265         die "Error: Option $lvalue defined more than once!\n";
 266     }
 267     if ($rvalue =~ /^\s*$/) {
 268         delete $opt{$lvalue};
 269     } else {
 270         $opt{$lvalue} = $rvalue;
 271     }
 272 }
 273
 274 sub read_config {
 275     my ($config) = @_;
 276
 277     open(IN, $config) || die "can't read file $config";
 278
 279     my $name = $config;
 280     $name =~ s,.*/(.*),$1,;
 281
 282     my $test_num = 0;
 283     my $default = 1;
 284     my $repeat = 1;
 285     my $num_tests_set = 0;
 286     my $skip = 0;
 287     my $rest;
 288
 289     while (<IN>) {
 290
 291         # ignore blank lines and comments
 292         next if (/^\s*$/ || /\s*\#/);
 293
 294         if (/^\s*TEST_START(.*)/) {
 295
 296             $rest = $1;
 297
 298             if ($num_tests_set) {
 299                 die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 300             }
 301
 302             my $old_test_num = $test_num;
 303             my $old_repeat = $repeat;
 304
 305             $test_num += $repeat;
 306             $default = 0;
 307             $repeat = 1;
 308
 309             if ($rest =~ /\s+SKIP(.*)/) {
 310                 $rest = $1;
 311                 $skip = 1;
 312             } else {
 313                 $skip = 0;
 314             }
 315
 316             if ($rest =~ /\s+ITERATE\s+(\d+)(.*)$/) {
 317                 $repeat = $1;
 318                 $rest = $2;
 319                 $repeat_tests{"$test_num"} = $repeat;
 320             }
 321
 322             if ($rest =~ /\s+SKIP(.*)/) {
 323                 $rest = $1;
 324                 $skip = 1;
 325             }
 326
 327             if ($rest !~ /^\s*$/) {
 328                 die "$name: $.: Gargbage found after TEST_START\n$_";
 329             }
 330
 331             if ($skip) {
 332                 $test_num = $old_test_num;
 333                 $repeat = $old_repeat;
 334             }
 335
 336         } elsif (/^\s*DEFAULTS(.*)$/) {
 337             $default = 1;
 338
 339             $rest = $1;
 340
 341             if ($rest =~ /\s+SKIP(.*)/) {
 342                 $rest = $1;
 343                 $skip = 1;
 344             } else {
 345                 $skip = 0;
 346             }
 347
 348             if ($rest !~ /^\s*$/) {
 349                 die "$name: $.: Gargbage found after DEFAULTS\n$_";
 350             }
 351
 352         } elsif (/^\s*([A-Z_\[\]\d]+)\s*=\s*(.*?)\s*$/) {
 353
 354             next if ($skip);
 355
 356             my $lvalue = $1;
 357             my $rvalue = $2;
 358
 359             if (!$default &&
 360                 ($lvalue eq "NUM_TESTS" ||
 361                  $lvalue eq "LOG_FILE" ||
 362                  $lvalue eq "CLEAR_LOG")) {
 363                 die "$name: $.: $lvalue must be set in DEFAULTS section\n";
 364             }
 365
 366             if ($lvalue eq "NUM_TESTS") {
 367                 if ($test_num) {
 368                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 369                 }
 370                 if (!$default) {
 371                     die "$name: $.: NUM_TESTS must be set in default section\n";
 372                 }
 373                 $num_tests_set = 1;
 374             }
 375
 376             if ($default || $lvalue =~ /\[\d+\]$/) {
 377                 set_value($lvalue, $rvalue);
 378             } else {
 379                 my $val = "$lvalue\[$test_num\]";
 380                 set_value($val, $rvalue);
 381
 382                 if ($repeat > 1) {
 383                     $repeats{$val} = $repeat;
 384                 }
 385             }
 386         } else {
 387             die "$name: $.: Garbage found in config\n$_";
 388         }
 389     }
 390
 391     close(IN);
 392
 393     if ($test_num) {
 394         $test_num += $repeat - 1;
 395         $opt{"NUM_TESTS"} = $test_num;
 396     }
 397
 398     # make sure we have all mandatory configs
 399     get_ktest_configs;
 400
 401     # set any defaults
 402
 403     foreach my $default (keys %default) {
 404         if (!defined($opt{$default})) {
 405             $opt{$default} = $default{$default};
 406         }
 407     }
 408 }
 409
 410 sub _logit {
 411     if (defined($opt{"LOG_FILE"})) {
 412         open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
 413         print OUT @_;
 414         close(OUT);
 415     }
 416 }
 417
 418 sub logit {
 419     if (defined($opt{"LOG_FILE"})) {
 420         _logit @_;
 421     } else {
 422         print @_;
 423     }
 424 }
 425
 426 sub doprint {
 427     print @_;
 428     _logit @_;
 429 }
 430
 431 sub run_command;
 432
 433 sub reboot {
 434     # try to reboot normally
 435     if (run_command $reboot) {
 436         if (defined($powercycle_after_reboot)) {
 437             sleep $powercycle_after_reboot;
 438             run_command "$power_cycle";
 439         }
 440     } else {
 441         # nope? power cycle it.
 442         run_command "$power_cycle";
 443     }
 444 }
 445
 446 sub do_not_reboot {
 447     my $i = $iteration;
 448
 449     return $test_type eq "build" ||
 450         ($test_type eq "patchcheck" && $opt{"PATCHCHECK_TYPE[$i]"} eq "build") ||
 451         ($test_type eq "bisect" && $opt{"BISECT_TYPE[$i]"} eq "build");
 452 }
 453
 454 sub dodie {
 455     doprint "CRITICAL FAILURE... ", @_, "\n";
 456
 457     my $i = $iteration;
 458
 459     if ($reboot_on_error && !do_not_reboot) {
 460
 461         doprint "REBOOTING\n";
 462         reboot;
 463
 464     } elsif ($poweroff_on_error && defined($power_off)) {
 465         doprint "POWERING OFF\n";
 466         `$power_off`;
 467     }
 468
 469     if (defined($opt{"LOG_FILE"})) {
 470         print " See $opt{LOG_FILE} for more info.\n";
 471     }
 472
 473     die @_, "\n";
 474 }
 475
 476 sub open_console {
 477     my ($fp) = @_;
 478
 479     my $flags;
 480
 481     my $pid = open($fp, "$console|") or
 482         dodie "Can't open console $console";
 483
 484     $flags = fcntl($fp, F_GETFL, 0) or
 485         dodie "Can't get flags for the socket: $!";
 486     $flags = fcntl($fp, F_SETFL, $flags | O_NONBLOCK) or
 487         dodie "Can't set flags for the socket: $!";
 488
 489     return $pid;
 490 }
 491
 492 sub close_console {
 493     my ($fp, $pid) = @_;
 494
 495     doprint "kill child process $pid\n";
 496     kill 2, $pid;
 497
 498     print "closing!\n";
 499     close($fp);
 500 }
 501
 502 sub start_monitor {
 503     if ($monitor_cnt++) {
 504         return;
 505     }
 506     $monitor_fp = \*MONFD;
 507     $monitor_pid = open_console $monitor_fp;
 508
 509     return;
 510
 511     open(MONFD, "Stop perl from warning about single use of MONFD");
 512 }
 513
 514 sub end_monitor {
 515     if (--$monitor_cnt) {
 516         return;
 517     }
 518     close_console($monitor_fp, $monitor_pid);
 519 }
 520
 521 sub wait_for_monitor {
 522     my ($time) = @_;
 523     my $line;
 524
 525     doprint "** Wait for monitor to settle down **\n";
 526
 527     # read the monitor and wait for the system to calm down
 528     do {
 529         $line = wait_for_input($monitor_fp, $time);
 530         print "$line" if (defined($line));
 531     } while (defined($line));
 532     print "** Monitor flushed **\n";
 533 }
 534
 535 sub fail {
 536
 537         if ($die_on_failure) {
 538                 dodie @_;
 539         }
 540
 541         doprint "FAILED\n";
 542
 543         my $i = $iteration;
 544
 545         # no need to reboot for just building.
 546         if (!do_not_reboot) {
 547             doprint "REBOOTING\n";
 548             reboot;
 549             start_monitor;
 550             wait_for_monitor $sleep_time;
 551             end_monitor;
 552         }
 553
 554         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 555         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 556         doprint "KTEST RESULT: TEST $i Failed: ", @_, "\n";
 557         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 558         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 559
 560         return 1 if (!defined($store_failures));
 561
 562         my @t = localtime;
 563         my $date = sprintf "%04d%02d%02d%02d%02d%02d",
 564                 1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
 565
 566         my $type = $build_type;
 567         if ($type =~ /useconfig/) {
 568             $type = "useconfig";
 569         }
 570
 571         my $dir = "$machine-$test_type-$type-fail-$date";
 572         my $faildir = "$store_failures/$dir";
 573
 574         if (!-d $faildir) {
 575             mkpath($faildir) or
 576                 die "can't create $faildir";
 577         }
 578         if (-f "$output_config") {
 579             cp "$output_config", "$faildir/config" or
 580                 die "failed to copy .config";
 581         }
 582         if (-f $buildlog) {
 583             cp $buildlog, "$faildir/buildlog" or
 584                 die "failed to move $buildlog";
 585         }
 586         if (-f $dmesg) {
 587             cp $dmesg, "$faildir/dmesg" or
 588                 die "failed to move $dmesg";
 589         }
 590
 591         doprint "*** Saved info to $faildir ***\n";
 592
 593         return 1;
 594 }
 595
 596 sub run_command {
 597     my ($command) = @_;
 598     my $dolog = 0;
 599     my $dord = 0;
 600     my $pid;
 601
 602     $command =~ s/\$SSH_USER/$ssh_user/g;
 603     $command =~ s/\$MACHINE/$machine/g;
 604
 605     doprint("$command ... ");
 606
 607     $pid = open(CMD, "$command 2>&1 |") or
 608         (fail "unable to exec $command" and return 0);
 609
 610     if (defined($opt{"LOG_FILE"})) {
 611         open(LOG, ">>$opt{LOG_FILE}") or
 612             dodie "failed to write to log";
 613         $dolog = 1;
 614     }
 615
 616     if (defined($redirect)) {
 617         open (RD, ">$redirect") or
 618             dodie "failed to write to redirect $redirect";
 619         $dord = 1;
 620     }
 621
 622     while (<CMD>) {
 623         print LOG if ($dolog);
 624         print RD  if ($dord);
 625     }
 626
 627     waitpid($pid, 0);
 628     my $failed = $?;
 629
 630     close(CMD);
 631     close(LOG) if ($dolog);
 632     close(RD)  if ($dord);
 633
 634     if ($failed) {
 635         doprint "FAILED!\n";
 636     } else {
 637         doprint "SUCCESS\n";
 638     }
 639
 640     return !$failed;
 641 }
 642
 643 sub run_ssh {
 644     my ($cmd) = @_;
 645     my $cp_exec = $ssh_exec;
 646
 647     $cp_exec =~ s/\$SSH_COMMAND/$cmd/g;
 648     return run_command "$cp_exec";
 649 }
 650
 651 sub run_scp {
 652     my ($src, $dst) = @_;
 653     my $cp_scp = $scp_to_target;
 654
 655     $cp_scp =~ s/\$SRC_FILE/$src/g;
 656     $cp_scp =~ s/\$DST_FILE/$dst/g;
 657
 658     return run_command "$cp_scp";
 659 }
 660
 661 sub get_grub_index {
 662
 663     if ($reboot_type ne "grub") {
 664         return;
 665     }
 666     return if (defined($grub_number));
 667
 668     doprint "Find grub menu ... ";
 669     $grub_number = -1;
 670
 671     my $ssh_grub = $ssh_exec;
 672     $ssh_grub =~ s,\$SSH_COMMAND,cat /boot/grub/menu.lst,g;
 673
 674     open(IN, "$ssh_grub |")
 675         or die "unable to get menu.lst";
 676
 677     while (<IN>) {
 678         if (/^\s*title\s+$grub_menu\s*$/) {
 679             $grub_number++;
 680             last;
 681         } elsif (/^\s*title\s/) {
 682             $grub_number++;
 683         }
 684     }
 685     close(IN);
 686
 687     die "Could not find '$grub_menu' in /boot/grub/menu on $machine"
 688         if ($grub_number < 0);
 689     doprint "$grub_number\n";
 690 }
 691
 692 sub wait_for_input
 693 {
 694     my ($fp, $time) = @_;
 695     my $rin;
 696     my $ready;
 697     my $line;
 698     my $ch;
 699
 700     if (!defined($time)) {
 701         $time = $timeout;
 702     }
 703
 704     $rin = '';
 705     vec($rin, fileno($fp), 1) = 1;
 706     $ready = select($rin, undef, undef, $time);
 707
 708     $line = "";
 709
 710     # try to read one char at a time
 711     while (sysread $fp, $ch, 1) {
 712         $line .= $ch;
 713         last if ($ch eq "\n");
 714     }
 715
 716     if (!length($line)) {
 717         return undef;
 718     }
 719
 720     return $line;
 721 }
 722
 723 sub reboot_to {
 724     if ($reboot_type eq "grub") {
 725         run_ssh "'(echo \"savedefault --default=$grub_number --once\" | grub --batch; reboot)'";
 726         return;
 727     }
 728
 729     run_command "$reboot_script";
 730 }
 731
 732 sub get_sha1 {
 733     my ($commit) = @_;
 734
 735     doprint "git rev-list --max-count=1 $commit ... ";
 736     my $sha1 = `git rev-list --max-count=1 $commit`;
 737     my $ret = $?;
 738
 739     logit $sha1;
 740
 741     if ($ret) {
 742         doprint "FAILED\n";
 743         dodie "Failed to get git $commit";
 744     }
 745
 746     print "SUCCESS\n";
 747
 748     chomp $sha1;
 749
 750     return $sha1;
 751 }
 752
 753 sub monitor {
 754     my $booted = 0;
 755     my $bug = 0;
 756     my $skip_call_trace = 0;
 757     my $loops;
 758
 759     wait_for_monitor 5;
 760
 761     my $line;
 762     my $full_line = "";
 763
 764     open(DMESG, "> $dmesg") or
 765         die "unable to write to $dmesg";
 766
 767     reboot_to;
 768
 769     my $success_start;
 770     my $failure_start;
 771
 772     for (;;) {
 773
 774         if ($booted) {
 775             $line = wait_for_input($monitor_fp, $booted_timeout);
 776         } else {
 777             $line = wait_for_input($monitor_fp);
 778         }
 779
 780         last if (!defined($line));
 781
 782         doprint $line;
 783         print DMESG $line;
 784
 785         # we are not guaranteed to get a full line
 786         $full_line .= $line;
 787
 788         if ($full_line =~ /$success_line/) {
 789             $booted = 1;
 790             $success_start = time;
 791         }
 792
 793         if ($booted && defined($stop_after_success) &&
 794             $stop_after_success >= 0) {
 795             my $now = time;
 796             if ($now - $success_start >= $stop_after_success) {
 797                 doprint "Test forced to stop after $stop_after_success seconds after success\n";
 798                 last;
 799             }
 800         }
 801
 802         if ($full_line =~ /\[ backtrace testing \]/) {
 803             $skip_call_trace = 1;
 804         }
 805
 806         if ($full_line =~ /call trace:/i) {
 807             if (!$bug && !$skip_call_trace) {
 808                 $bug = 1;
 809                 $failure_start = time;
 810             }
 811         }
 812
 813         if ($bug && defined($stop_after_failure) &&
 814             $stop_after_failure >= 0) {
 815             my $now = time;
 816             if ($now - $failure_start >= $stop_after_failure) {
 817                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
 818                 last;
 819             }
 820         }
 821
 822         if ($full_line =~ /\[ end of backtrace testing \]/) {
 823             $skip_call_trace = 0;
 824         }
 825
 826         if ($full_line =~ /Kernel panic -/) {
 827             $failure_start = time;
 828             $bug = 1;
 829         }
 830
 831         if ($line =~ /\n/) {
 832             $full_line = "";
 833         }
 834     }
 835
 836     close(DMESG);
 837
 838     if ($bug) {
 839         return 0 if ($in_bisect);
 840         fail "failed - got a bug report" and return 0;
 841     }
 842
 843     if (!$booted) {
 844         return 0 if ($in_bisect);
 845         fail "failed - never got a boot prompt." and return 0;
 846     }
 847
 848     return 1;
 849 }
 850
 851 sub install {
 852
 853     run_scp "$outputdir/$build_target", "$target_image" or
 854         dodie "failed to copy image";
 855
 856     my $install_mods = 0;
 857
 858     # should we process modules?
 859     $install_mods = 0;
 860     open(IN, "$output_config") or dodie("Can't read config file");
 861     while (<IN>) {
 862         if (/CONFIG_MODULES(=y)?/) {
 863             $install_mods = 1 if (defined($1));
 864             last;
 865         }
 866     }
 867     close(IN);
 868
 869     if (!$install_mods) {
 870         doprint "No modules needed\n";
 871         return;
 872     }
 873
 874     run_command "$make INSTALL_MOD_PATH=$tmpdir modules_install" or
 875         dodie "Failed to install modules";
 876
 877     my $modlib = "/lib/modules/$version";
 878     my $modtar = "ktest-mods.tar.bz2";
 879
 880     run_ssh "rm -rf $modlib" or
 881         dodie "failed to remove old mods: $modlib";
 882
 883     # would be nice if scp -r did not follow symbolic links
 884     run_command "cd $tmpdir && tar -cjf $modtar lib/modules/$version" or
 885         dodie "making tarball";
 886
 887     run_scp "$tmpdir/$modtar", "/tmp" or
 888         dodie "failed to copy modules";
 889
 890     unlink "$tmpdir/$modtar";
 891
 892     run_ssh "'(cd / && tar xf /tmp/$modtar)'" or
 893         dodie "failed to tar modules";
 894
 895     run_ssh "rm -f /tmp/$modtar";
 896
 897     return if (!defined($post_install));
 898
 899     my $cp_post_install = $post_install;
 900     $cp_post_install = s/\$KERNEL_VERSION/$version/g;
 901     run_command "$cp_post_install" or
 902         dodie "Failed to run post install";
 903 }
 904
 905 sub check_buildlog {
 906     my ($patch) = @_;
 907
 908     my @files = `git show $patch | diffstat -l`;
 909
 910     open(IN, "git show $patch |") or
 911         dodie "failed to show $patch";
 912     while (<IN>) {
 913         if (m,^--- a/(.*),) {
 914             chomp $1;
 915             $files[$#files] = $1;
 916         }
 917     }
 918     close(IN);
 919
 920     open(IN, $buildlog) or dodie "Can't open $buildlog";
 921     while (<IN>) {
 922         if (/^\s*(.*?):.*(warning|error)/) {
 923             my $err = $1;
 924             foreach my $file (@files) {
 925                 my $fullpath = "$builddir/$file";
 926                 if ($file eq $err || $fullpath eq $err) {
 927                     fail "$file built with warnings" and return 0;
 928                 }
 929             }
 930         }
 931     }
 932     close(IN);
 933
 934     return 1;
 935 }
 936
 937 sub make_oldconfig {
 938     my ($defconfig) = @_;
 939
 940     if (!run_command "$defconfig $make oldnoconfig") {
 941         # Perhaps oldnoconfig doesn't exist in this version of the kernel
 942         # try a yes '' | oldconfig
 943         doprint "oldnoconfig failed, trying yes '' | make oldconfig\n";
 944         run_command "yes '' | $defconfig $make oldconfig" or
 945             dodie "failed make config oldconfig";
 946     }
 947 }
 948
 949 sub build {
 950     my ($type) = @_;
 951     my $defconfig = "";
 952
 953     unlink $buildlog;
 954
 955     if ($type =~ /^useconfig:(.*)/) {
 956         run_command "cp $1 $output_config" or
 957             dodie "could not copy $1 to .config";
 958
 959         $type = "oldconfig";
 960     }
 961
 962     # old config can ask questions
 963     if ($type eq "oldconfig") {
 964         $type = "oldnoconfig";
 965
 966         # allow for empty configs
 967         run_command "touch $output_config";
 968
 969         run_command "mv $output_config $outputdir/config_temp" or
 970             dodie "moving .config";
 971
 972         if (!$noclean && !run_command "$make mrproper") {
 973             dodie "make mrproper";
 974         }
 975
 976         run_command "mv $outputdir/config_temp $output_config" or
 977             dodie "moving config_temp";
 978
 979     } elsif (!$noclean) {
 980         unlink "$output_config";
 981         run_command "$make mrproper" or
 982             dodie "make mrproper";
 983     }
 984
 985     # add something to distinguish this build
 986     open(OUT, "> $outputdir/localversion") or dodie("Can't make localversion file");
 987     print OUT "$localversion\n";
 988     close(OUT);
 989
 990     if (defined($minconfig)) {
 991         $defconfig = "KCONFIG_ALLCONFIG=$minconfig";
 992     }
 993
 994     if ($type eq "oldnoconfig") {
 995         make_oldconfig $defconfig;
 996     } else {
 997         run_command "$defconfig $make $type" or
 998             dodie "failed make config";
 999     }
1000
1001     $redirect = "$buildlog";
1002     if (!run_command "$make $build_options") {
1003         undef $redirect;
1004         # bisect may need this to pass
1005         return 0 if ($in_bisect);
1006         fail "failed build" and return 0;
1007     }
1008     undef $redirect;
1009
1010     return 1;
1011 }
1012
1013 sub halt {
1014     if (!run_ssh "halt" or defined($power_off)) {
1015         if (defined($poweroff_after_halt)) {
1016             sleep $poweroff_after_halt;
1017             run_command "$power_off";
1018         }
1019     } else {
1020         # nope? the zap it!
1021         run_command "$power_off";
1022     }
1023 }
1024
1025 sub success {
1026     my ($i) = @_;
1027
1028     $successes++;
1029
1030     doprint "\n\n*******************************************\n";
1031     doprint     "*******************************************\n";
1032     doprint     "KTEST RESULT: TEST $i SUCCESS!!!!         **\n";
1033     doprint     "*******************************************\n";
1034     doprint     "*******************************************\n";
1035
1036     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
1037         doprint "Reboot and wait $sleep_time seconds\n";
1038         reboot;
1039         start_monitor;
1040         wait_for_monitor $sleep_time;
1041         end_monitor;
1042     }
1043 }
1044
1045 sub get_version {
1046     # get the release name
1047     doprint "$make kernelrelease ... ";
1048     $version = `$make kernelrelease | tail -1`;
1049     chomp($version);
1050     doprint "$version\n";
1051 }
1052
1053 sub answer_bisect {
1054     for (;;) {
1055         doprint "Pass or fail? [p/f]";
1056         my $ans = <STDIN>;
1057         chomp $ans;
1058         if ($ans eq "p" || $ans eq "P") {
1059             return 1;
1060         } elsif ($ans eq "f" || $ans eq "F") {
1061             return 0;
1062         } else {
1063             print "Please answer 'P' or 'F'\n";
1064         }
1065     }
1066 }
1067
1068 sub child_run_test {
1069     my $failed = 0;
1070
1071     # child should have no power
1072     $reboot_on_error = 0;
1073     $poweroff_on_error = 0;
1074     $die_on_failure = 1;
1075
1076     run_command $run_test or $failed = 1;
1077     exit $failed;
1078 }
1079
1080 my $child_done;
1081
1082 sub child_finished {
1083     $child_done = 1;
1084 }
1085
1086 sub do_run_test {
1087     my $child_pid;
1088     my $child_exit;
1089     my $line;
1090     my $full_line;
1091     my $bug = 0;
1092
1093     wait_for_monitor 1;
1094
1095     doprint "run test $run_test\n";
1096
1097     $child_done = 0;
1098
1099     $SIG{CHLD} = qw(child_finished);
1100
1101     $child_pid = fork;
1102
1103     child_run_test if (!$child_pid);
1104
1105     $full_line = "";
1106
1107     do {
1108         $line = wait_for_input($monitor_fp, 1);
1109         if (defined($line)) {
1110
1111             # we are not guaranteed to get a full line
1112             $full_line .= $line;
1113
1114             if ($full_line =~ /call trace:/i) {
1115                 $bug = 1;
1116             }
1117
1118             if ($full_line =~ /Kernel panic -/) {
1119                 $bug = 1;
1120             }
1121
1122             if ($line =~ /\n/) {
1123                 $full_line = "";
1124             }
1125         }
1126     } while (!$child_done && !$bug);
1127
1128     if ($bug) {
1129         doprint "Detected kernel crash!\n";
1130         # kill the child with extreme prejudice
1131         kill 9, $child_pid;
1132     }
1133
1134     waitpid $child_pid, 0;
1135     $child_exit = $?;
1136
1137     if ($bug || $child_exit) {
1138         return 0 if $in_bisect;
1139         fail "test failed" and return 0;
1140     }
1141     return 1;
1142 }
1143
1144 sub run_git_bisect {
1145     my ($command) = @_;
1146
1147     doprint "$command ... ";
1148
1149     my $output = `$command 2>&1`;
1150     my $ret = $?;
1151
1152     logit $output;
1153
1154     if ($ret) {
1155         doprint "FAILED\n";
1156         dodie "Failed to git bisect";
1157     }
1158
1159     doprint "SUCCESS\n";
1160     if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
1161         doprint "$1 [$2]\n";
1162     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
1163         $bisect_bad = $1;
1164         doprint "Found bad commit... $1\n";
1165         return 0;
1166     } else {
1167         # we already logged it, just print it now.
1168         print $output;
1169     }
1170
1171     return 1;
1172 }
1173
1174 sub bisect_reboot {
1175     doprint "Reboot and sleep $bisect_sleep_time seconds\n";
1176     reboot;
1177     start_monitor;
1178     wait_for_monitor $bisect_sleep_time;
1179     end_monitor;
1180 }
1181
1182 # returns 1 on success, 0 on failure, -1 on skip
1183 sub run_bisect_test {
1184     my ($type, $buildtype) = @_;
1185
1186     my $failed = 0;
1187     my $result;
1188     my $output;
1189     my $ret;
1190
1191     $in_bisect = 1;
1192
1193     build $buildtype or $failed = 1;
1194
1195     if ($type ne "build") {
1196         if ($failed && $bisect_skip) {
1197             $in_bisect = 0;
1198             return -1;
1199         }
1200         dodie "Failed on build" if $failed;
1201
1202         # Now boot the box
1203         get_grub_index;
1204         get_version;
1205         install;
1206
1207         start_monitor;
1208         monitor or $failed = 1;
1209
1210         if ($type ne "boot") {
1211             if ($failed && $bisect_skip) {
1212                 end_monitor;
1213                 bisect_reboot;
1214                 $in_bisect = 0;
1215                 return -1;
1216             }
1217             dodie "Failed on boot" if $failed;
1218
1219             do_run_test or $failed = 1;
1220         }
1221         end_monitor;
1222     }
1223
1224     if ($failed) {
1225         $result = 0;
1226
1227         # reboot the box to a good kernel
1228         if ($type ne "build") {
1229             bisect_reboot;
1230         }
1231     } else {
1232         $result = 1;
1233     }
1234     $in_bisect = 0;
1235
1236     return $result;
1237 }
1238
1239 sub run_bisect {
1240     my ($type) = @_;
1241     my $buildtype = "oldconfig";
1242
1243     # We should have a minconfig to use?
1244     if (defined($minconfig)) {
1245         $buildtype = "useconfig:$minconfig";
1246     }
1247
1248     my $ret = run_bisect_test $type, $buildtype;
1249
1250     if ($bisect_manual) {
1251         $ret = answer_bisect;
1252     }
1253
1254     # Are we looking for where it worked, not failed?
1255     if ($reverse_bisect) {
1256         $ret = !$ret;
1257     }
1258
1259     if ($ret > 0) {
1260         return "good";
1261     } elsif ($ret == 0) {
1262         return  "bad";
1263     } elsif ($bisect_skip) {
1264         doprint "HIT A BAD COMMIT ... SKIPPING\n";
1265         return "skip";
1266     }
1267 }
1268
1269 sub bisect {
1270     my ($i) = @_;
1271
1272     my $result;
1273
1274     die "BISECT_GOOD[$i] not defined\n" if (!defined($opt{"BISECT_GOOD[$i]"}));
1275     die "BISECT_BAD[$i] not defined\n"  if (!defined($opt{"BISECT_BAD[$i]"}));
1276     die "BISECT_TYPE[$i] not defined\n" if (!defined($opt{"BISECT_TYPE[$i]"}));
1277
1278     my $good = $opt{"BISECT_GOOD[$i]"};
1279     my $bad = $opt{"BISECT_BAD[$i]"};
1280     my $type = $opt{"BISECT_TYPE[$i]"};
1281     my $start = $opt{"BISECT_START[$i]"};
1282     my $replay = $opt{"BISECT_REPLAY[$i]"};
1283     my $start_files = $opt{"BISECT_FILES[$i]"};
1284
1285     if (defined($start_files)) {
1286         $start_files = " -- " . $start_files;
1287     } else {
1288         $start_files = "";
1289     }
1290
1291     # convert to true sha1's
1292     $good = get_sha1($good);
1293     $bad = get_sha1($bad);
1294
1295     if (defined($opt{"BISECT_REVERSE[$i]"}) &&
1296         $opt{"BISECT_REVERSE[$i]"} == 1) {
1297         doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
1298         $reverse_bisect = 1;
1299     } else {
1300         $reverse_bisect = 0;
1301     }
1302
1303     # Can't have a test without having a test to run
1304     if ($type eq "test" && !defined($run_test)) {
1305         $type = "boot";
1306     }
1307
1308     my $check = $opt{"BISECT_CHECK[$i]"};
1309     if (defined($check) && $check ne "0") {
1310
1311         # get current HEAD
1312         my $head = get_sha1("HEAD");
1313
1314         if ($check ne "good") {
1315             doprint "TESTING BISECT BAD [$bad]\n";
1316             run_command "git checkout $bad" or
1317                 die "Failed to checkout $bad";
1318
1319             $result = run_bisect $type;
1320
1321             if ($result ne "bad") {
1322                 fail "Tested BISECT_BAD [$bad] and it succeeded" and return 0;
1323             }
1324         }
1325
1326         if ($check ne "bad") {
1327             doprint "TESTING BISECT GOOD [$good]\n";
1328             run_command "git checkout $good" or
1329                 die "Failed to checkout $good";
1330
1331             $result = run_bisect $type;
1332
1333             if ($result ne "good") {
1334                 fail "Tested BISECT_GOOD [$good] and it failed" and return 0;
1335             }
1336         }
1337
1338         # checkout where we started
1339         run_command "git checkout $head" or
1340             die "Failed to checkout $head";
1341     }
1342
1343     run_command "git bisect start$start_files" or
1344         dodie "could not start bisect";
1345
1346     run_command "git bisect good $good" or
1347         dodie "could not set bisect good to $good";
1348
1349     run_git_bisect "git bisect bad $bad" or
1350         dodie "could not set bisect bad to $bad";
1351
1352     if (defined($replay)) {
1353         run_command "git bisect replay $replay" or
1354             dodie "failed to run replay";
1355     }
1356
1357     if (defined($start)) {
1358         run_command "git checkout $start" or
1359             dodie "failed to checkout $start";
1360     }
1361
1362     my $test;
1363     do {
1364         $result = run_bisect $type;
1365         $test = run_git_bisect "git bisect $result";
1366     } while ($test);
1367
1368     run_command "git bisect log" or
1369         dodie "could not capture git bisect log";
1370
1371     run_command "git bisect reset" or
1372         dodie "could not reset git bisect";
1373
1374     doprint "Bad commit was [$bisect_bad]\n";
1375
1376     success $i;
1377 }
1378
1379 my %config_ignore;
1380 my %config_set;
1381
1382 my %config_list;
1383 my %null_config;
1384
1385 my %dependency;
1386
1387 sub process_config_ignore {
1388     my ($config) = @_;
1389
1390     open (IN, $config)
1391         or dodie "Failed to read $config";
1392
1393     while (<IN>) {
1394         if (/^(.*?(CONFIG\S*)(=.*| is not set))/) {
1395             $config_ignore{$2} = $1;
1396         }
1397     }
1398
1399     close(IN);
1400 }
1401
1402 sub read_current_config {
1403     my ($config_ref) = @_;
1404
1405     %{$config_ref} = ();
1406     undef %{$config_ref};
1407
1408     my @key = keys %{$config_ref};
1409     if ($#key >= 0) {
1410         print "did not delete!\n";
1411         exit;
1412     }
1413     open (IN, "$output_config");
1414
1415     while (<IN>) {
1416         if (/^(CONFIG\S+)=(.*)/) {
1417             ${$config_ref}{$1} = $2;
1418         }
1419     }
1420     close(IN);
1421 }
1422
1423 sub get_dependencies {
1424     my ($config) = @_;
1425
1426     my $arr = $dependency{$config};
1427     if (!defined($arr)) {
1428         return ();
1429     }
1430
1431     my @deps = @{$arr};
1432
1433     foreach my $dep (@{$arr}) {
1434         print "ADD DEP $dep\n";
1435         @deps = (@deps, get_dependencies $dep);
1436     }
1437
1438     return @deps;
1439 }
1440
1441 sub create_config {
1442     my @configs = @_;
1443
1444     open(OUT, ">$output_config") or dodie "Can not write to $output_config";
1445
1446     foreach my $config (@configs) {
1447         print OUT "$config_set{$config}\n";
1448         my @deps = get_dependencies $config;
1449         foreach my $dep (@deps) {
1450             print OUT "$config_set{$dep}\n";
1451         }
1452     }
1453
1454     foreach my $config (keys %config_ignore) {
1455         print OUT "$config_ignore{$config}\n";
1456     }
1457     close(OUT);
1458
1459 #    exit;
1460     make_oldconfig "";
1461 }
1462
1463 sub compare_configs {
1464     my (%a, %b) = @_;
1465
1466     foreach my $item (keys %a) {
1467         if (!defined($b{$item})) {
1468             print "diff $item\n";
1469             return 1;
1470         }
1471         delete $b{$item};
1472     }
1473
1474     my @keys = keys %b;
1475     if ($#keys) {
1476         print "diff2 $keys[0]\n";
1477     }
1478     return -1 if ($#keys >= 0);
1479
1480     return 0;
1481 }
1482
1483 sub run_config_bisect_test {
1484     my ($type) = @_;
1485
1486     return run_bisect_test $type, "oldconfig";
1487 }
1488
1489 sub process_passed {
1490     my (%configs) = @_;
1491
1492     doprint "These configs had no failure: (Enabling them for further compiles)\n";
1493     # Passed! All these configs are part of a good compile.
1494     # Add them to the min options.
1495     foreach my $config (keys %configs) {
1496         if (defined($config_list{$config})) {
1497             doprint " removing $config\n";
1498             $config_ignore{$config} = $config_list{$config};
1499             delete $config_list{$config};
1500         }
1501     }
1502     doprint "config copied to $outputdir/config_good\n";
1503     run_command "cp -f $output_config $outputdir/config_good";
1504 }
1505
1506 sub process_failed {
1507     my ($config) = @_;
1508
1509     doprint "\n\n***************************************\n";
1510     doprint "Found bad config: $config\n";
1511     doprint "***************************************\n\n";
1512 }
1513
1514 sub run_config_bisect {
1515
1516     my @start_list = keys %config_list;
1517
1518     if ($#start_list < 0) {
1519         doprint "No more configs to test!!!\n";
1520         return -1;
1521     }
1522
1523     doprint "***** RUN TEST ***\n";
1524     my $type = $opt{"CONFIG_BISECT_TYPE[$iteration]"};
1525     my $ret;
1526     my %current_config;
1527
1528     my $count = $#start_list + 1;
1529     doprint "  $count configs to test\n";
1530
1531     my $half = int($#start_list / 2);
1532
1533     do {
1534         my @tophalf = @start_list[0 .. $half];
1535
1536         create_config @tophalf;
1537         read_current_config \%current_config;
1538
1539         $count = $#tophalf + 1;
1540         doprint "Testing $count configs\n";
1541         my $found = 0;
1542         # make sure we test something
1543         foreach my $config (@tophalf) {
1544             if (defined($current_config{$config})) {
1545                 logit " $config\n";
1546                 $found = 1;
1547             }
1548         }
1549         if (!$found) {
1550             # try the other half
1551             doprint "Top half produced no set configs, trying bottom half\n";
1552             @tophalf = @start_list[$half .. $#start_list];
1553             create_config @tophalf;
1554             read_current_config \%current_config;
1555             foreach my $config (@tophalf) {
1556                 if (defined($current_config{$config})) {
1557                     logit " $config\n";
1558                     $found = 1;
1559                 }
1560             }
1561             if (!$found) {
1562                 doprint "Failed: Can't make new config with current configs\n";
1563                 foreach my $config (@start_list) {
1564                     doprint "  CONFIG: $config\n";
1565                 }
1566                 return -1;
1567             }
1568             $count = $#tophalf + 1;
1569             doprint "Testing $count configs\n";
1570         }
1571
1572         $ret = run_config_bisect_test $type;
1573         if ($bisect_manual) {
1574             $ret = answer_bisect;
1575         }
1576         if ($ret) {
1577             process_passed %current_config;
1578             return 0;
1579         }
1580
1581         doprint "This config had a failure.\n";
1582         doprint "Removing these configs that were not set in this config:\n";
1583         doprint "config copied to $outputdir/config_bad\n";
1584         run_command "cp -f $output_config $outputdir/config_bad";
1585
1586         # A config exists in this group that was bad.
1587         foreach my $config (keys %config_list) {
1588             if (!defined($current_config{$config})) {
1589                 doprint " removing $config\n";
1590                 delete $config_list{$config};
1591             }
1592         }
1593
1594         @start_list = @tophalf;
1595
1596         if ($#start_list == 0) {
1597             process_failed $start_list[0];
1598             return 1;
1599         }
1600
1601         # remove half the configs we are looking at and see if
1602         # they are good.
1603         $half = int($#start_list / 2);
1604     } while ($half > 0);
1605
1606     # we found a single config, try it again unless we are running manually
1607
1608     if ($bisect_manual) {
1609         process_failed $start_list[0];
1610         return 1;
1611     }
1612
1613     my @tophalf = @start_list[0 .. 0];
1614
1615     $ret = run_config_bisect_test $type;
1616     if ($ret) {
1617         process_passed %current_config;
1618         return 0;
1619     }
1620
1621     process_failed $start_list[0];
1622     return 1;
1623 }
1624
1625 sub config_bisect {
1626     my ($i) = @_;
1627
1628     my $start_config = $opt{"CONFIG_BISECT[$i]"};
1629
1630     my $tmpconfig = "$tmpdir/use_config";
1631
1632     # Make the file with the bad config and the min config
1633     if (defined($minconfig)) {
1634         # read the min config for things to ignore
1635         run_command "cp $minconfig $tmpconfig" or
1636             dodie "failed to copy $minconfig to $tmpconfig";
1637     } else {
1638         unlink $tmpconfig;
1639     }
1640
1641     # Add other configs
1642     if (defined($addconfig)) {
1643         run_command "cat $addconfig >> $tmpconfig" or
1644             dodie "failed to append $addconfig";
1645     }
1646
1647     my $defconfig = "";
1648     if (-f $tmpconfig) {
1649         $defconfig = "KCONFIG_ALLCONFIG=$tmpconfig";
1650         process_config_ignore $tmpconfig;
1651     }
1652
1653     # now process the start config
1654     run_command "cp $start_config $output_config" or
1655         dodie "failed to copy $start_config to $output_config";
1656
1657     # read directly what we want to check
1658     my %config_check;
1659     open (IN, $output_config)
1660         or dodie "faied to open $output_config";
1661
1662     while (<IN>) {
1663         if (/^((CONFIG\S*)=.*)/) {
1664             $config_check{$2} = $1;
1665         }
1666     }
1667     close(IN);
1668
1669     # Now run oldconfig with the minconfig (and addconfigs)
1670     make_oldconfig $defconfig;
1671
1672     # check to see what we lost (or gained)
1673     open (IN, $output_config)
1674         or dodie "Failed to read $start_config";
1675
1676     my %removed_configs;
1677     my %added_configs;
1678
1679     while (<IN>) {
1680         if (/^((CONFIG\S*)=.*)/) {
1681             # save off all options
1682             $config_set{$2} = $1;
1683             if (defined($config_check{$2})) {
1684                 if (defined($config_ignore{$2})) {
1685                     $removed_configs{$2} = $1;
1686                 } else {
1687                     $config_list{$2} = $1;
1688                 }
1689             } elsif (!defined($config_ignore{$2})) {
1690                 $added_configs{$2} = $1;
1691                 $config_list{$2} = $1;
1692             }
1693         }
1694     }
1695     close(IN);
1696
1697     my @confs = keys %removed_configs;
1698     if ($#confs >= 0) {
1699         doprint "Configs overridden by default configs and removed from check:\n";
1700         foreach my $config (@confs) {
1701             doprint " $config\n";
1702         }
1703     }
1704     @confs = keys %added_configs;
1705     if ($#confs >= 0) {
1706         doprint "Configs appearing in make oldconfig and added:\n";
1707         foreach my $config (@confs) {
1708             doprint " $config\n";
1709         }
1710     }
1711
1712     my %config_test;
1713     my $once = 0;
1714
1715     # Sometimes kconfig does weird things. We must make sure
1716     # that the config we autocreate has everything we need
1717     # to test, otherwise we may miss testing configs, or
1718     # may not be able to create a new config.
1719     # Here we create a config with everything set.
1720     create_config (keys %config_list);
1721     read_current_config \%config_test;
1722     foreach my $config (keys %config_list) {
1723         if (!defined($config_test{$config})) {
1724             if (!$once) {
1725                 $once = 1;
1726                 doprint "Configs not produced by kconfig (will not be checked):\n";
1727             }
1728             doprint "  $config\n";
1729             delete $config_list{$config};
1730         }
1731     }
1732     my $ret;
1733     do {
1734         $ret = run_config_bisect;
1735     } while (!$ret);
1736
1737     return $ret if ($ret < 0);
1738
1739     success $i;
1740 }
1741
1742 sub patchcheck {
1743     my ($i) = @_;
1744
1745     die "PATCHCHECK_START[$i] not defined\n"
1746         if (!defined($opt{"PATCHCHECK_START[$i]"}));
1747     die "PATCHCHECK_TYPE[$i] not defined\n"
1748         if (!defined($opt{"PATCHCHECK_TYPE[$i]"}));
1749
1750     my $start = $opt{"PATCHCHECK_START[$i]"};
1751
1752     my $end = "HEAD";
1753     if (defined($opt{"PATCHCHECK_END[$i]"})) {
1754         $end = $opt{"PATCHCHECK_END[$i]"};
1755     }
1756
1757     # Get the true sha1's since we can use things like HEAD~3
1758     $start = get_sha1($start);
1759     $end = get_sha1($end);
1760
1761     my $type = $opt{"PATCHCHECK_TYPE[$i]"};
1762
1763     # Can't have a test without having a test to run
1764     if ($type eq "test" && !defined($run_test)) {
1765         $type = "boot";
1766     }
1767
1768     open (IN, "git log --pretty=oneline $end|") or
1769         dodie "could not get git list";
1770
1771     my @list;
1772
1773     while (<IN>) {
1774         chomp;
1775         $list[$#list+1] = $_;
1776         last if (/^$start/);
1777     }
1778     close(IN);
1779
1780     if ($list[$#list] !~ /^$start/) {
1781         fail "SHA1 $start not found";
1782     }
1783
1784     # go backwards in the list
1785     @list = reverse @list;
1786
1787     my $save_clean = $noclean;
1788
1789     $in_patchcheck = 1;
1790     foreach my $item (@list) {
1791         my $sha1 = $item;
1792         $sha1 =~ s/^([[:xdigit:]]+).*/$1/;
1793
1794         doprint "\nProcessing commit $item\n\n";
1795
1796         run_command "git checkout $sha1" or
1797             die "Failed to checkout $sha1";
1798
1799         # only clean on the first and last patch
1800         if ($item eq $list[0] ||
1801             $item eq $list[$#list]) {
1802             $noclean = $save_clean;
1803         } else {
1804             $noclean = 1;
1805         }
1806
1807         if (defined($minconfig)) {
1808             build "useconfig:$minconfig" or return 0;
1809         } else {
1810             # ?? no config to use?
1811             build "oldconfig" or return 0;
1812         }
1813
1814         check_buildlog $sha1 or return 0;
1815
1816         next if ($type eq "build");
1817
1818         get_grub_index;
1819         get_version;
1820         install;
1821
1822         my $failed = 0;
1823
1824         start_monitor;
1825         monitor or $failed = 1;
1826
1827         if (!$failed && $type ne "boot"){
1828             do_run_test or $failed = 1;
1829         }
1830         end_monitor;
1831         return 0 if ($failed);
1832
1833     }
1834     $in_patchcheck = 0;
1835     success $i;
1836
1837     return 1;
1838 }
1839
1840 $#ARGV < 1 or die "ktest.pl version: $VERSION\n   usage: ktest.pl config-file\n";
1841
1842 if ($#ARGV == 0) {
1843     $ktest_config = $ARGV[0];
1844     if (! -f $ktest_config) {
1845         print "$ktest_config does not exist.\n";
1846         my $ans;
1847         for (;;) {
1848             print "Create it? [Y/n] ";
1849             $ans = <STDIN>;
1850             chomp $ans;
1851             if ($ans =~ /^\s*$/) {
1852                 $ans = "y";
1853             }
1854             last if ($ans =~ /^y$/i || $ans =~ /^n$/i);
1855             print "Please answer either 'y' or 'n'.\n";
1856         }
1857         if ($ans !~ /^y$/i) {
1858             exit 0;
1859         }
1860     }
1861 } else {
1862     $ktest_config = "ktest.conf";
1863 }
1864
1865 if (! -f $ktest_config) {
1866     open(OUT, ">$ktest_config") or die "Can not create $ktest_config";
1867     print OUT << "EOF"
1868 # Generated by ktest.pl
1869 #
1870 # Define each test with TEST_START
1871 # The config options below it will override the defaults
1872 TEST_START
1873
1874 DEFAULTS
1875 EOF
1876 ;
1877     close(OUT);
1878 }
1879 read_config $ktest_config;
1880
1881 # Append any configs entered in manually to the config file.
1882 my @new_configs = keys %entered_configs;
1883 if ($#new_configs >= 0) {
1884     print "\nAppending entered in configs to $ktest_config\n";
1885     open(OUT, ">>$ktest_config") or die "Can not append to $ktest_config";
1886     foreach my $config (@new_configs) {
1887         print OUT "$config = $entered_configs{$config}\n";
1888         $opt{$config} = $entered_configs{$config};
1889     }
1890 }
1891
1892 if ($opt{"CLEAR_LOG"} && defined($opt{"LOG_FILE"})) {
1893     unlink $opt{"LOG_FILE"};
1894 }
1895
1896 doprint "\n\nSTARTING AUTOMATED TESTS\n\n";
1897
1898 for (my $i = 0, my $repeat = 1; $i <= $opt{"NUM_TESTS"}; $i += $repeat) {
1899
1900     if (!$i) {
1901         doprint "DEFAULT OPTIONS:\n";
1902     } else {
1903         doprint "\nTEST $i OPTIONS";
1904         if (defined($repeat_tests{$i})) {
1905             $repeat = $repeat_tests{$i};
1906             doprint " ITERATE $repeat";
1907         }
1908         doprint "\n";
1909     }
1910
1911     foreach my $option (sort keys %opt) {
1912
1913         if ($option =~ /\[(\d+)\]$/) {
1914             next if ($i != $1);
1915         } else {
1916             next if ($i);
1917         }
1918
1919         doprint "$option = $opt{$option}\n";
1920     }
1921 }
1922
1923 sub set_test_option {
1924     my ($name, $i) = @_;
1925
1926     my $option = "$name\[$i\]";
1927
1928     if (defined($opt{$option})) {
1929         return $opt{$option};
1930     }
1931
1932     foreach my $test (keys %repeat_tests) {
1933         if ($i >= $test &&
1934             $i < $test + $repeat_tests{$test}) {
1935             $option = "$name\[$test\]";
1936             if (defined($opt{$option})) {
1937                 return $opt{$option};
1938             }
1939         }
1940     }
1941
1942     if (defined($opt{$name})) {
1943         return $opt{$name};
1944     }
1945
1946     return undef;
1947 }
1948
1949 # First we need to do is the builds
1950 for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
1951
1952     $iteration = $i;
1953
1954     my $makecmd = set_test_option("MAKE_CMD", $i);
1955
1956     $machine = set_test_option("MACHINE", $i);
1957     $ssh_user = set_test_option("SSH_USER", $i);
1958     $tmpdir = set_test_option("TMP_DIR", $i);
1959     $outputdir = set_test_option("OUTPUT_DIR", $i);
1960     $builddir = set_test_option("BUILD_DIR", $i);
1961     $test_type = set_test_option("TEST_TYPE", $i);
1962     $build_type = set_test_option("BUILD_TYPE", $i);
1963     $build_options = set_test_option("BUILD_OPTIONS", $i);
1964     $power_cycle = set_test_option("POWER_CYCLE", $i);
1965     $reboot = set_test_option("REBOOT", $i);
1966     $noclean = set_test_option("BUILD_NOCLEAN", $i);
1967     $minconfig = set_test_option("MIN_CONFIG", $i);
1968     $run_test = set_test_option("TEST", $i);
1969     $addconfig = set_test_option("ADD_CONFIG", $i);
1970     $reboot_type = set_test_option("REBOOT_TYPE", $i);
1971     $grub_menu = set_test_option("GRUB_MENU", $i);
1972     $post_install = set_test_option("POST_INSTALL", $i);
1973     $reboot_script = set_test_option("REBOOT_SCRIPT", $i);
1974     $reboot_on_error = set_test_option("REBOOT_ON_ERROR", $i);
1975     $poweroff_on_error = set_test_option("POWEROFF_ON_ERROR", $i);
1976     $die_on_failure = set_test_option("DIE_ON_FAILURE", $i);
1977     $power_off = set_test_option("POWER_OFF", $i);
1978     $powercycle_after_reboot = set_test_option("POWERCYCLE_AFTER_REBOOT", $i);
1979     $poweroff_after_halt = set_test_option("POWEROFF_AFTER_HALT", $i);
1980     $sleep_time = set_test_option("SLEEP_TIME", $i);
1981     $bisect_sleep_time = set_test_option("BISECT_SLEEP_TIME", $i);
1982     $bisect_manual = set_test_option("BISECT_MANUAL", $i);
1983     $bisect_skip = set_test_option("BISECT_SKIP", $i);
1984     $store_failures = set_test_option("STORE_FAILURES", $i);
1985     $timeout = set_test_option("TIMEOUT", $i);
1986     $booted_timeout = set_test_option("BOOTED_TIMEOUT", $i);
1987     $console = set_test_option("CONSOLE", $i);
1988     $success_line = set_test_option("SUCCESS_LINE", $i);
1989     $stop_after_success = set_test_option("STOP_AFTER_SUCCESS", $i);
1990     $stop_after_failure = set_test_option("STOP_AFTER_FAILURE", $i);
1991     $build_target = set_test_option("BUILD_TARGET", $i);
1992     $ssh_exec = set_test_option("SSH_EXEC", $i);
1993     $scp_to_target = set_test_option("SCP_TO_TARGET", $i);
1994     $target_image = set_test_option("TARGET_IMAGE", $i);
1995     $localversion = set_test_option("LOCALVERSION", $i);
1996
1997     chdir $builddir || die "can't change directory to $builddir";
1998
1999     if (!-d $tmpdir) {
2000         mkpath($tmpdir) or
2001             die "can't create $tmpdir";
2002     }
2003
2004     $ENV{"SSH_USER"} = $ssh_user;
2005     $ENV{"MACHINE"} = $machine;
2006
2007     $target = "$ssh_user\@$machine";
2008
2009     $buildlog = "$tmpdir/buildlog-$machine";
2010     $dmesg = "$tmpdir/dmesg-$machine";
2011     $make = "$makecmd O=$outputdir";
2012     $output_config = "$outputdir/.config";
2013
2014     if ($reboot_type eq "grub") {
2015         dodie "GRUB_MENU not defined" if (!defined($grub_menu));
2016     } elsif (!defined($reboot_script)) {
2017         dodie "REBOOT_SCRIPT not defined"
2018     }
2019
2020     my $run_type = $build_type;
2021     if ($test_type eq "patchcheck") {
2022         $run_type = $opt{"PATCHCHECK_TYPE[$i]"};
2023     } elsif ($test_type eq "bisect") {
2024         $run_type = $opt{"BISECT_TYPE[$i]"};
2025     } elsif ($test_type eq "config_bisect") {
2026         $run_type = $opt{"CONFIG_BISECT_TYPE[$i]"};
2027     }
2028
2029     # mistake in config file?
2030     if (!defined($run_type)) {
2031         $run_type = "ERROR";
2032     }
2033
2034     doprint "\n\n";
2035     doprint "RUNNING TEST $i of $opt{NUM_TESTS} with option $test_type $run_type\n\n";
2036
2037     unlink $dmesg;
2038     unlink $buildlog;
2039
2040     if (!defined($minconfig)) {
2041         $minconfig = $addconfig;
2042
2043     } elsif (defined($addconfig)) {
2044         run_command "cat $addconfig $minconfig > $tmpdir/add_config" or
2045             dodie "Failed to create temp config";
2046         $minconfig = "$tmpdir/add_config";
2047     }
2048
2049     my $checkout = $opt{"CHECKOUT[$i]"};
2050     if (defined($checkout)) {
2051         run_command "git checkout $checkout" or
2052             die "failed to checkout $checkout";
2053     }
2054
2055     if ($test_type eq "bisect") {
2056         bisect $i;
2057         next;
2058     } elsif ($test_type eq "config_bisect") {
2059         config_bisect $i;
2060         next;
2061     } elsif ($test_type eq "patchcheck") {
2062         patchcheck $i;
2063         next;
2064     }
2065
2066     if ($build_type ne "nobuild") {
2067         build $build_type or next;
2068     }
2069
2070     if ($test_type ne "build") {
2071         get_grub_index;
2072         get_version;
2073         install;
2074
2075         my $failed = 0;
2076         start_monitor;
2077         monitor or $failed = 1;;
2078
2079         if (!$failed && $test_type ne "boot" && defined($run_test)) {
2080             do_run_test or $failed = 1;
2081         }
2082         end_monitor;
2083         next if ($failed);
2084     }
2085
2086     success $i;
2087 }
2088
2089 if ($opt{"POWEROFF_ON_SUCCESS"}) {
2090     halt;
2091 } elsif ($opt{"REBOOT_ON_SUCCESS"} && !do_not_reboot) {
2092     reboot;
2093 }
2094
2095 doprint "\n    $successes of $opt{NUM_TESTS} tests were successful\n\n";
2096
2097 exit 0;