tools/testing/ktest/ktest.pl

   1 #!/usr/bin/perl -w
   2 #
   3 # Copywrite 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
   4 # Licensed under the terms of the GNU GPL License version 2
   5 #
   6
   7 use strict;
   8 use IPC::Open2;
   9 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
  10 use File::Path qw(mkpath);
  11 use File::Copy qw(cp);
  12 use FileHandle;
  13
  14 $#ARGV >= 0 || die "usage: ktest.pl config-file\n";
  15
  16 $| = 1;
  17
  18 my %opt;
  19 my %repeat_tests;
  20 my %repeats;
  21 my %default;
  22
  23 #default opts
  24 $default{"NUM_TESTS"}           = 1;
  25 $default{"REBOOT_TYPE"}         = "grub";
  26 $default{"TEST_TYPE"}           = "test";
  27 $default{"BUILD_TYPE"}          = "randconfig";
  28 $default{"MAKE_CMD"}            = "make";
  29 $default{"TIMEOUT"}             = 120;
  30 $default{"TMP_DIR"}             = "/tmp/ktest";
  31 $default{"SLEEP_TIME"}          = 60;   # sleep time between tests
  32 $default{"BUILD_NOCLEAN"}       = 0;
  33 $default{"REBOOT_ON_ERROR"}     = 0;
  34 $default{"POWEROFF_ON_ERROR"}   = 0;
  35 $default{"REBOOT_ON_SUCCESS"}   = 1;
  36 $default{"POWEROFF_ON_SUCCESS"} = 0;
  37 $default{"BUILD_OPTIONS"}       = "";
  38 $default{"BISECT_SLEEP_TIME"}   = 60;   # sleep time between bisects
  39 $default{"CLEAR_LOG"}           = 0;
  40 $default{"SUCCESS_LINE"}        = "login:";
  41 $default{"BOOTED_TIMEOUT"}      = 1;
  42 $default{"DIE_ON_FAILURE"}      = 1;
  43
  44 my $version;
  45 my $machine;
  46 my $tmpdir;
  47 my $builddir;
  48 my $outputdir;
  49 my $test_type;
  50 my $build_type;
  51 my $build_options;
  52 my $reboot_type;
  53 my $reboot_script;
  54 my $power_cycle;
  55 my $reboot_on_error;
  56 my $poweroff_on_error;
  57 my $die_on_failure;
  58 my $powercycle_after_reboot;
  59 my $poweroff_after_halt;
  60 my $power_off;
  61 my $grub_menu;
  62 my $grub_number;
  63 my $target;
  64 my $make;
  65 my $post_install;
  66 my $noclean;
  67 my $minconfig;
  68 my $addconfig;
  69 my $in_bisect = 0;
  70 my $bisect_bad = "";
  71 my $reverse_bisect;
  72 my $in_patchcheck = 0;
  73 my $run_test;
  74 my $redirect;
  75 my $buildlog;
  76 my $dmesg;
  77 my $monitor_fp;
  78 my $monitor_pid;
  79 my $monitor_cnt = 0;
  80 my $sleep_time;
  81 my $bisect_sleep_time;
  82 my $store_failures;
  83 my $timeout;
  84 my $booted_timeout;
  85 my $console;
  86 my $success_line;
  87 my $build_target;
  88 my $target_image;
  89 my $localversion;
  90 my $iteration = 0;
  91
  92 sub set_value {
  93     my ($lvalue, $rvalue) = @_;
  94
  95     if (defined($opt{$lvalue})) {
  96         die "Error: Option $lvalue defined more than once!\n";
  97     }
  98     $opt{$lvalue} = $rvalue;
  99 }
 100
 101 sub read_config {
 102     my ($config) = @_;
 103
 104     open(IN, $config) || die "can't read file $config";
 105
 106     my $name = $config;
 107     $name =~ s,.*/(.*),$1,;
 108
 109     my $test_num = 0;
 110     my $default = 1;
 111     my $repeat = 1;
 112     my $num_tests_set = 0;
 113     my $skip = 0;
 114     my $rest;
 115
 116     while (<IN>) {
 117
 118         # ignore blank lines and comments
 119         next if (/^\s*$/ || /\s*\#/);
 120
 121         if (/^\s*TEST_START(.*)/) {
 122
 123             $rest = $1;
 124
 125             if ($num_tests_set) {
 126                 die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 127             }
 128
 129             my $old_test_num = $test_num;
 130
 131             $test_num += $repeat;
 132             $default = 0;
 133             $repeat = 1;
 134
 135             if ($rest =~ /\s+SKIP(.*)/) {
 136                 $rest = $1;
 137                 $skip = 1;
 138             } else {
 139                 $skip = 0;
 140             }
 141
 142             if ($rest =~ /\s+ITERATE\s+(\d+)(.*)$/) {
 143                 $repeat = $1;
 144                 $rest = $2;
 145                 $repeat_tests{"$test_num"} = $repeat;
 146             }
 147
 148             if ($rest =~ /\s+SKIP(.*)/) {
 149                 $rest = $1;
 150                 $skip = 1;
 151             }
 152
 153             if ($rest !~ /^\s*$/) {
 154                 die "$name: $.: Gargbage found after TEST_START\n$_";
 155             }
 156
 157             if ($skip) {
 158                 $test_num = $old_test_num;
 159                 $repeat = 1;
 160             }
 161
 162         } elsif (/^\s*DEFAULTS(.*)$/) {
 163             $default = 1;
 164
 165             $rest = $1;
 166
 167             if ($rest =~ /\s+SKIP(.*)/) {
 168                 $rest = $1;
 169                 $skip = 1;
 170             } else {
 171                 $skip = 0;
 172             }
 173
 174             if ($rest !~ /^\s*$/) {
 175                 die "$name: $.: Gargbage found after DEFAULTS\n$_";
 176             }
 177
 178         } elsif (/^\s*([A-Z_\[\]\d]+)\s*=\s*(.*?)\s*$/) {
 179
 180             next if ($skip);
 181
 182             my $lvalue = $1;
 183             my $rvalue = $2;
 184
 185             if (!$default &&
 186                 ($lvalue eq "NUM_TESTS" ||
 187                  $lvalue eq "LOG_FILE" ||
 188                  $lvalue eq "CLEAR_LOG")) {
 189                 die "$name: $.: $lvalue must be set in DEFAULTS section\n";
 190             }
 191
 192             if ($lvalue eq "NUM_TESTS") {
 193                 if ($test_num) {
 194                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 195                 }
 196                 if (!$default) {
 197                     die "$name: $.: NUM_TESTS must be set in default section\n";
 198                 }
 199                 $num_tests_set = 1;
 200             }
 201
 202             if ($default || $lvalue =~ /\[\d+\]$/) {
 203                 set_value($lvalue, $rvalue);
 204             } else {
 205                 my $val = "$lvalue\[$test_num\]";
 206                 set_value($val, $rvalue);
 207
 208                 if ($repeat > 1) {
 209                     $repeats{$val} = $repeat;
 210                 }
 211             }
 212         } else {
 213             die "$name: $.: Garbage found in config\n$_";
 214         }
 215     }
 216
 217     close(IN);
 218
 219     if ($test_num) {
 220         $test_num += $repeat - 1;
 221         $opt{"NUM_TESTS"} = $test_num;
 222     }
 223
 224     # set any defaults
 225
 226     foreach my $default (keys %default) {
 227         if (!defined($opt{$default})) {
 228             $opt{$default} = $default{$default};
 229         }
 230     }
 231 }
 232
 233 sub logit {
 234     if (defined($opt{"LOG_FILE"})) {
 235         open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
 236         print OUT @_;
 237         close(OUT);
 238     }
 239 }
 240
 241 sub doprint {
 242     print @_;
 243     logit @_;
 244 }
 245
 246 sub run_command;
 247
 248 sub reboot {
 249     # try to reboot normally
 250     if (run_command "ssh $target reboot") {
 251         if (defined($powercycle_after_reboot)) {
 252             sleep $powercycle_after_reboot;
 253             run_command "$power_cycle";
 254         }
 255     } else {
 256         # nope? power cycle it.
 257         run_command "$power_cycle";
 258     }
 259 }
 260
 261 sub do_not_reboot {
 262     my $i = $iteration;
 263
 264     return $test_type eq "build" ||
 265         ($test_type eq "patchcheck" && $opt{"PATCHCHECK_TYPE[$i]"} eq "build") ||
 266         ($test_type eq "bisect" && $opt{"BISECT_TYPE[$i]"} eq "build");
 267 }
 268
 269 sub dodie {
 270     doprint "CRITICAL FAILURE... ", @_, "\n";
 271
 272     my $i = $iteration;
 273
 274     if ($reboot_on_error && !do_not_reboot) {
 275
 276         doprint "REBOOTING\n";
 277         reboot;
 278
 279     } elsif ($poweroff_on_error && defined($power_off)) {
 280         doprint "POWERING OFF\n";
 281         `$power_off`;
 282     }
 283
 284     die @_, "\n";
 285 }
 286
 287 sub open_console {
 288     my ($fp) = @_;
 289
 290     my $flags;
 291
 292     my $pid = open($fp, "$console|") or
 293         dodie "Can't open console $console";
 294
 295     $flags = fcntl($fp, F_GETFL, 0) or
 296         dodie "Can't get flags for the socket: $!";
 297     $flags = fcntl($fp, F_SETFL, $flags | O_NONBLOCK) or
 298         dodie "Can't set flags for the socket: $!";
 299
 300     return $pid;
 301 }
 302
 303 sub close_console {
 304     my ($fp, $pid) = @_;
 305
 306     doprint "kill child process $pid\n";
 307     kill 2, $pid;
 308
 309     print "closing!\n";
 310     close($fp);
 311 }
 312
 313 sub start_monitor {
 314     if ($monitor_cnt++) {
 315         return;
 316     }
 317     $monitor_fp = \*MONFD;
 318     $monitor_pid = open_console $monitor_fp;
 319
 320     return;
 321
 322     open(MONFD, "Stop perl from warning about single use of MONFD");
 323 }
 324
 325 sub end_monitor {
 326     if (--$monitor_cnt) {
 327         return;
 328     }
 329     close_console($monitor_fp, $monitor_pid);
 330 }
 331
 332 sub wait_for_monitor {
 333     my ($time) = @_;
 334     my $line;
 335
 336     doprint "** Wait for monitor to settle down **\n";
 337
 338     # read the monitor and wait for the system to calm down
 339     do {
 340         $line = wait_for_input($monitor_fp, $time);
 341         print "$line" if (defined($line));
 342     } while (defined($line));
 343     print "** Monitor flushed **\n";
 344 }
 345
 346 sub fail {
 347
 348         if ($die_on_failure) {
 349                 dodie @_;
 350         }
 351
 352         doprint "FAILED\n";
 353
 354         my $i = $iteration;
 355
 356         # no need to reboot for just building.
 357         if (!do_not_reboot) {
 358             doprint "REBOOTING\n";
 359             reboot;
 360             start_monitor;
 361             wait_for_monitor $sleep_time;
 362             end_monitor;
 363         }
 364
 365         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 366         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 367         doprint "**** Failed: ", @_, " ****\n";
 368         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 369         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 370
 371         return 1 if (!defined($store_failures));
 372
 373         my @t = localtime;
 374         my $date = sprintf "%04d%02d%02d%02d%02d%02d",
 375                 1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
 376
 377         my $dir = "$machine-$test_type-$build_type-fail-$date";
 378         my $faildir = "$store_failures/$dir";
 379
 380         if (!-d $faildir) {
 381             mkpath($faildir) or
 382                 die "can't create $faildir";
 383         }
 384         if (-f "$outputdir/.config") {
 385             cp "$outputdir/.config", "$faildir/config" or
 386                 die "failed to copy .config";
 387         }
 388         if (-f $buildlog) {
 389             cp $buildlog, "$faildir/buildlog" or
 390                 die "failed to move $buildlog";
 391         }
 392         if (-f $dmesg) {
 393             cp $dmesg, "$faildir/dmesg" or
 394                 die "failed to move $dmesg";
 395         }
 396
 397         doprint "*** Saved info to $faildir ***\n";
 398
 399         return 1;
 400 }
 401
 402 sub run_command {
 403     my ($command) = @_;
 404     my $dolog = 0;
 405     my $dord = 0;
 406     my $pid;
 407
 408     doprint("$command ... ");
 409
 410     $pid = open(CMD, "$command 2>&1 |") or
 411         (fail "unable to exec $command" and return 0);
 412
 413     if (defined($opt{"LOG_FILE"})) {
 414         open(LOG, ">>$opt{LOG_FILE}") or
 415             dodie "failed to write to log";
 416         $dolog = 1;
 417     }
 418
 419     if (defined($redirect)) {
 420         open (RD, ">$redirect") or
 421             dodie "failed to write to redirect $redirect";
 422         $dord = 1;
 423     }
 424
 425     while (<CMD>) {
 426         print LOG if ($dolog);
 427         print RD  if ($dord);
 428     }
 429
 430     waitpid($pid, 0);
 431     my $failed = $?;
 432
 433     close(CMD);
 434     close(LOG) if ($dolog);
 435     close(RD)  if ($dord);
 436
 437     if ($failed) {
 438         doprint "FAILED!\n";
 439     } else {
 440         doprint "SUCCESS\n";
 441     }
 442
 443     return !$failed;
 444 }
 445
 446 sub get_grub_index {
 447
 448     if ($reboot_type ne "grub") {
 449         return;
 450     }
 451     return if (defined($grub_number));
 452
 453     doprint "Find grub menu ... ";
 454     $grub_number = -1;
 455     open(IN, "ssh $target cat /boot/grub/menu.lst |")
 456         or die "unable to get menu.lst";
 457     while (<IN>) {
 458         if (/^\s*title\s+$grub_menu\s*$/) {
 459             $grub_number++;
 460             last;
 461         } elsif (/^\s*title\s/) {
 462             $grub_number++;
 463         }
 464     }
 465     close(IN);
 466
 467     die "Could not find '$grub_menu' in /boot/grub/menu on $machine"
 468         if ($grub_number < 0);
 469     doprint "$grub_number\n";
 470 }
 471
 472 sub wait_for_input
 473 {
 474     my ($fp, $time) = @_;
 475     my $rin;
 476     my $ready;
 477     my $line;
 478     my $ch;
 479
 480     if (!defined($time)) {
 481         $time = $timeout;
 482     }
 483
 484     $rin = '';
 485     vec($rin, fileno($fp), 1) = 1;
 486     $ready = select($rin, undef, undef, $time);
 487
 488     $line = "";
 489
 490     # try to read one char at a time
 491     while (sysread $fp, $ch, 1) {
 492         $line .= $ch;
 493         last if ($ch eq "\n");
 494     }
 495
 496     if (!length($line)) {
 497         return undef;
 498     }
 499
 500     return $line;
 501 }
 502
 503 sub reboot_to {
 504     if ($reboot_type eq "grub") {
 505         run_command "ssh $target '(echo \"savedefault --default=$grub_number --once\" | grub --batch; reboot)'";
 506         return;
 507     }
 508
 509     run_command "$reboot_script";
 510 }
 511
 512 sub get_sha1 {
 513     my ($commit) = @_;
 514
 515     doprint "git rev-list --max-count=1 $commit ... ";
 516     my $sha1 = `git rev-list --max-count=1 $commit`;
 517     my $ret = $?;
 518
 519     logit $sha1;
 520
 521     if ($ret) {
 522         doprint "FAILED\n";
 523         dodie "Failed to get git $commit";
 524     }
 525
 526     print "SUCCESS\n";
 527
 528     chomp $sha1;
 529
 530     return $sha1;
 531 }
 532
 533 sub monitor {
 534     my $booted = 0;
 535     my $bug = 0;
 536     my $skip_call_trace = 0;
 537     my $loops;
 538
 539     wait_for_monitor 5;
 540
 541     my $line;
 542     my $full_line = "";
 543
 544     open(DMESG, "> $dmesg") or
 545         die "unable to write to $dmesg";
 546
 547     reboot_to;
 548
 549     for (;;) {
 550
 551         if ($booted) {
 552             $line = wait_for_input($monitor_fp, $booted_timeout);
 553         } else {
 554             $line = wait_for_input($monitor_fp);
 555         }
 556
 557         last if (!defined($line));
 558
 559         doprint $line;
 560         print DMESG $line;
 561
 562         # we are not guaranteed to get a full line
 563         $full_line .= $line;
 564
 565         if ($full_line =~ /$success_line/) {
 566             $booted = 1;
 567         }
 568
 569         if ($full_line =~ /\[ backtrace testing \]/) {
 570             $skip_call_trace = 1;
 571         }
 572
 573         if ($full_line =~ /call trace:/i) {
 574             $bug = 1 if (!$skip_call_trace);
 575         }
 576
 577         if ($full_line =~ /\[ end of backtrace testing \]/) {
 578             $skip_call_trace = 0;
 579         }
 580
 581         if ($full_line =~ /Kernel panic -/) {
 582             $bug = 1;
 583         }
 584
 585         if ($line =~ /\n/) {
 586             $full_line = "";
 587         }
 588     }
 589
 590     close(DMESG);
 591
 592     if ($bug) {
 593         return 0 if ($in_bisect);
 594         fail "failed - got a bug report" and return 0;
 595     }
 596
 597     if (!$booted) {
 598         return 0 if ($in_bisect);
 599         fail "failed - never got a boot prompt." and return 0;
 600     }
 601
 602     return 1;
 603 }
 604
 605 sub install {
 606
 607     run_command "scp $outputdir/$build_target $target:$target_image" or
 608         dodie "failed to copy image";
 609
 610     my $install_mods = 0;
 611
 612     # should we process modules?
 613     $install_mods = 0;
 614     open(IN, "$outputdir/.config") or dodie("Can't read config file");
 615     while (<IN>) {
 616         if (/CONFIG_MODULES(=y)?/) {
 617             $install_mods = 1 if (defined($1));
 618             last;
 619         }
 620     }
 621     close(IN);
 622
 623     if (!$install_mods) {
 624         doprint "No modules needed\n";
 625         return;
 626     }
 627
 628     run_command "$make INSTALL_MOD_PATH=$tmpdir modules_install" or
 629         dodie "Failed to install modules";
 630
 631     my $modlib = "/lib/modules/$version";
 632     my $modtar = "ktest-mods.tar.bz2";
 633
 634     run_command "ssh $target rm -rf $modlib" or
 635         dodie "failed to remove old mods: $modlib";
 636
 637     # would be nice if scp -r did not follow symbolic links
 638     run_command "cd $tmpdir && tar -cjf $modtar lib/modules/$version" or
 639         dodie "making tarball";
 640
 641     run_command "scp $tmpdir/$modtar $target:/tmp" or
 642         dodie "failed to copy modules";
 643
 644     unlink "$tmpdir/$modtar";
 645
 646     run_command "ssh $target '(cd / && tar xf /tmp/$modtar)'" or
 647         dodie "failed to tar modules";
 648
 649     run_command "ssh $target rm -f /tmp/$modtar";
 650
 651     return if (!defined($post_install));
 652
 653     my $save_env = $ENV{KERNEL_VERSION};
 654
 655     $ENV{KERNEL_VERSION} = $version;
 656     run_command "$post_install" or
 657         dodie "Failed to run post install";
 658
 659     $ENV{KERNEL_VERSION} = $save_env;
 660 }
 661
 662 sub check_buildlog {
 663     my ($patch) = @_;
 664
 665     my @files = `git show $patch | diffstat -l`;
 666
 667     open(IN, "git show $patch |") or
 668         dodie "failed to show $patch";
 669     while (<IN>) {
 670         if (m,^--- a/(.*),) {
 671             chomp $1;
 672             $files[$#files] = $1;
 673         }
 674     }
 675     close(IN);
 676
 677     open(IN, $buildlog) or dodie "Can't open $buildlog";
 678     while (<IN>) {
 679         if (/^\s*(.*?):.*(warning|error)/) {
 680             my $err = $1;
 681             foreach my $file (@files) {
 682                 my $fullpath = "$builddir/$file";
 683                 if ($file eq $err || $fullpath eq $err) {
 684                     fail "$file built with warnings" and return 0;
 685                 }
 686             }
 687         }
 688     }
 689     close(IN);
 690
 691     return 1;
 692 }
 693
 694 sub build {
 695     my ($type) = @_;
 696     my $defconfig = "";
 697     my $append = "";
 698
 699     unlink $buildlog;
 700
 701     if ($type =~ /^useconfig:(.*)/) {
 702         run_command "cp $1 $outputdir/.config" or
 703             dodie "could not copy $1 to .config";
 704
 705         $type = "oldconfig";
 706     }
 707
 708     # old config can ask questions
 709     if ($type eq "oldconfig") {
 710         $append = "yes ''|";
 711
 712         # allow for empty configs
 713         run_command "touch $outputdir/.config";
 714
 715         run_command "mv $outputdir/.config $outputdir/config_temp" or
 716             dodie "moving .config";
 717
 718         if (!$noclean && !run_command "$make mrproper") {
 719             dodie "make mrproper";
 720         }
 721
 722         run_command "mv $outputdir/config_temp $outputdir/.config" or
 723             dodie "moving config_temp";
 724
 725     } elsif (!$noclean) {
 726         unlink "$outputdir/.config";
 727         run_command "$make mrproper" or
 728             dodie "make mrproper";
 729     }
 730
 731     # add something to distinguish this build
 732     open(OUT, "> $outputdir/localversion") or dodie("Can't make localversion file");
 733     print OUT "$localversion\n";
 734     close(OUT);
 735
 736     if (defined($minconfig)) {
 737         $defconfig = "KCONFIG_ALLCONFIG=$minconfig";
 738     }
 739
 740     run_command "$append $defconfig $make $type" or
 741         dodie "failed make config";
 742
 743     $redirect = "$buildlog";
 744     if (!run_command "$make $build_options") {
 745         undef $redirect;
 746         # bisect may need this to pass
 747         return 0 if ($in_bisect);
 748         fail "failed build" and return 0;
 749     }
 750     undef $redirect;
 751
 752     return 1;
 753 }
 754
 755 sub halt {
 756     if (!run_command "ssh $target halt" or defined($power_off)) {
 757         if (defined($poweroff_after_halt)) {
 758             sleep $poweroff_after_halt;
 759             run_command "$power_off";
 760         }
 761     } else {
 762         # nope? the zap it!
 763         run_command "$power_off";
 764     }
 765 }
 766
 767 sub success {
 768     my ($i) = @_;
 769
 770     doprint "\n\n*******************************************\n";
 771     doprint     "*******************************************\n";
 772     doprint     "**           TEST $i SUCCESS!!!!         **\n";
 773     doprint     "*******************************************\n";
 774     doprint     "*******************************************\n";
 775
 776     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
 777         doprint "Reboot and wait $sleep_time seconds\n";
 778         reboot;
 779         start_monitor;
 780         wait_for_monitor $sleep_time;
 781         end_monitor;
 782     }
 783 }
 784
 785 sub get_version {
 786     # get the release name
 787     doprint "$make kernelrelease ... ";
 788     $version = `$make kernelrelease | tail -1`;
 789     chomp($version);
 790     doprint "$version\n";
 791 }
 792
 793 sub child_run_test {
 794     my $failed = 0;
 795
 796     # child should have no power
 797     $reboot_on_error = 0;
 798     $poweroff_on_error = 0;
 799     $die_on_failure = 1;
 800
 801     run_command $run_test or $failed = 1;
 802     exit $failed;
 803 }
 804
 805 my $child_done;
 806
 807 sub child_finished {
 808     $child_done = 1;
 809 }
 810
 811 sub do_run_test {
 812     my $child_pid;
 813     my $child_exit;
 814     my $line;
 815     my $full_line;
 816     my $bug = 0;
 817
 818     wait_for_monitor 1;
 819
 820     doprint "run test $run_test\n";
 821
 822     $child_done = 0;
 823
 824     $SIG{CHLD} = qw(child_finished);
 825
 826     $child_pid = fork;
 827
 828     child_run_test if (!$child_pid);
 829
 830     $full_line = "";
 831
 832     do {
 833         $line = wait_for_input($monitor_fp, 1);
 834         if (defined($line)) {
 835
 836             # we are not guaranteed to get a full line
 837             $full_line .= $line;
 838
 839             if ($full_line =~ /call trace:/i) {
 840                 $bug = 1;
 841             }
 842
 843             if ($full_line =~ /Kernel panic -/) {
 844                 $bug = 1;
 845             }
 846
 847             if ($line =~ /\n/) {
 848                 $full_line = "";
 849             }
 850         }
 851     } while (!$child_done && !$bug);
 852
 853     if ($bug) {
 854         doprint "Detected kernel crash!\n";
 855         # kill the child with extreme prejudice
 856         kill 9, $child_pid;
 857     }
 858
 859     waitpid $child_pid, 0;
 860     $child_exit = $?;
 861
 862     if ($bug || $child_exit) {
 863         return 0 if $in_bisect;
 864         fail "test failed" and return 0;
 865     }
 866     return 1;
 867 }
 868
 869 sub run_git_bisect {
 870     my ($command) = @_;
 871
 872     doprint "$command ... ";
 873
 874     my $output = `$command 2>&1`;
 875     my $ret = $?;
 876
 877     logit $output;
 878
 879     if ($ret) {
 880         doprint "FAILED\n";
 881         dodie "Failed to git bisect";
 882     }
 883
 884     doprint "SUCCESS\n";
 885     if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
 886         doprint "$1 [$2]\n";
 887     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
 888         $bisect_bad = $1;
 889         doprint "Found bad commit... $1\n";
 890         return 0;
 891     } else {
 892         # we already logged it, just print it now.
 893         print $output;
 894     }
 895
 896     return 1;
 897 }
 898
 899 sub run_bisect {
 900     my ($type) = @_;
 901
 902     my $failed = 0;
 903     my $result;
 904     my $output;
 905     my $ret;
 906
 907     if (defined($minconfig)) {
 908         build "useconfig:$minconfig" or $failed = 1;
 909     } else {
 910         # ?? no config to use?
 911         build "oldconfig" or $failed = 1;
 912     }
 913
 914     if ($type ne "build") {
 915         dodie "Failed on build" if $failed;
 916
 917         # Now boot the box
 918         get_grub_index;
 919         get_version;
 920         install;
 921
 922         start_monitor;
 923         monitor or $failed = 1;
 924
 925         if ($type ne "boot") {
 926             dodie "Failed on boot" if $failed;
 927
 928             do_run_test or $failed = 1;
 929         }
 930         end_monitor;
 931     }
 932
 933     if ($failed) {
 934         $result = "bad";
 935
 936         # reboot the box to a good kernel
 937         if ($type ne "build") {
 938             doprint "Reboot and sleep $bisect_sleep_time seconds\n";
 939             reboot;
 940             start_monitor;
 941             wait_for_monitor $bisect_sleep_time;
 942             end_monitor;
 943         }
 944     } else {
 945         $result = "good";
 946     }
 947
 948     # Are we looking for where it worked, not failed?
 949     if ($reverse_bisect) {
 950         if ($failed) {
 951             $result = "good";
 952         } else {
 953             $result = "bad";
 954         }
 955     }
 956
 957     return $result;
 958 }
 959
 960 sub bisect {
 961     my ($i) = @_;
 962
 963     my $result;
 964
 965     die "BISECT_GOOD[$i] not defined\n" if (!defined($opt{"BISECT_GOOD[$i]"}));
 966     die "BISECT_BAD[$i] not defined\n"  if (!defined($opt{"BISECT_BAD[$i]"}));
 967     die "BISECT_TYPE[$i] not defined\n" if (!defined($opt{"BISECT_TYPE[$i]"}));
 968
 969     my $good = $opt{"BISECT_GOOD[$i]"};
 970     my $bad = $opt{"BISECT_BAD[$i]"};
 971     my $type = $opt{"BISECT_TYPE[$i]"};
 972     my $start = $opt{"BISECT_START[$i]"};
 973     my $replay = $opt{"BISECT_REPLAY[$i]"};
 974
 975     # convert to true sha1's
 976     $good = get_sha1($good);
 977     $bad = get_sha1($bad);
 978
 979     if (defined($opt{"BISECT_REVERSE[$i]"}) &&
 980         $opt{"BISECT_REVERSE[$i]"} == 1) {
 981         doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
 982         $reverse_bisect = 1;
 983     } else {
 984         $reverse_bisect = 0;
 985     }
 986
 987     $in_bisect = 1;
 988
 989     # Can't have a test without having a test to run
 990     if ($type eq "test" && !defined($run_test)) {
 991         $type = "boot";
 992     }
 993
 994     my $check = $opt{"BISECT_CHECK[$i]"};
 995     if (defined($check) && $check ne "0") {
 996
 997         # get current HEAD
 998         my $head = get_sha1("HEAD");
 999
1000         if ($check ne "good") {
1001             doprint "TESTING BISECT BAD [$bad]\n";
1002             run_command "git checkout $bad" or
1003                 die "Failed to checkout $bad";
1004
1005             $result = run_bisect $type;
1006
1007             if ($result ne "bad") {
1008                 fail "Tested BISECT_BAD [$bad] and it succeeded" and return 0;
1009             }
1010         }
1011
1012         if ($check ne "bad") {
1013             doprint "TESTING BISECT GOOD [$good]\n";
1014             run_command "git checkout $good" or
1015                 die "Failed to checkout $good";
1016
1017             $result = run_bisect $type;
1018
1019             if ($result ne "good") {
1020                 fail "Tested BISECT_GOOD [$good] and it failed" and return 0;
1021             }
1022         }
1023
1024         # checkout where we started
1025         run_command "git checkout $head" or
1026             die "Failed to checkout $head";
1027     }
1028
1029     run_command "git bisect start" or
1030         dodie "could not start bisect";
1031
1032     run_command "git bisect good $good" or
1033         dodie "could not set bisect good to $good";
1034
1035     run_git_bisect "git bisect bad $bad" or
1036         dodie "could not set bisect bad to $bad";
1037
1038     if (defined($replay)) {
1039         run_command "git bisect replay $replay" or
1040             dodie "failed to run replay";
1041     }
1042
1043     if (defined($start)) {
1044         run_command "git checkout $start" or
1045             dodie "failed to checkout $start";
1046     }
1047
1048     my $test;
1049     do {
1050         $result = run_bisect $type;
1051         $test = run_git_bisect "git bisect $result";
1052     } while ($test);
1053
1054     run_command "git bisect log" or
1055         dodie "could not capture git bisect log";
1056
1057     run_command "git bisect reset" or
1058         dodie "could not reset git bisect";
1059
1060     doprint "Bad commit was [$bisect_bad]\n";
1061
1062     $in_bisect = 0;
1063
1064     success $i;
1065 }
1066
1067 sub patchcheck {
1068     my ($i) = @_;
1069
1070     die "PATCHCHECK_START[$i] not defined\n"
1071         if (!defined($opt{"PATCHCHECK_START[$i]"}));
1072     die "PATCHCHECK_TYPE[$i] not defined\n"
1073         if (!defined($opt{"PATCHCHECK_TYPE[$i]"}));
1074
1075     my $start = $opt{"PATCHCHECK_START[$i]"};
1076
1077     my $end = "HEAD";
1078     if (defined($opt{"PATCHCHECK_END[$i]"})) {
1079         $end = $opt{"PATCHCHECK_END[$i]"};
1080     }
1081
1082     # Get the true sha1's since we can use things like HEAD~3
1083     $start = get_sha1($start);
1084     $end = get_sha1($end);
1085
1086     my $type = $opt{"PATCHCHECK_TYPE[$i]"};
1087
1088     # Can't have a test without having a test to run
1089     if ($type eq "test" && !defined($run_test)) {
1090         $type = "boot";
1091     }
1092
1093     open (IN, "git log --pretty=oneline $end|") or
1094         dodie "could not get git list";
1095
1096     my @list;
1097
1098     while (<IN>) {
1099         chomp;
1100         $list[$#list+1] = $_;
1101         last if (/^$start/);
1102     }
1103     close(IN);
1104
1105     if ($list[$#list] !~ /^$start/) {
1106         fail "SHA1 $start not found";
1107     }
1108
1109     # go backwards in the list
1110     @list = reverse @list;
1111
1112     my $save_clean = $noclean;
1113
1114     $in_patchcheck = 1;
1115     foreach my $item (@list) {
1116         my $sha1 = $item;
1117         $sha1 =~ s/^([[:xdigit:]]+).*/$1/;
1118
1119         doprint "\nProcessing commit $item\n\n";
1120
1121         run_command "git checkout $sha1" or
1122             die "Failed to checkout $sha1";
1123
1124         # only clean on the first and last patch
1125         if ($item eq $list[0] ||
1126             $item eq $list[$#list]) {
1127             $noclean = $save_clean;
1128         } else {
1129             $noclean = 1;
1130         }
1131
1132         if (defined($minconfig)) {
1133             build "useconfig:$minconfig" or return 0;
1134         } else {
1135             # ?? no config to use?
1136             build "oldconfig" or return 0;
1137         }
1138
1139         check_buildlog $sha1 or return 0;
1140
1141         next if ($type eq "build");
1142
1143         get_grub_index;
1144         get_version;
1145         install;
1146
1147         my $failed = 0;
1148
1149         start_monitor;
1150         monitor or $failed = 1;
1151
1152         if (!$failed && $type ne "boot"){
1153             do_run_test or $failed = 1;
1154         }
1155         end_monitor;
1156         return 0 if ($failed);
1157
1158     }
1159     $in_patchcheck = 0;
1160     success $i;
1161
1162     return 1;
1163 }
1164
1165 read_config $ARGV[0];
1166
1167 # mandatory configs
1168 die "MACHINE not defined\n"             if (!defined($opt{"MACHINE"}));
1169 die "SSH_USER not defined\n"            if (!defined($opt{"SSH_USER"}));
1170 die "BUILD_DIR not defined\n"           if (!defined($opt{"BUILD_DIR"}));
1171 die "OUTPUT_DIR not defined\n"          if (!defined($opt{"OUTPUT_DIR"}));
1172 die "BUILD_TARGET not defined\n"        if (!defined($opt{"BUILD_TARGET"}));
1173 die "TARGET_IMAGE not defined\n"        if (!defined($opt{"TARGET_IMAGE"}));
1174 die "POWER_CYCLE not defined\n"         if (!defined($opt{"POWER_CYCLE"}));
1175 die "CONSOLE not defined\n"             if (!defined($opt{"CONSOLE"}));
1176 die "LOCALVERSION not defined\n"        if (!defined($opt{"LOCALVERSION"}));
1177
1178 if ($opt{"CLEAR_LOG"} && defined($opt{"LOG_FILE"})) {
1179     unlink $opt{"LOG_FILE"};
1180 }
1181
1182 doprint "\n\nSTARTING AUTOMATED TESTS\n\n";
1183
1184 for (my $i = 0, my $repeat = 1; $i <= $opt{"NUM_TESTS"}; $i += $repeat) {
1185
1186     if (!$i) {
1187         doprint "DEFAULT OPTIONS:\n";
1188     } else {
1189         doprint "\nTEST $i OPTIONS";
1190         if (defined($repeat_tests{$i})) {
1191             $repeat = $repeat_tests{$i};
1192             doprint " ITERATE $repeat";
1193         }
1194         doprint "\n";
1195     }
1196
1197     foreach my $option (sort keys %opt) {
1198
1199         if ($option =~ /\[(\d+)\]$/) {
1200             next if ($i != $1);
1201         } else {
1202             next if ($i);
1203         }
1204
1205         doprint "$option = $opt{$option}\n";
1206     }
1207 }
1208
1209 sub set_test_option {
1210     my ($name, $i) = @_;
1211
1212     my $option = "$name\[$i\]";
1213
1214     if (defined($opt{$option})) {
1215         return $opt{$option};
1216     }
1217
1218     foreach my $test (keys %repeat_tests) {
1219         if ($i >= $test &&
1220             $i < $test + $repeat_tests{$test}) {
1221             $option = "$name\[$test\]";
1222             if (defined($opt{$option})) {
1223                 return $opt{$option};
1224             }
1225         }
1226     }
1227
1228     if (defined($opt{$name})) {
1229         return $opt{$name};
1230     }
1231
1232     return undef;
1233 }
1234
1235 # First we need to do is the builds
1236 for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
1237
1238     $iteration = $i;
1239
1240     my $ssh_user = set_test_option("SSH_USER", $i);
1241     my $makecmd = set_test_option("MAKE_CMD", $i);
1242
1243     $machine = set_test_option("MACHINE", $i);
1244     $tmpdir = set_test_option("TMP_DIR", $i);
1245     $outputdir = set_test_option("OUTPUT_DIR", $i);
1246     $builddir = set_test_option("BUILD_DIR", $i);
1247     $test_type = set_test_option("TEST_TYPE", $i);
1248     $build_type = set_test_option("BUILD_TYPE", $i);
1249     $build_options = set_test_option("BUILD_OPTIONS", $i);
1250     $power_cycle = set_test_option("POWER_CYCLE", $i);
1251     $noclean = set_test_option("BUILD_NOCLEAN", $i);
1252     $minconfig = set_test_option("MIN_CONFIG", $i);
1253     $run_test = set_test_option("TEST", $i);
1254     $addconfig = set_test_option("ADD_CONFIG", $i);
1255     $reboot_type = set_test_option("REBOOT_TYPE", $i);
1256     $grub_menu = set_test_option("GRUB_MENU", $i);
1257     $post_install = set_test_option("POST_INSTALL", $i);
1258     $reboot_script = set_test_option("REBOOT_SCRIPT", $i);
1259     $reboot_on_error = set_test_option("REBOOT_ON_ERROR", $i);
1260     $poweroff_on_error = set_test_option("POWEROFF_ON_ERROR", $i);
1261     $die_on_failure = set_test_option("DIE_ON_FAILURE", $i);
1262     $power_off = set_test_option("POWER_OFF", $i);
1263     $powercycle_after_reboot = set_test_option("POWERCYCLE_AFTER_REBOOT", $i);
1264     $poweroff_after_halt = set_test_option("POWEROFF_AFTER_HALT", $i);
1265     $sleep_time = set_test_option("SLEEP_TIME", $i);
1266     $bisect_sleep_time = set_test_option("BISECT_SLEEP_TIME", $i);
1267     $store_failures = set_test_option("STORE_FAILURES", $i);
1268     $timeout = set_test_option("TIMEOUT", $i);
1269     $booted_timeout = set_test_option("BOOTED_TIMEOUT", $i);
1270     $console = set_test_option("CONSOLE", $i);
1271     $success_line = set_test_option("SUCCESS_LINE", $i);
1272     $build_target = set_test_option("BUILD_TARGET", $i);
1273     $target_image = set_test_option("TARGET_IMAGE", $i);
1274     $localversion = set_test_option("LOCALVERSION", $i);
1275
1276     chdir $builddir || die "can't change directory to $builddir";
1277
1278     if (!-d $tmpdir) {
1279         mkpath($tmpdir) or
1280             die "can't create $tmpdir";
1281     }
1282
1283     $target = "$ssh_user\@$machine";
1284
1285     $buildlog = "$tmpdir/buildlog-$machine";
1286     $dmesg = "$tmpdir/dmesg-$machine";
1287     $make = "$makecmd O=$outputdir";
1288
1289     if ($reboot_type eq "grub") {
1290         dodie "GRUB_MENU not defined" if (!defined($grub_menu));
1291     } elsif (!defined($reboot_script)) {
1292         dodie "REBOOT_SCRIPT not defined"
1293     }
1294
1295     my $run_type = $build_type;
1296     if ($test_type eq "patchcheck") {
1297         $run_type = $opt{"PATCHCHECK_TYPE[$i]"};
1298     } elsif ($test_type eq "bisect") {
1299         $run_type = $opt{"BISECT_TYPE[$i]"};
1300     }
1301
1302     # mistake in config file?
1303     if (!defined($run_type)) {
1304         $run_type = "ERROR";
1305     }
1306
1307     doprint "\n\n";
1308     doprint "RUNNING TEST $i of $opt{NUM_TESTS} with option $test_type $run_type\n\n";
1309
1310     unlink $dmesg;
1311     unlink $buildlog;
1312
1313     if (!defined($minconfig)) {
1314         $minconfig = $addconfig;
1315
1316     } elsif (defined($addconfig)) {
1317         run_command "cat $addconfig $minconfig > $tmpdir/use_config" or
1318             dodie "Failed to create temp config";
1319         $minconfig = "$tmpdir/use_config";
1320     }
1321
1322     my $checkout = $opt{"CHECKOUT[$i]"};
1323     if (defined($checkout)) {
1324         run_command "git checkout $checkout" or
1325             die "failed to checkout $checkout";
1326     }
1327
1328     if ($test_type eq "bisect") {
1329         bisect $i;
1330         next;
1331     } elsif ($test_type eq "patchcheck") {
1332         patchcheck $i;
1333         next;
1334     }
1335
1336     if ($build_type ne "nobuild") {
1337         build $build_type or next;
1338     }
1339
1340     if ($test_type ne "build") {
1341         get_grub_index;
1342         get_version;
1343         install;
1344
1345         my $failed = 0;
1346         start_monitor;
1347         monitor or $failed = 1;;
1348
1349         if (!$failed && $test_type ne "boot" && defined($run_test)) {
1350             do_run_test or $failed = 1;
1351         }
1352         end_monitor;
1353         next if ($failed);
1354     }
1355
1356     success $i;
1357 }
1358
1359 if ($opt{"POWEROFF_ON_SUCCESS"}) {
1360     halt;
1361 } elsif ($opt{"REBOOT_ON_SUCCESS"} && !do_not_reboot) {
1362     reboot;
1363 }
1364
1365 exit 0;