Commit | Line | Data |
---|---|---|
2545eb61 | 1 | #!/usr/bin/perl -w |
d6ce2a0b SR |
2 | # |
3 | # Copywrite 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc. | |
4 | # Licensed under the terms of the GNU GPL License version 2 | |
5 | # | |
2545eb61 SR |
6 | |
7 | use strict; | |
8 | use IPC::Open2; | |
9 | use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK); | |
7faafbd6 SR |
10 | use File::Path qw(mkpath); |
11 | use File::Copy qw(cp); | |
2545eb61 SR |
12 | use FileHandle; |
13 | ||
14 | $#ARGV >= 0 || die "usage: autotest.pl config-file\n"; | |
15 | ||
16 | $| = 1; | |
17 | ||
18 | my %opt; | |
19 | ||
20 | #default opts | |
21 | $opt{"NUM_BUILDS"} = 5; | |
7faafbd6 | 22 | $opt{"BUILD_TYPE"} = "randconfig"; |
2545eb61 | 23 | $opt{"MAKE_CMD"} = "make"; |
2b7d9b21 | 24 | $opt{"TIMEOUT"} = 120; |
2545eb61 SR |
25 | $opt{"TMP_DIR"} = "/tmp/autotest"; |
26 | $opt{"SLEEP_TIME"} = 60; # sleep time between tests | |
5c42fc5b | 27 | $opt{"BUILD_NOCLEAN"} = 0; |
75c3fda7 | 28 | $opt{"REBOOT_ON_ERROR"} = 0; |
5c42fc5b | 29 | $opt{"POWEROFF_ON_ERROR"} = 0; |
1a5cfce3 | 30 | $opt{"REBOOT_ON_SUCCESS"} = 1; |
5c42fc5b | 31 | $opt{"POWEROFF_ON_SUCCESS"} = 0; |
75c3fda7 | 32 | $opt{"BUILD_OPTIONS"} = ""; |
5a391fbf | 33 | $opt{"BISECT_SLEEP_TIME"} = 10; # sleep time between bisects |
2b7d9b21 SR |
34 | $opt{"CLEAR_LOG"} = 0; |
35 | $opt{"SUCCESS_LINE"} = "login:"; | |
36 | $opt{"BOOTED_TIMEOUT"} = 1; | |
37 | $opt{"DIE_ON_FAILURE"} = 1; | |
2545eb61 SR |
38 | |
39 | my $version; | |
7faafbd6 | 40 | my $build_type; |
2545eb61 SR |
41 | my $grub_number; |
42 | my $target; | |
43 | my $make; | |
5c42fc5b | 44 | my $noclean; |
5f9b6ced | 45 | my $minconfig; |
2b7d9b21 | 46 | my $addconfig; |
5f9b6ced SR |
47 | my $in_bisect = 0; |
48 | my $bisect_bad = ""; | |
d6ce2a0b | 49 | my $reverse_bisect; |
6c5ee0be | 50 | my $in_patchcheck = 0; |
5a391fbf | 51 | my $run_test; |
6c5ee0be | 52 | my $redirect; |
7faafbd6 SR |
53 | my $buildlog; |
54 | my $dmesg; | |
55 | my $monitor_fp; | |
56 | my $monitor_pid; | |
57 | my $monitor_cnt = 0; | |
2545eb61 SR |
58 | |
59 | sub read_config { | |
60 | my ($config) = @_; | |
61 | ||
62 | open(IN, $config) || die "can't read file $config"; | |
63 | ||
64 | while (<IN>) { | |
65 | ||
66 | # ignore blank lines and comments | |
67 | next if (/^\s*$/ || /\s*\#/); | |
68 | ||
69 | if (/^\s*(\S+)\s*=\s*(.*?)\s*$/) { | |
70 | my $lvalue = $1; | |
71 | my $rvalue = $2; | |
72 | ||
73 | $opt{$lvalue} = $rvalue; | |
74 | } | |
75 | } | |
76 | ||
77 | close(IN); | |
78 | } | |
79 | ||
5f9b6ced | 80 | sub logit { |
2545eb61 SR |
81 | if (defined($opt{"LOG_FILE"})) { |
82 | open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}"; | |
83 | print OUT @_; | |
84 | close(OUT); | |
85 | } | |
86 | } | |
87 | ||
5f9b6ced SR |
88 | sub doprint { |
89 | print @_; | |
90 | logit @_; | |
91 | } | |
92 | ||
7faafbd6 SR |
93 | sub run_command; |
94 | ||
95 | sub reboot { | |
96 | # try to reboot normally | |
97 | if (!run_command "ssh $target reboot") { | |
98 | # nope? power cycle it. | |
99 | run_command "$opt{POWER_CYCLE}"; | |
100 | } | |
101 | } | |
102 | ||
5c42fc5b | 103 | sub dodie { |
5a391fbf | 104 | doprint "CRITICAL FAILURE... ", @_, "\n"; |
5c42fc5b | 105 | |
75c3fda7 SR |
106 | if ($opt{"REBOOT_ON_ERROR"}) { |
107 | doprint "REBOOTING\n"; | |
7faafbd6 | 108 | reboot; |
75c3fda7 SR |
109 | |
110 | } elsif ($opt{"POWEROFF_ON_ERROR"} && defined($opt{"POWER_OFF"})) { | |
5c42fc5b SR |
111 | doprint "POWERING OFF\n"; |
112 | `$opt{"POWER_OFF"}`; | |
113 | } | |
75c3fda7 | 114 | |
5c42fc5b SR |
115 | die @_; |
116 | } | |
117 | ||
7faafbd6 SR |
118 | sub open_console { |
119 | my ($fp) = @_; | |
120 | ||
121 | my $flags; | |
122 | ||
123 | my $pid = open($fp, "$opt{CONSOLE}|") or | |
124 | dodie "Can't open console $opt{CONSOLE}"; | |
125 | ||
126 | $flags = fcntl($fp, F_GETFL, 0) or | |
127 | dodie "Can't get flags for the socket: $!\n"; | |
128 | $flags = fcntl($fp, F_SETFL, $flags | O_NONBLOCK) or | |
129 | dodie "Can't set flags for the socket: $!\n"; | |
130 | ||
131 | return $pid; | |
132 | } | |
133 | ||
134 | sub close_console { | |
135 | my ($fp, $pid) = @_; | |
136 | ||
137 | doprint "kill child process $pid\n"; | |
138 | kill 2, $pid; | |
139 | ||
140 | print "closing!\n"; | |
141 | close($fp); | |
142 | } | |
143 | ||
144 | sub start_monitor { | |
145 | if ($monitor_cnt++) { | |
146 | return; | |
147 | } | |
148 | $monitor_fp = \*MONFD; | |
149 | $monitor_pid = open_console $monitor_fp; | |
150 | } | |
151 | ||
152 | sub end_monitor { | |
153 | if (--$monitor_cnt) { | |
154 | return; | |
155 | } | |
156 | close_console($monitor_fp, $monitor_pid); | |
157 | } | |
158 | ||
159 | sub wait_for_monitor { | |
160 | my ($time) = @_; | |
161 | my $line; | |
162 | ||
163 | doprint "Wait for monitor to settle down.\n"; | |
164 | ||
165 | # read the monitor and wait for the system to calm down | |
166 | do { | |
167 | $line = wait_for_input($monitor_fp, $time); | |
168 | } while (defined($line)); | |
169 | } | |
170 | ||
2b7d9b21 SR |
171 | sub fail { |
172 | ||
173 | if ($opt{"DIE_ON_FAILURE"}) { | |
174 | dodie @_; | |
175 | } | |
176 | ||
177 | doprint "Failed: ", @_, "\n"; | |
7faafbd6 SR |
178 | |
179 | doprint "REBOOTING\n"; | |
180 | reboot; | |
181 | start_monitor; | |
182 | wait_for_monitor $opt{"SLEEP_TIME"}; | |
183 | end_monitor; | |
184 | ||
185 | return 1 if (!defined($opt{"STORE_FAILURES"})); | |
186 | ||
187 | my @t = localtime; | |
188 | my $date = sprintf "%04d%02d%02d%02d%02d%02d", | |
189 | 1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0]; | |
190 | ||
191 | my $dir = "$opt{MACHINE}-$build_type-fail-$date"; | |
192 | my $faildir = "$opt{STORE_FAILURES}/$dir"; | |
193 | ||
194 | if (!-d $faildir) { | |
195 | mkpath($faildir) or | |
196 | die "can't create $opt{STORE_FAILURES}"; | |
197 | } | |
198 | if (-f "$opt{OUTPUT_DIR}/.config") { | |
199 | cp "$opt{OUTPUT_DIR}/.config", "$faildir/config" or | |
200 | die "failed to copy .config"; | |
201 | } | |
202 | if (-f $buildlog) { | |
203 | cp $buildlog, "$faildir/buildlog" or | |
204 | die "failed to move $buildlog"; | |
205 | } | |
206 | if (-f $dmesg) { | |
207 | cp $dmesg, "$faildir/dmesg" or | |
208 | die "failed to move $dmesg"; | |
209 | } | |
210 | ||
211 | doprint "*** Saved info to $faildir ***\n"; | |
212 | ||
2b7d9b21 SR |
213 | return 1; |
214 | } | |
215 | ||
2545eb61 SR |
216 | sub run_command { |
217 | my ($command) = @_; | |
d6ce2a0b SR |
218 | my $dolog = 0; |
219 | my $dord = 0; | |
220 | my $pid; | |
221 | ||
222 | doprint("$command ... "); | |
223 | ||
224 | $pid = open(CMD, "$command 2>&1 |") or | |
2b7d9b21 | 225 | (fail "unable to exec $command" and return 0); |
2545eb61 SR |
226 | |
227 | if (defined($opt{"LOG_FILE"})) { | |
d6ce2a0b SR |
228 | open(LOG, ">>$opt{LOG_FILE}") or |
229 | dodie "failed to write to log"; | |
230 | $dolog = 1; | |
6c5ee0be SR |
231 | } |
232 | ||
233 | if (defined($redirect)) { | |
d6ce2a0b SR |
234 | open (RD, ">$redirect") or |
235 | dodie "failed to write to redirect $redirect"; | |
236 | $dord = 1; | |
2545eb61 SR |
237 | } |
238 | ||
d6ce2a0b SR |
239 | while (<CMD>) { |
240 | print LOG if ($dolog); | |
241 | print RD if ($dord); | |
242 | } | |
2545eb61 | 243 | |
d6ce2a0b | 244 | waitpid($pid, 0); |
2545eb61 SR |
245 | my $failed = $?; |
246 | ||
d6ce2a0b SR |
247 | close(CMD); |
248 | close(LOG) if ($dolog); | |
249 | close(RD) if ($dord); | |
250 | ||
2545eb61 SR |
251 | if ($failed) { |
252 | doprint "FAILED!\n"; | |
253 | } else { | |
254 | doprint "SUCCESS\n"; | |
255 | } | |
256 | ||
5f9b6ced SR |
257 | return !$failed; |
258 | } | |
259 | ||
260 | sub get_grub_index { | |
261 | ||
5a391fbf | 262 | return if (defined($grub_number)); |
5f9b6ced SR |
263 | |
264 | doprint "Find grub menu ... "; | |
265 | $grub_number = -1; | |
266 | open(IN, "ssh $target cat /boot/grub/menu.lst |") | |
267 | or die "unable to get menu.lst"; | |
268 | while (<IN>) { | |
269 | if (/^\s*title\s+$opt{GRUB_MENU}\s*$/) { | |
270 | $grub_number++; | |
271 | last; | |
272 | } elsif (/^\s*title\s/) { | |
273 | $grub_number++; | |
274 | } | |
275 | } | |
276 | close(IN); | |
277 | ||
278 | die "Could not find '$opt{GRUB_MENU}' in /boot/grub/menu on $opt{MACHINE}" | |
279 | if ($grub_number < 0); | |
280 | doprint "$grub_number\n"; | |
2545eb61 SR |
281 | } |
282 | ||
283 | my $timeout = $opt{"TIMEOUT"}; | |
284 | ||
285 | sub wait_for_input | |
286 | { | |
287 | my ($fp, $time) = @_; | |
288 | my $rin; | |
289 | my $ready; | |
290 | my $line; | |
291 | my $ch; | |
292 | ||
293 | if (!defined($time)) { | |
294 | $time = $timeout; | |
295 | } | |
296 | ||
297 | $rin = ''; | |
298 | vec($rin, fileno($fp), 1) = 1; | |
299 | $ready = select($rin, undef, undef, $time); | |
300 | ||
301 | $line = ""; | |
302 | ||
303 | # try to read one char at a time | |
304 | while (sysread $fp, $ch, 1) { | |
305 | $line .= $ch; | |
306 | last if ($ch eq "\n"); | |
307 | } | |
308 | ||
309 | if (!length($line)) { | |
310 | return undef; | |
311 | } | |
312 | ||
313 | return $line; | |
314 | } | |
315 | ||
75c3fda7 | 316 | sub reboot_to { |
2545eb61 SR |
317 | run_command "ssh $target '(echo \"savedefault --default=$grub_number --once\" | grub --batch; reboot)'"; |
318 | } | |
319 | ||
5a391fbf | 320 | sub monitor { |
2545eb61 SR |
321 | my $booted = 0; |
322 | my $bug = 0; | |
5c42fc5b | 323 | my $skip_call_trace = 0; |
2b7d9b21 | 324 | my $loops; |
2545eb61 | 325 | |
7faafbd6 | 326 | wait_for_monitor 5; |
2545eb61 SR |
327 | |
328 | my $line; | |
329 | my $full_line = ""; | |
330 | ||
7faafbd6 SR |
331 | open(DMESG, "> $dmesg") or |
332 | die "unable to write to $dmesg"; | |
2545eb61 | 333 | |
75c3fda7 | 334 | reboot_to; |
2545eb61 SR |
335 | |
336 | for (;;) { | |
337 | ||
2b7d9b21 | 338 | if ($booted) { |
7faafbd6 | 339 | $line = wait_for_input($monitor_fp, $opt{"BOOTED_TIMEOUT"}); |
2b7d9b21 | 340 | } else { |
7faafbd6 | 341 | $line = wait_for_input($monitor_fp); |
2b7d9b21 | 342 | } |
2545eb61 SR |
343 | |
344 | last if (!defined($line)); | |
345 | ||
346 | doprint $line; | |
7faafbd6 | 347 | print DMESG $line; |
2545eb61 SR |
348 | |
349 | # we are not guaranteed to get a full line | |
350 | $full_line .= $line; | |
351 | ||
2b7d9b21 | 352 | if ($full_line =~ /$opt{"SUCCESS_LINE"}/) { |
2545eb61 SR |
353 | $booted = 1; |
354 | } | |
355 | ||
5c42fc5b SR |
356 | if ($full_line =~ /\[ backtrace testing \]/) { |
357 | $skip_call_trace = 1; | |
358 | } | |
359 | ||
2545eb61 | 360 | if ($full_line =~ /call trace:/i) { |
5c42fc5b SR |
361 | $bug = 1 if (!$skip_call_trace); |
362 | } | |
363 | ||
364 | if ($full_line =~ /\[ end of backtrace testing \]/) { | |
365 | $skip_call_trace = 0; | |
366 | } | |
367 | ||
368 | if ($full_line =~ /Kernel panic -/) { | |
2545eb61 SR |
369 | $bug = 1; |
370 | } | |
371 | ||
372 | if ($line =~ /\n/) { | |
373 | $full_line = ""; | |
374 | } | |
375 | } | |
376 | ||
7faafbd6 | 377 | close(DMESG); |
2545eb61 SR |
378 | |
379 | if (!$booted) { | |
2b7d9b21 SR |
380 | return 0 if ($in_bisect); |
381 | fail "failed - never got a boot prompt.\n" and return 0; | |
2545eb61 SR |
382 | } |
383 | ||
384 | if ($bug) { | |
2b7d9b21 SR |
385 | return 0 if ($in_bisect); |
386 | fail "failed - got a bug report\n" and return 0; | |
2545eb61 | 387 | } |
5f9b6ced | 388 | |
2b7d9b21 | 389 | return 1; |
2545eb61 SR |
390 | } |
391 | ||
392 | sub install { | |
393 | ||
5f9b6ced | 394 | run_command "scp $opt{OUTPUT_DIR}/$opt{BUILD_TARGET} $target:$opt{TARGET_IMAGE}" or |
5c42fc5b | 395 | dodie "failed to copy image"; |
2545eb61 | 396 | |
5f9b6ced | 397 | my $install_mods = 0; |
2545eb61 | 398 | |
5f9b6ced SR |
399 | # should we process modules? |
400 | $install_mods = 0; | |
401 | open(IN, "$opt{OUTPUT_DIR}/.config") or dodie("Can't read config file"); | |
402 | while (<IN>) { | |
403 | if (/CONFIG_MODULES(=y)?/) { | |
404 | $install_mods = 1 if (defined($1)); | |
405 | last; | |
5c42fc5b | 406 | } |
5f9b6ced SR |
407 | } |
408 | close(IN); | |
5c42fc5b | 409 | |
5f9b6ced SR |
410 | if (!$install_mods) { |
411 | doprint "No modules needed\n"; | |
412 | return; | |
413 | } | |
2545eb61 | 414 | |
5f9b6ced SR |
415 | run_command "$make INSTALL_MOD_PATH=$opt{TMP_DIR} modules_install" or |
416 | dodie "Failed to install modules"; | |
5c42fc5b | 417 | |
5f9b6ced SR |
418 | my $modlib = "/lib/modules/$version"; |
419 | my $modtar = "autotest-mods.tar.bz2"; | |
5c42fc5b | 420 | |
5f9b6ced SR |
421 | run_command "ssh $target rm -rf $modlib" or |
422 | dodie "failed to remove old mods: $modlib"; | |
5c42fc5b | 423 | |
5f9b6ced SR |
424 | # would be nice if scp -r did not follow symbolic links |
425 | run_command "cd $opt{TMP_DIR} && tar -cjf $modtar lib/modules/$version" or | |
426 | dodie "making tarball"; | |
427 | ||
428 | run_command "scp $opt{TMP_DIR}/$modtar $target:/tmp" or | |
429 | dodie "failed to copy modules"; | |
430 | ||
431 | unlink "$opt{TMP_DIR}/$modtar"; | |
432 | ||
433 | run_command "ssh $target '(cd / && tar xf /tmp/$modtar)'" or | |
434 | dodie "failed to tar modules"; | |
2545eb61 | 435 | |
5f9b6ced | 436 | run_command "ssh $target rm -f /tmp/$modtar"; |
2545eb61 SR |
437 | } |
438 | ||
6c5ee0be SR |
439 | sub check_buildlog { |
440 | my ($patch) = @_; | |
441 | ||
6c5ee0be SR |
442 | my @files = `git show $patch | diffstat -l`; |
443 | ||
444 | open(IN, "git show $patch |") or | |
445 | dodie "failed to show $patch"; | |
446 | while (<IN>) { | |
447 | if (m,^--- a/(.*),) { | |
448 | chomp $1; | |
449 | $files[$#files] = $1; | |
450 | } | |
451 | } | |
452 | close(IN); | |
453 | ||
454 | open(IN, $buildlog) or dodie "Can't open $buildlog"; | |
455 | while (<IN>) { | |
456 | if (/^\s*(.*?):.*(warning|error)/) { | |
457 | my $err = $1; | |
458 | foreach my $file (@files) { | |
459 | my $fullpath = "$opt{BUILD_DIR}/$file"; | |
460 | if ($file eq $err || $fullpath eq $err) { | |
2b7d9b21 | 461 | fail "$file built with warnings" and return 0; |
6c5ee0be SR |
462 | } |
463 | } | |
464 | } | |
465 | } | |
466 | close(IN); | |
2b7d9b21 SR |
467 | |
468 | return 1; | |
6c5ee0be SR |
469 | } |
470 | ||
2545eb61 SR |
471 | sub build { |
472 | my ($type) = @_; | |
5c42fc5b SR |
473 | my $defconfig = ""; |
474 | my $append = ""; | |
475 | ||
7faafbd6 SR |
476 | unlink $buildlog; |
477 | ||
75c3fda7 | 478 | if ($type =~ /^useconfig:(.*)/) { |
5f9b6ced | 479 | run_command "cp $1 $opt{OUTPUT_DIR}/.config" or |
75c3fda7 | 480 | dodie "could not copy $1 to .config"; |
5f9b6ced | 481 | |
75c3fda7 SR |
482 | $type = "oldconfig"; |
483 | } | |
484 | ||
5c42fc5b SR |
485 | # old config can ask questions |
486 | if ($type eq "oldconfig") { | |
487 | $append = "yes ''|"; | |
75c3fda7 SR |
488 | |
489 | # allow for empty configs | |
490 | run_command "touch $opt{OUTPUT_DIR}/.config"; | |
491 | ||
5f9b6ced | 492 | run_command "mv $opt{OUTPUT_DIR}/.config $opt{OUTPUT_DIR}/config_temp" or |
5c42fc5b | 493 | dodie "moving .config"; |
2545eb61 | 494 | |
5f9b6ced | 495 | if (!$noclean && !run_command "$make mrproper") { |
5c42fc5b SR |
496 | dodie "make mrproper"; |
497 | } | |
2545eb61 | 498 | |
5f9b6ced | 499 | run_command "mv $opt{OUTPUT_DIR}/config_temp $opt{OUTPUT_DIR}/.config" or |
5c42fc5b | 500 | dodie "moving config_temp"; |
5c42fc5b SR |
501 | |
502 | } elsif (!$noclean) { | |
503 | unlink "$opt{OUTPUT_DIR}/.config"; | |
5f9b6ced | 504 | run_command "$make mrproper" or |
5c42fc5b | 505 | dodie "make mrproper"; |
5c42fc5b | 506 | } |
2545eb61 SR |
507 | |
508 | # add something to distinguish this build | |
5c42fc5b | 509 | open(OUT, "> $opt{OUTPUT_DIR}/localversion") or dodie("Can't make localversion file"); |
2545eb61 SR |
510 | print OUT "$opt{LOCALVERSION}\n"; |
511 | close(OUT); | |
512 | ||
5f9b6ced SR |
513 | if (defined($minconfig)) { |
514 | $defconfig = "KCONFIG_ALLCONFIG=$minconfig"; | |
2545eb61 SR |
515 | } |
516 | ||
5f9b6ced | 517 | run_command "$defconfig $append $make $type" or |
5c42fc5b | 518 | dodie "failed make config"; |
2545eb61 | 519 | |
7faafbd6 | 520 | $redirect = "$opt{TMP_DIR}/buildlog"; |
5f9b6ced | 521 | if (!run_command "$make $opt{BUILD_OPTIONS}") { |
6c5ee0be | 522 | undef $redirect; |
5f9b6ced | 523 | # bisect may need this to pass |
2b7d9b21 SR |
524 | return 0 if ($in_bisect); |
525 | fail "failed build" and return 0; | |
2545eb61 | 526 | } |
6c5ee0be | 527 | undef $redirect; |
5f9b6ced | 528 | |
2b7d9b21 | 529 | return 1; |
2545eb61 SR |
530 | } |
531 | ||
75c3fda7 | 532 | sub halt { |
5f9b6ced | 533 | if (!run_command "ssh $target halt" or defined($opt{"POWER_OFF"})) { |
75c3fda7 SR |
534 | # nope? the zap it! |
535 | run_command "$opt{POWER_OFF}"; | |
536 | } | |
537 | } | |
538 | ||
5f9b6ced SR |
539 | sub success { |
540 | my ($i) = @_; | |
541 | ||
542 | doprint "\n\n*******************************************\n"; | |
543 | doprint "*******************************************\n"; | |
544 | doprint "** SUCCESS!!!! **\n"; | |
545 | doprint "*******************************************\n"; | |
546 | doprint "*******************************************\n"; | |
547 | ||
548 | if ($i != $opt{"NUM_BUILDS"}) { | |
7faafbd6 | 549 | doprint "Reboot and wait $opt{SLEEP_TIME} seconds\n"; |
5f9b6ced | 550 | reboot; |
7faafbd6 SR |
551 | start_monitor; |
552 | wait_for_monitor $opt{"SLEEP_TIME"}; | |
553 | end_monitor; | |
5f9b6ced SR |
554 | } |
555 | } | |
556 | ||
557 | sub get_version { | |
558 | # get the release name | |
559 | doprint "$make kernelrelease ... "; | |
560 | $version = `$make kernelrelease | tail -1`; | |
561 | chomp($version); | |
562 | doprint "$version\n"; | |
563 | } | |
564 | ||
5a391fbf | 565 | sub child_run_test { |
7faafbd6 | 566 | my $failed = 0; |
5a391fbf | 567 | |
7faafbd6 SR |
568 | # child should have no power |
569 | $opt{"REBOOT_ON_ERROR"} = 0; | |
570 | $opt{"POWEROFF_ON_ERROR"} = 0; | |
571 | $opt{"DIE_ON_FAILURE"} = 1; | |
572 | ||
573 | run_command $run_test or $failed = 1; | |
5a391fbf SR |
574 | exit $failed; |
575 | } | |
576 | ||
577 | my $child_done; | |
578 | ||
579 | sub child_finished { | |
580 | $child_done = 1; | |
581 | } | |
582 | ||
583 | sub do_run_test { | |
584 | my $child_pid; | |
585 | my $child_exit; | |
5a391fbf SR |
586 | my $line; |
587 | my $full_line; | |
588 | my $bug = 0; | |
5a391fbf | 589 | |
7faafbd6 | 590 | wait_for_monitor 1; |
5a391fbf | 591 | |
7faafbd6 | 592 | doprint "run test $run_test\n"; |
5a391fbf SR |
593 | |
594 | $child_done = 0; | |
595 | ||
596 | $SIG{CHLD} = qw(child_finished); | |
597 | ||
598 | $child_pid = fork; | |
599 | ||
600 | child_run_test if (!$child_pid); | |
601 | ||
602 | $full_line = ""; | |
603 | ||
604 | do { | |
7faafbd6 | 605 | $line = wait_for_input($monitor_fp, 1); |
5a391fbf SR |
606 | if (defined($line)) { |
607 | ||
608 | # we are not guaranteed to get a full line | |
609 | $full_line .= $line; | |
610 | ||
611 | if ($full_line =~ /call trace:/i) { | |
612 | $bug = 1; | |
613 | } | |
614 | ||
615 | if ($full_line =~ /Kernel panic -/) { | |
616 | $bug = 1; | |
617 | } | |
618 | ||
619 | if ($line =~ /\n/) { | |
620 | $full_line = ""; | |
621 | } | |
622 | } | |
623 | } while (!$child_done && !$bug); | |
624 | ||
625 | if ($bug) { | |
626 | doprint "Detected kernel crash!\n"; | |
627 | # kill the child with extreme prejudice | |
628 | kill 9, $child_pid; | |
629 | } | |
630 | ||
631 | waitpid $child_pid, 0; | |
632 | $child_exit = $?; | |
633 | ||
5a391fbf | 634 | if ($bug || $child_exit) { |
2b7d9b21 SR |
635 | return 0 if $in_bisect; |
636 | fail "test failed" and return 0; | |
5a391fbf | 637 | } |
2b7d9b21 | 638 | return 1; |
5a391fbf SR |
639 | } |
640 | ||
5f9b6ced SR |
641 | sub run_bisect { |
642 | my ($type) = @_; | |
643 | ||
2b7d9b21 | 644 | my $failed = 0; |
5f9b6ced SR |
645 | my $result; |
646 | my $output; | |
647 | my $ret; | |
648 | ||
5f9b6ced | 649 | if (defined($minconfig)) { |
2b7d9b21 | 650 | build "useconfig:$minconfig" or $failed = 1; |
5f9b6ced SR |
651 | } else { |
652 | # ?? no config to use? | |
2b7d9b21 | 653 | build "oldconfig" or $failed = 1; |
5f9b6ced SR |
654 | } |
655 | ||
656 | if ($type ne "build") { | |
7faafbd6 | 657 | dodie "Failed on build" if $failed; |
5f9b6ced SR |
658 | |
659 | # Now boot the box | |
660 | get_grub_index; | |
661 | get_version; | |
662 | install; | |
7faafbd6 SR |
663 | |
664 | start_monitor; | |
2b7d9b21 | 665 | monitor or $failed = 1; |
5f9b6ced SR |
666 | |
667 | if ($type ne "boot") { | |
7faafbd6 | 668 | dodie "Failed on boot" if $failed; |
5a391fbf | 669 | |
2b7d9b21 | 670 | do_run_test or $failed = 1; |
5f9b6ced | 671 | } |
7faafbd6 | 672 | end_monitor; |
5f9b6ced SR |
673 | } |
674 | ||
675 | if ($failed) { | |
676 | $result = "bad"; | |
5a391fbf SR |
677 | |
678 | # reboot the box to a good kernel | |
679 | if ($type eq "boot") { | |
7faafbd6 | 680 | doprint "Reboot and sleep $opt{BISECT_SLEEP_TIME} seconds\n"; |
5a391fbf | 681 | reboot; |
7faafbd6 SR |
682 | start_monitor; |
683 | wait_for_monitor $opt{"BISECT_SLEEP_TIME"}; | |
684 | end_monitor; | |
5a391fbf | 685 | } |
5f9b6ced SR |
686 | } else { |
687 | $result = "good"; | |
688 | } | |
689 | ||
d6ce2a0b SR |
690 | # Are we looking for where it worked, not failed? |
691 | if ($reverse_bisect) { | |
692 | if ($failed) { | |
693 | $result = "good"; | |
694 | } else { | |
695 | $result = "bad"; | |
696 | } | |
697 | } | |
698 | ||
5f9b6ced SR |
699 | doprint "git bisect $result ... "; |
700 | $output = `git bisect $result 2>&1`; | |
701 | $ret = $?; | |
702 | ||
703 | logit $output; | |
704 | ||
705 | if ($ret) { | |
706 | doprint "FAILED\n"; | |
2b7d9b21 | 707 | fail "Failed to git bisect"; |
5f9b6ced SR |
708 | } |
709 | ||
710 | doprint "SUCCESS\n"; | |
5a391fbf | 711 | if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) { |
5f9b6ced SR |
712 | doprint "$1 [$2]\n"; |
713 | } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) { | |
714 | $bisect_bad = $1; | |
715 | doprint "Found bad commit... $1\n"; | |
716 | return 0; | |
5a391fbf SR |
717 | } else { |
718 | # we already logged it, just print it now. | |
719 | print $output; | |
5f9b6ced SR |
720 | } |
721 | ||
722 | ||
723 | return 1; | |
724 | } | |
725 | ||
726 | sub bisect { | |
727 | my ($i) = @_; | |
728 | ||
729 | my $result; | |
730 | ||
731 | die "BISECT_GOOD[$i] not defined\n" if (!defined($opt{"BISECT_GOOD[$i]"})); | |
732 | die "BISECT_BAD[$i] not defined\n" if (!defined($opt{"BISECT_BAD[$i]"})); | |
733 | die "BISECT_TYPE[$i] not defined\n" if (!defined($opt{"BISECT_TYPE[$i]"})); | |
734 | ||
735 | my $good = $opt{"BISECT_GOOD[$i]"}; | |
736 | my $bad = $opt{"BISECT_BAD[$i]"}; | |
737 | my $type = $opt{"BISECT_TYPE[$i]"}; | |
738 | ||
d6ce2a0b SR |
739 | if (defined($opt{"BISECT_REVERSE[$i]"}) && |
740 | $opt{"BISECT_REVERSE[$i]"} == 1) { | |
741 | doprint "Performing a reverse bisect (bad is good, good is bad!)\n"; | |
742 | $reverse_bisect = 1; | |
743 | } else { | |
744 | $reverse_bisect = 0; | |
745 | } | |
746 | ||
5f9b6ced SR |
747 | $in_bisect = 1; |
748 | ||
749 | run_command "git bisect start" or | |
2b7d9b21 | 750 | fail "could not start bisect"; |
5f9b6ced SR |
751 | |
752 | run_command "git bisect good $good" or | |
2b7d9b21 | 753 | fail "could not set bisect good to $good"; |
5f9b6ced SR |
754 | |
755 | run_command "git bisect bad $bad" or | |
2b7d9b21 | 756 | fail "could not set bisect good to $bad"; |
5f9b6ced | 757 | |
5a391fbf SR |
758 | # Can't have a test without having a test to run |
759 | if ($type eq "test" && !defined($run_test)) { | |
760 | $type = "boot"; | |
761 | } | |
762 | ||
5f9b6ced SR |
763 | do { |
764 | $result = run_bisect $type; | |
765 | } while ($result); | |
766 | ||
767 | run_command "git bisect log" or | |
768 | dodie "could not capture git bisect log"; | |
769 | ||
770 | run_command "git bisect reset" or | |
771 | dodie "could not reset git bisect"; | |
772 | ||
773 | doprint "Bad commit was [$bisect_bad]\n"; | |
774 | ||
775 | $in_bisect = 0; | |
776 | ||
777 | success $i; | |
778 | } | |
779 | ||
6c5ee0be SR |
780 | sub patchcheck { |
781 | my ($i) = @_; | |
782 | ||
783 | die "PATCHCHECK_START[$i] not defined\n" | |
784 | if (!defined($opt{"PATCHCHECK_START[$i]"})); | |
785 | die "PATCHCHECK_TYPE[$i] not defined\n" | |
786 | if (!defined($opt{"PATCHCHECK_TYPE[$i]"})); | |
787 | ||
788 | my $start = $opt{"PATCHCHECK_START[$i]"}; | |
789 | ||
790 | my $end = "HEAD"; | |
791 | if (defined($opt{"PATCHCHECK_END[$i]"})) { | |
792 | $end = $opt{"PATCHCHECK_END[$i]"}; | |
793 | } | |
794 | ||
795 | my $type = $opt{"PATCHCHECK_TYPE[$i]"}; | |
796 | ||
797 | # Can't have a test without having a test to run | |
798 | if ($type eq "test" && !defined($run_test)) { | |
799 | $type = "boot"; | |
800 | } | |
801 | ||
802 | open (IN, "git log --pretty=oneline $end|") or | |
803 | dodie "could not get git list"; | |
804 | ||
805 | my @list; | |
806 | ||
807 | while (<IN>) { | |
808 | chomp; | |
809 | $list[$#list+1] = $_; | |
810 | last if (/^$start/); | |
811 | } | |
812 | close(IN); | |
813 | ||
814 | if ($list[$#list] !~ /^$start/) { | |
2b7d9b21 | 815 | fail "SHA1 $start not found"; |
6c5ee0be SR |
816 | } |
817 | ||
818 | # go backwards in the list | |
819 | @list = reverse @list; | |
820 | ||
821 | my $save_clean = $noclean; | |
822 | ||
823 | $in_patchcheck = 1; | |
824 | foreach my $item (@list) { | |
825 | my $sha1 = $item; | |
826 | $sha1 =~ s/^([[:xdigit:]]+).*/$1/; | |
827 | ||
828 | doprint "\nProcessing commit $item\n\n"; | |
829 | ||
830 | run_command "git checkout $sha1" or | |
831 | die "Failed to checkout $sha1"; | |
832 | ||
833 | # only clean on the first and last patch | |
834 | if ($item eq $list[0] || | |
835 | $item eq $list[$#list]) { | |
836 | $noclean = $save_clean; | |
837 | } else { | |
838 | $noclean = 1; | |
839 | } | |
840 | ||
841 | if (defined($minconfig)) { | |
2b7d9b21 | 842 | build "useconfig:$minconfig" or return 0; |
6c5ee0be SR |
843 | } else { |
844 | # ?? no config to use? | |
2b7d9b21 | 845 | build "oldconfig" or return 0; |
6c5ee0be SR |
846 | } |
847 | ||
2b7d9b21 | 848 | check_buildlog $sha1 or return 0; |
6c5ee0be SR |
849 | |
850 | next if ($type eq "build"); | |
851 | ||
852 | get_grub_index; | |
853 | get_version; | |
854 | install; | |
6c5ee0be | 855 | |
7faafbd6 SR |
856 | my $failed = 0; |
857 | ||
858 | start_monitor; | |
859 | monitor or $failed = 1; | |
860 | ||
861 | if (!$failed && $type ne "boot"){ | |
862 | do_run_test or $failed = 1; | |
863 | } | |
864 | end_monitor; | |
865 | return 0 if ($failed); | |
866 | ||
6c5ee0be SR |
867 | } |
868 | $in_patchcheck = 0; | |
869 | success $i; | |
2b7d9b21 SR |
870 | |
871 | return 1; | |
6c5ee0be SR |
872 | } |
873 | ||
2545eb61 SR |
874 | read_config $ARGV[0]; |
875 | ||
876 | # mandatory configs | |
877 | die "MACHINE not defined\n" if (!defined($opt{"MACHINE"})); | |
878 | die "SSH_USER not defined\n" if (!defined($opt{"SSH_USER"})); | |
879 | die "BUILD_DIR not defined\n" if (!defined($opt{"BUILD_DIR"})); | |
880 | die "OUTPUT_DIR not defined\n" if (!defined($opt{"OUTPUT_DIR"})); | |
881 | die "BUILD_TARGET not defined\n" if (!defined($opt{"BUILD_TARGET"})); | |
75c3fda7 | 882 | die "TARGET_IMAGE not defined\n" if (!defined($opt{"TARGET_IMAGE"})); |
2545eb61 SR |
883 | die "POWER_CYCLE not defined\n" if (!defined($opt{"POWER_CYCLE"})); |
884 | die "CONSOLE not defined\n" if (!defined($opt{"CONSOLE"})); | |
885 | die "LOCALVERSION not defined\n" if (!defined($opt{"LOCALVERSION"})); | |
886 | die "GRUB_MENU not defined\n" if (!defined($opt{"GRUB_MENU"})); | |
887 | ||
888 | chdir $opt{"BUILD_DIR"} || die "can't change directory to $opt{BUILD_DIR}"; | |
889 | ||
890 | $target = "$opt{SSH_USER}\@$opt{MACHINE}"; | |
891 | ||
2b7d9b21 SR |
892 | if ($opt{"CLEAR_LOG"} && defined($opt{"LOG_FILE"})) { |
893 | unlink $opt{"LOG_FILE"}; | |
894 | } | |
2545eb61 | 895 | |
2b7d9b21 SR |
896 | doprint "\n\nSTARTING AUTOMATED TESTS\n\n"; |
897 | ||
898 | foreach my $option (sort keys %opt) { | |
899 | doprint "$option = $opt{$option}\n"; | |
900 | } | |
2545eb61 | 901 | |
7faafbd6 SR |
902 | $buildlog = "$opt{TMP_DIR}/buildlog"; |
903 | $dmesg = "$opt{TMP_DIR}/dmesg"; | |
2545eb61 SR |
904 | $make = "$opt{MAKE_CMD} O=$opt{OUTPUT_DIR}"; |
905 | ||
5a391fbf SR |
906 | sub set_build_option { |
907 | my ($name, $i) = @_; | |
2545eb61 | 908 | |
5a391fbf | 909 | my $option = "$name\[$i\]"; |
5c42fc5b | 910 | |
5a391fbf SR |
911 | if (defined($opt{$option})) { |
912 | return $opt{$option}; | |
5f9b6ced SR |
913 | } |
914 | ||
5a391fbf SR |
915 | if (defined($opt{$name})) { |
916 | return $opt{$name}; | |
2545eb61 SR |
917 | } |
918 | ||
5a391fbf SR |
919 | return undef; |
920 | } | |
921 | ||
922 | # First we need to do is the builds | |
923 | for (my $i = 1; $i <= $opt{"NUM_BUILDS"}; $i++) { | |
1a5cfce3 | 924 | |
7faafbd6 | 925 | $build_type = set_build_option("BUILD_TYPE", $i); |
5a391fbf SR |
926 | $noclean = set_build_option("BUILD_NOCLEAN", $i); |
927 | $minconfig = set_build_option("MIN_CONFIG", $i); | |
928 | $run_test = set_build_option("TEST", $i); | |
2b7d9b21 | 929 | $addconfig = set_build_option("ADD_CONFIG", $i); |
5a391fbf | 930 | |
2545eb61 | 931 | doprint "\n\n"; |
7faafbd6 SR |
932 | doprint "RUNNING TEST $i of $opt{NUM_BUILDS} with option $build_type\n\n"; |
933 | ||
934 | unlink $dmesg; | |
935 | unlink $buildlog; | |
2545eb61 | 936 | |
2b7d9b21 SR |
937 | if (!defined($minconfig)) { |
938 | $minconfig = $addconfig; | |
939 | ||
940 | } elsif (defined($addconfig)) { | |
941 | run_command "cat $addconfig $minconfig > $opt{TMP_DIR}/use_config" or | |
942 | dodie "Failed to create temp config"; | |
943 | $minconfig = "$opt{TMP_DIR}/use_config"; | |
944 | } | |
945 | ||
6c5ee0be SR |
946 | my $checkout = $opt{"CHECKOUT[$i]"}; |
947 | if (defined($checkout)) { | |
948 | run_command "git checkout $checkout" or | |
949 | die "failed to checkout $checkout"; | |
950 | } | |
951 | ||
7faafbd6 | 952 | if ($build_type eq "bisect") { |
5f9b6ced SR |
953 | bisect $i; |
954 | next; | |
7faafbd6 | 955 | } elsif ($build_type eq "patchcheck") { |
6c5ee0be SR |
956 | patchcheck $i; |
957 | next; | |
2545eb61 | 958 | } |
2545eb61 | 959 | |
7faafbd6 SR |
960 | if ($build_type ne "nobuild") { |
961 | build $build_type or next; | |
2545eb61 SR |
962 | } |
963 | ||
5f9b6ced SR |
964 | get_grub_index; |
965 | get_version; | |
2545eb61 | 966 | install; |
5a391fbf | 967 | |
7faafbd6 SR |
968 | my $failed = 0; |
969 | start_monitor; | |
970 | monitor or $failed = 1;; | |
971 | if (!$failed && defined($run_test)) { | |
972 | do_run_test or $failed = 1; | |
5a391fbf | 973 | } |
7faafbd6 SR |
974 | end_monitor; |
975 | next if ($failed); | |
5a391fbf | 976 | |
5f9b6ced | 977 | success $i; |
2545eb61 SR |
978 | } |
979 | ||
5c42fc5b | 980 | if ($opt{"POWEROFF_ON_SUCCESS"}) { |
75c3fda7 | 981 | halt; |
1a5cfce3 | 982 | } elsif ($opt{"REBOOT_ON_SUCCESS"}) { |
75c3fda7 | 983 | reboot; |
5c42fc5b | 984 | } |
75c3fda7 | 985 | |
2545eb61 | 986 | exit 0; |