Commit | Line | Data |
---|---|---|
5a699617 RO |
1 | #!/usr/bin/python |
2 | # | |
3 | # Copyright (C) 2014 Free Software Foundation, Inc. | |
4 | # | |
5 | # This script is free software; you can redistribute it and/or modify | |
6 | # it under the terms of the GNU General Public License as published by | |
7 | # the Free Software Foundation; either version 3, or (at your option) | |
8 | # any later version. | |
9 | ||
10 | import sys | |
11 | import getopt | |
12 | import re | |
13 | import io | |
14 | from datetime import datetime | |
15 | from operator import attrgetter | |
16 | ||
17 | # True if unrecognised lines should cause a fatal error. Might want to turn | |
18 | # this on by default later. | |
19 | strict = False | |
20 | ||
21 | # True if the order of .log segments should match the .sum file, false if | |
22 | # they should keep the original order. | |
23 | sort_logs = True | |
24 | ||
25 | # A version of open() that is safe against whatever binary output | |
26 | # might be added to the log. | |
27 | def safe_open (filename): | |
28 | if sys.version_info >= (3, 0): | |
29 | return open (filename, 'r', errors = 'surrogateescape') | |
30 | return open (filename, 'r') | |
31 | ||
32 | # Force stdout to handle escape sequences from a safe_open file. | |
33 | if sys.version_info >= (3, 0): | |
34 | sys.stdout = io.TextIOWrapper (sys.stdout.buffer, | |
35 | errors = 'surrogateescape') | |
36 | ||
37 | class Named: | |
38 | def __init__ (self, name): | |
39 | self.name = name | |
40 | ||
41 | class ToolRun (Named): | |
42 | def __init__ (self, name): | |
43 | Named.__init__ (self, name) | |
44 | # The variations run for this tool, mapped by --target_board name. | |
45 | self.variations = dict() | |
46 | ||
47 | # Return the VariationRun for variation NAME. | |
48 | def get_variation (self, name): | |
49 | if name not in self.variations: | |
50 | self.variations[name] = VariationRun (name) | |
51 | return self.variations[name] | |
52 | ||
53 | class VariationRun (Named): | |
54 | def __init__ (self, name): | |
55 | Named.__init__ (self, name) | |
56 | # A segment of text before the harness runs start, describing which | |
57 | # baseboard files were loaded for the target. | |
58 | self.header = None | |
59 | # The harnesses run for this variation, mapped by filename. | |
60 | self.harnesses = dict() | |
61 | # A list giving the number of times each type of result has | |
62 | # been seen. | |
63 | self.counts = [] | |
64 | ||
65 | # Return the HarnessRun for harness NAME. | |
66 | def get_harness (self, name): | |
67 | if name not in self.harnesses: | |
68 | self.harnesses[name] = HarnessRun (name) | |
69 | return self.harnesses[name] | |
70 | ||
71 | class HarnessRun (Named): | |
72 | def __init__ (self, name): | |
73 | Named.__init__ (self, name) | |
74 | # Segments of text that make up the harness run, mapped by a test-based | |
75 | # key that can be used to order them. | |
76 | self.segments = dict() | |
77 | # Segments of text that make up the harness run but which have | |
78 | # no recognized test results. These are typically harnesses that | |
79 | # are completely skipped for the target. | |
80 | self.empty = [] | |
81 | # A list of results. Each entry is a pair in which the first element | |
82 | # is a unique sorting key and in which the second is the full | |
83 | # PASS/FAIL line. | |
84 | self.results = [] | |
85 | ||
86 | # Add a segment of text to the harness run. If the segment includes | |
87 | # test results, KEY is an example of one of them, and can be used to | |
88 | # combine the individual segments in order. If the segment has no | |
89 | # test results (e.g. because the harness doesn't do anything for the | |
90 | # current configuration) then KEY is None instead. In that case | |
91 | # just collect the segments in the order that we see them. | |
92 | def add_segment (self, key, segment): | |
93 | if key: | |
94 | assert key not in self.segments | |
95 | self.segments[key] = segment | |
96 | else: | |
97 | self.empty.append (segment) | |
98 | ||
99 | class Segment: | |
100 | def __init__ (self, filename, start): | |
101 | self.filename = filename | |
102 | self.start = start | |
103 | self.lines = 0 | |
104 | ||
105 | class Prog: | |
106 | def __init__ (self): | |
107 | # The variations specified on the command line. | |
108 | self.variations = [] | |
109 | # The variations seen in the input files. | |
110 | self.known_variations = set() | |
111 | # The tools specified on the command line. | |
112 | self.tools = [] | |
113 | # Whether to create .sum rather than .log output. | |
114 | self.do_sum = True | |
115 | # Regexps used while parsing. | |
116 | self.test_run_re = re.compile (r'^Test Run By (\S+) on (.*)$') | |
117 | self.tool_re = re.compile (r'^\t\t=== (.*) tests ===$') | |
118 | self.result_re = re.compile (r'^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED' | |
119 | r'|WARNING|ERROR|UNSUPPORTED|UNTESTED' | |
120 | r'|KFAIL):\s*(.+)') | |
121 | self.completed_re = re.compile (r'.* completed at (.*)') | |
122 | # Pieces of text to write at the head of the output. | |
123 | # start_line is a pair in which the first element is a datetime | |
124 | # and in which the second is the associated 'Test Run By' line. | |
125 | self.start_line = None | |
126 | self.native_line = '' | |
127 | self.target_line = '' | |
128 | self.host_line = '' | |
129 | self.acats_premable = '' | |
130 | # Pieces of text to write at the end of the output. | |
131 | # end_line is like start_line but for the 'runtest completed' line. | |
132 | self.acats_failures = [] | |
133 | self.version_output = '' | |
134 | self.end_line = None | |
135 | # Known summary types. | |
136 | self.count_names = [ | |
137 | '# of DejaGnu errors\t\t', | |
138 | '# of expected passes\t\t', | |
139 | '# of unexpected failures\t', | |
140 | '# of unexpected successes\t', | |
141 | '# of expected failures\t\t', | |
142 | '# of unknown successes\t\t', | |
143 | '# of known failures\t\t', | |
144 | '# of untested testcases\t\t', | |
145 | '# of unresolved testcases\t', | |
146 | '# of unsupported tests\t\t' | |
147 | ] | |
148 | self.runs = dict() | |
149 | ||
150 | def usage (self): | |
151 | name = sys.argv[0] | |
152 | sys.stderr.write ('Usage: ' + name | |
153 | + ''' [-t tool] [-l variant-list] [-L] log-or-sum-file ... | |
154 | ||
155 | tool The tool (e.g. g++, libffi) for which to create a | |
156 | new test summary file. If not specified then output | |
157 | is created for all tools. | |
158 | variant-list One or more test variant names. If the list is | |
159 | not specified then one is constructed from all | |
160 | variants in the files for <tool>. | |
161 | sum-file A test summary file with the format of those | |
162 | created by runtest from DejaGnu. | |
163 | If -L is used, merge *.log files instead of *.sum. In this | |
164 | mode the exact order of lines may not be preserved, just different | |
165 | Running *.exp chunks should be in correct order. | |
166 | ''') | |
167 | sys.exit (1) | |
168 | ||
169 | def fatal (self, what, string): | |
170 | if not what: | |
171 | what = sys.argv[0] | |
172 | sys.stderr.write (what + ': ' + string + '\n') | |
173 | sys.exit (1) | |
174 | ||
175 | # Parse the command-line arguments. | |
176 | def parse_cmdline (self): | |
177 | try: | |
178 | (options, self.files) = getopt.getopt (sys.argv[1:], 'l:t:L') | |
179 | if len (self.files) == 0: | |
180 | self.usage() | |
181 | for (option, value) in options: | |
182 | if option == '-l': | |
183 | self.variations.append (value) | |
184 | elif option == '-t': | |
185 | self.tools.append (value) | |
186 | else: | |
187 | self.do_sum = False | |
188 | except getopt.GetoptError as e: | |
189 | self.fatal (None, e.msg) | |
190 | ||
191 | # Try to parse time string TIME, returning an arbitrary time on failure. | |
192 | # Getting this right is just a nice-to-have so failures should be silent. | |
193 | def parse_time (self, time): | |
194 | try: | |
195 | return datetime.strptime (time, '%c') | |
196 | except ValueError: | |
197 | return datetime.now() | |
198 | ||
199 | # Parse an integer and abort on failure. | |
200 | def parse_int (self, filename, value): | |
201 | try: | |
202 | return int (value) | |
203 | except ValueError: | |
204 | self.fatal (filename, 'expected an integer, got: ' + value) | |
205 | ||
206 | # Return a list that represents no test results. | |
207 | def zero_counts (self): | |
208 | return [0 for x in self.count_names] | |
209 | ||
210 | # Return the ToolRun for tool NAME. | |
211 | def get_tool (self, name): | |
212 | if name not in self.runs: | |
213 | self.runs[name] = ToolRun (name) | |
214 | return self.runs[name] | |
215 | ||
216 | # Add the result counts in list FROMC to TOC. | |
217 | def accumulate_counts (self, toc, fromc): | |
218 | for i in range (len (self.count_names)): | |
219 | toc[i] += fromc[i] | |
220 | ||
221 | # Parse the list of variations after 'Schedule of variations:'. | |
222 | # Return the number seen. | |
223 | def parse_variations (self, filename, file): | |
224 | num_variations = 0 | |
225 | while True: | |
226 | line = file.readline() | |
227 | if line == '': | |
228 | self.fatal (filename, 'could not parse variation list') | |
229 | if line == '\n': | |
230 | break | |
231 | self.known_variations.add (line.strip()) | |
232 | num_variations += 1 | |
233 | return num_variations | |
234 | ||
235 | # Parse from the first line after 'Running target ...' to the end | |
236 | # of the run's summary. | |
237 | def parse_run (self, filename, file, tool, variation, num_variations): | |
238 | header = None | |
239 | harness = None | |
240 | segment = None | |
241 | final_using = 0 | |
242 | ||
243 | # If this is the first run for this variation, add any text before | |
244 | # the first harness to the header. | |
245 | if not variation.header: | |
246 | segment = Segment (filename, file.tell()) | |
247 | variation.header = segment | |
248 | ||
249 | # Parse the rest of the summary (the '# of ' lines). | |
250 | if len (variation.counts) == 0: | |
251 | variation.counts = self.zero_counts() | |
252 | ||
253 | # Parse up until the first line of the summary. | |
254 | if num_variations == 1: | |
255 | end = '\t\t=== ' + tool.name + ' Summary ===\n' | |
256 | else: | |
257 | end = ('\t\t=== ' + tool.name + ' Summary for ' | |
258 | + variation.name + ' ===\n') | |
259 | while True: | |
260 | line = file.readline() | |
261 | if line == '': | |
262 | self.fatal (filename, 'no recognised summary line') | |
263 | if line == end: | |
264 | break | |
265 | ||
266 | # Look for the start of a new harness. | |
267 | if line.startswith ('Running ') and line.endswith (' ...\n'): | |
268 | # Close off the current harness segment, if any. | |
269 | if harness: | |
270 | segment.lines -= final_using | |
271 | harness.add_segment (first_key, segment) | |
272 | name = line[len ('Running '):-len(' ...\n')] | |
273 | harness = variation.get_harness (name) | |
274 | segment = Segment (filename, file.tell()) | |
275 | first_key = None | |
276 | final_using = 0 | |
277 | continue | |
278 | ||
279 | # Record test results. Associate the first test result with | |
280 | # the harness segment, so that if a run for a particular harness | |
281 | # has been split up, we can reassemble the individual segments | |
282 | # in a sensible order. | |
283 | # | |
284 | # dejagnu sometimes issues warnings about the testing environment | |
285 | # before running any tests. Treat them as part of the header | |
286 | # rather than as a test result. | |
287 | match = self.result_re.match (line) | |
288 | if match and (harness or not line.startswith ('WARNING:')): | |
289 | if not harness: | |
290 | self.fatal (filename, 'saw test result before harness name') | |
291 | name = match.group (2) | |
292 | # Ugly hack to get the right order for gfortran. | |
293 | if name.startswith ('gfortran.dg/g77/'): | |
294 | name = 'h' + name | |
295 | key = (name, len (harness.results)) | |
296 | harness.results.append ((key, line)) | |
297 | if not first_key and sort_logs: | |
298 | first_key = key | |
299 | if line.startswith ('ERROR: (DejaGnu)'): | |
300 | for i in range (len (self.count_names)): | |
301 | if 'DejaGnu errors' in self.count_names[i]: | |
302 | variation.counts[i] += 1 | |
303 | break | |
304 | ||
305 | # 'Using ...' lines are only interesting in a header. Splitting | |
306 | # the test up into parallel runs leads to more 'Using ...' lines | |
307 | # than there would be in a single log. | |
308 | if line.startswith ('Using '): | |
309 | final_using += 1 | |
310 | else: | |
311 | final_using = 0 | |
312 | ||
313 | # Add other text to the current segment, if any. | |
314 | if segment: | |
315 | segment.lines += 1 | |
316 | ||
317 | # Close off the final harness segment, if any. | |
318 | if harness: | |
319 | segment.lines -= final_using | |
320 | harness.add_segment (first_key, segment) | |
321 | ||
322 | while True: | |
323 | before = file.tell() | |
324 | line = file.readline() | |
325 | if line == '': | |
326 | break | |
327 | if line == '\n': | |
328 | continue | |
329 | if not line.startswith ('# '): | |
330 | file.seek (before) | |
331 | break | |
332 | found = False | |
333 | for i in range (len (self.count_names)): | |
334 | if line.startswith (self.count_names[i]): | |
335 | count = line[len (self.count_names[i]):-1].strip() | |
336 | variation.counts[i] += self.parse_int (filename, count) | |
337 | found = True | |
338 | break | |
339 | if not found: | |
340 | self.fatal (filename, 'unknown test result: ' + line[:-1]) | |
341 | ||
342 | # Parse an acats run, which uses a different format from dejagnu. | |
343 | # We have just skipped over '=== acats configuration ==='. | |
344 | def parse_acats_run (self, filename, file): | |
345 | # Parse the preamble, which describes the configuration and logs | |
346 | # the creation of support files. | |
347 | record = (self.acats_premable == '') | |
348 | if record: | |
349 | self.acats_premable = '\t\t=== acats configuration ===\n' | |
350 | while True: | |
351 | line = file.readline() | |
352 | if line == '': | |
353 | self.fatal (filename, 'could not parse acats preamble') | |
354 | if line == '\t\t=== acats tests ===\n': | |
355 | break | |
356 | if record: | |
357 | self.acats_premable += line | |
358 | ||
359 | # Parse the test results themselves, using a dummy variation name. | |
360 | tool = self.get_tool ('acats') | |
361 | variation = tool.get_variation ('none') | |
362 | self.parse_run (filename, file, tool, variation, 1) | |
363 | ||
364 | # Parse the failure list. | |
365 | while True: | |
366 | before = file.tell() | |
367 | line = file.readline() | |
368 | if line.startswith ('*** FAILURES: '): | |
369 | self.acats_failures.append (line[len ('*** FAILURES: '):-1]) | |
370 | continue | |
371 | file.seek (before) | |
372 | break | |
373 | ||
374 | # Parse the final summary at the end of a log in order to capture | |
375 | # the version output that follows it. | |
376 | def parse_final_summary (self, filename, file): | |
377 | record = (self.version_output == '') | |
378 | while True: | |
379 | line = file.readline() | |
380 | if line == '': | |
381 | break | |
382 | if line.startswith ('# of '): | |
383 | continue | |
384 | if record: | |
385 | self.version_output += line | |
386 | if line == '\n': | |
387 | break | |
388 | ||
389 | # Parse a .log or .sum file. | |
390 | def parse_file (self, filename, file): | |
391 | tool = None | |
392 | target = None | |
393 | num_variations = 1 | |
394 | while True: | |
395 | line = file.readline() | |
396 | if line == '': | |
397 | return | |
398 | ||
399 | # Parse the list of variations, which comes before the test | |
400 | # runs themselves. | |
401 | if line.startswith ('Schedule of variations:'): | |
402 | num_variations = self.parse_variations (filename, file) | |
403 | continue | |
404 | ||
405 | # Parse a testsuite run for one tool/variation combination. | |
406 | if line.startswith ('Running target '): | |
407 | name = line[len ('Running target '):-1] | |
408 | if not tool: | |
409 | self.fatal (filename, 'could not parse tool name') | |
410 | if name not in self.known_variations: | |
411 | self.fatal (filename, 'unknown target: ' + name) | |
412 | self.parse_run (filename, file, tool, | |
413 | tool.get_variation (name), | |
414 | num_variations) | |
415 | # If there is only one variation then there is no separate | |
416 | # summary for it. Record any following version output. | |
417 | if num_variations == 1: | |
418 | self.parse_final_summary (filename, file) | |
419 | continue | |
420 | ||
421 | # Parse the start line. In the case where several files are being | |
422 | # parsed, pick the one with the earliest time. | |
423 | match = self.test_run_re.match (line) | |
424 | if match: | |
425 | time = self.parse_time (match.group (2)) | |
426 | if not self.start_line or self.start_line[0] > time: | |
427 | self.start_line = (time, line) | |
428 | continue | |
429 | ||
430 | # Parse the form used for native testing. | |
431 | if line.startswith ('Native configuration is '): | |
432 | self.native_line = line | |
433 | continue | |
434 | ||
435 | # Parse the target triplet. | |
436 | if line.startswith ('Target is '): | |
437 | self.target_line = line | |
438 | continue | |
439 | ||
440 | # Parse the host triplet. | |
441 | if line.startswith ('Host is '): | |
442 | self.host_line = line | |
443 | continue | |
444 | ||
445 | # Parse the acats premable. | |
446 | if line == '\t\t=== acats configuration ===\n': | |
447 | self.parse_acats_run (filename, file) | |
448 | continue | |
449 | ||
450 | # Parse the tool name. | |
451 | match = self.tool_re.match (line) | |
452 | if match: | |
453 | tool = self.get_tool (match.group (1)) | |
454 | continue | |
455 | ||
456 | # Skip over the final summary (which we instead create from | |
457 | # individual runs) and parse the version output. | |
458 | if tool and line == '\t\t=== ' + tool.name + ' Summary ===\n': | |
459 | if file.readline() != '\n': | |
460 | self.fatal (filename, 'expected blank line after summary') | |
461 | self.parse_final_summary (filename, file) | |
462 | continue | |
463 | ||
464 | # Parse the completion line. In the case where several files | |
465 | # are being parsed, pick the one with the latest time. | |
466 | match = self.completed_re.match (line) | |
467 | if match: | |
468 | time = self.parse_time (match.group (1)) | |
469 | if not self.end_line or self.end_line[0] < time: | |
470 | self.end_line = (time, line) | |
471 | continue | |
472 | ||
473 | # Sanity check to make sure that important text doesn't get | |
474 | # dropped accidentally. | |
475 | if strict and line.strip() != '': | |
476 | self.fatal (filename, 'unrecognised line: ' + line[:-1]) | |
477 | ||
478 | # Output a segment of text. | |
479 | def output_segment (self, segment): | |
480 | with safe_open (segment.filename) as file: | |
481 | file.seek (segment.start) | |
482 | for i in range (segment.lines): | |
483 | sys.stdout.write (file.readline()) | |
484 | ||
485 | # Output a summary giving the number of times each type of result has | |
486 | # been seen. | |
487 | def output_summary (self, tool, counts): | |
488 | for i in range (len (self.count_names)): | |
489 | name = self.count_names[i] | |
490 | # dejagnu only prints result types that were seen at least once, | |
491 | # but acats always prints a number of unexpected failures. | |
492 | if (counts[i] > 0 | |
493 | or (tool.name == 'acats' | |
494 | and name.startswith ('# of unexpected failures'))): | |
495 | sys.stdout.write ('%s%d\n' % (name, counts[i])) | |
496 | ||
497 | # Output unified .log or .sum information for a particular variation, | |
498 | # with a summary at the end. | |
499 | def output_variation (self, tool, variation): | |
500 | self.output_segment (variation.header) | |
501 | for harness in sorted (variation.harnesses.values(), | |
502 | key = attrgetter ('name')): | |
503 | sys.stdout.write ('Running ' + harness.name + ' ...\n') | |
504 | if self.do_sum: | |
505 | harness.results.sort() | |
506 | for (key, line) in harness.results: | |
507 | sys.stdout.write (line) | |
508 | else: | |
509 | # Rearrange the log segments into test order (but without | |
510 | # rearranging text within those segments). | |
511 | for key in sorted (harness.segments.keys()): | |
512 | self.output_segment (harness.segments[key]) | |
513 | for segment in harness.empty: | |
514 | self.output_segment (segment) | |
515 | if len (self.variations) > 1: | |
516 | sys.stdout.write ('\t\t=== ' + tool.name + ' Summary for ' | |
517 | + variation.name + ' ===\n\n') | |
518 | self.output_summary (tool, variation.counts) | |
519 | ||
520 | # Output unified .log or .sum information for a particular tool, | |
521 | # with a summary at the end. | |
522 | def output_tool (self, tool): | |
523 | counts = self.zero_counts() | |
524 | if tool.name == 'acats': | |
525 | # acats doesn't use variations, so just output everything. | |
526 | # It also has a different approach to whitespace. | |
527 | sys.stdout.write ('\t\t=== ' + tool.name + ' tests ===\n') | |
528 | for variation in tool.variations.values(): | |
529 | self.output_variation (tool, variation) | |
530 | self.accumulate_counts (counts, variation.counts) | |
531 | sys.stdout.write ('\t\t=== ' + tool.name + ' Summary ===\n') | |
532 | else: | |
533 | # Output the results in the usual dejagnu runtest format. | |
534 | sys.stdout.write ('\n\t\t=== ' + tool.name + ' tests ===\n\n' | |
535 | 'Schedule of variations:\n') | |
536 | for name in self.variations: | |
537 | if name in tool.variations: | |
538 | sys.stdout.write (' ' + name + '\n') | |
539 | sys.stdout.write ('\n') | |
540 | for name in self.variations: | |
541 | if name in tool.variations: | |
542 | variation = tool.variations[name] | |
543 | sys.stdout.write ('Running target ' | |
544 | + variation.name + '\n') | |
545 | self.output_variation (tool, variation) | |
546 | self.accumulate_counts (counts, variation.counts) | |
547 | sys.stdout.write ('\n\t\t=== ' + tool.name + ' Summary ===\n\n') | |
548 | self.output_summary (tool, counts) | |
549 | ||
550 | def main (self): | |
551 | self.parse_cmdline() | |
552 | try: | |
553 | # Parse the input files. | |
554 | for filename in self.files: | |
555 | with safe_open (filename) as file: | |
556 | self.parse_file (filename, file) | |
557 | ||
558 | # Decide what to output. | |
559 | if len (self.variations) == 0: | |
560 | self.variations = sorted (self.known_variations) | |
561 | else: | |
562 | for name in self.variations: | |
563 | if name not in self.known_variations: | |
564 | self.fatal (None, 'no results for ' + name) | |
565 | if len (self.tools) == 0: | |
566 | self.tools = sorted (self.runs.keys()) | |
567 | ||
568 | # Output the header. | |
569 | if self.start_line: | |
570 | sys.stdout.write (self.start_line[1]) | |
571 | sys.stdout.write (self.native_line) | |
572 | sys.stdout.write (self.target_line) | |
573 | sys.stdout.write (self.host_line) | |
574 | sys.stdout.write (self.acats_premable) | |
575 | ||
576 | # Output the main body. | |
577 | for name in self.tools: | |
578 | if name not in self.runs: | |
579 | self.fatal (None, 'no results for ' + name) | |
580 | self.output_tool (self.runs[name]) | |
581 | ||
582 | # Output the footer. | |
583 | if len (self.acats_failures) > 0: | |
584 | sys.stdout.write ('*** FAILURES: ' | |
585 | + ' '.join (self.acats_failures) + '\n') | |
586 | sys.stdout.write (self.version_output) | |
587 | if self.end_line: | |
588 | sys.stdout.write (self.end_line[1]) | |
589 | except IOError as e: | |
590 | self.fatal (e.filename, e.strerror) | |
591 | ||
592 | Prog().main() |