3 # Copyright (C) 2014 Free Software Foundation, Inc.
5 # This script is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 3, or (at your option)
14 from datetime
import datetime
15 from operator
import attrgetter
17 # True if unrecognised lines should cause a fatal error. Might want to turn
18 # this on by default later.
21 # True if the order of .log segments should match the .sum file, false if
22 # they should keep the original order.
25 # A version of open() that is safe against whatever binary output
26 # might be added to the log.
27 def safe_open (filename
):
28 if sys
.version_info
>= (3, 0):
29 return open (filename
, 'r', errors
= 'surrogateescape')
30 return open (filename
, 'r')
32 # Force stdout to handle escape sequences from a safe_open file.
33 if sys
.version_info
>= (3, 0):
34 sys
.stdout
= io
.TextIOWrapper (sys
.stdout
.buffer,
35 errors
= 'surrogateescape')
38 def __init__ (self
, name
):
41 class ToolRun (Named
):
42 def __init__ (self
, name
):
43 Named
.__init
__ (self
, name
)
44 # The variations run for this tool, mapped by --target_board name.
45 self
.variations
= dict()
47 # Return the VariationRun for variation NAME.
48 def get_variation (self
, name
):
49 if name
not in self
.variations
:
50 self
.variations
[name
] = VariationRun (name
)
51 return self
.variations
[name
]
53 class VariationRun (Named
):
54 def __init__ (self
, name
):
55 Named
.__init
__ (self
, name
)
56 # A segment of text before the harness runs start, describing which
57 # baseboard files were loaded for the target.
59 # The harnesses run for this variation, mapped by filename.
60 self
.harnesses
= dict()
61 # A list giving the number of times each type of result has
65 # Return the HarnessRun for harness NAME.
66 def get_harness (self
, name
):
67 if name
not in self
.harnesses
:
68 self
.harnesses
[name
] = HarnessRun (name
)
69 return self
.harnesses
[name
]
71 class HarnessRun (Named
):
72 def __init__ (self
, name
):
73 Named
.__init
__ (self
, name
)
74 # Segments of text that make up the harness run, mapped by a test-based
75 # key that can be used to order them.
76 self
.segments
= dict()
77 # Segments of text that make up the harness run but which have
78 # no recognized test results. These are typically harnesses that
79 # are completely skipped for the target.
81 # A list of results. Each entry is a pair in which the first element
82 # is a unique sorting key and in which the second is the full
86 # Add a segment of text to the harness run. If the segment includes
87 # test results, KEY is an example of one of them, and can be used to
88 # combine the individual segments in order. If the segment has no
89 # test results (e.g. because the harness doesn't do anything for the
90 # current configuration) then KEY is None instead. In that case
91 # just collect the segments in the order that we see them.
92 def add_segment (self
, key
, segment
):
94 assert key
not in self
.segments
95 self
.segments
[key
] = segment
97 self
.empty
.append (segment
)
100 def __init__ (self
, filename
, start
):
101 self
.filename
= filename
107 # The variations specified on the command line.
109 # The variations seen in the input files.
110 self
.known_variations
= set()
111 # The tools specified on the command line.
113 # Whether to create .sum rather than .log output.
115 # Regexps used while parsing.
116 self
.test_run_re
= re
.compile (r
'^Test Run By (\S+) on (.*)$')
117 self
.tool_re
= re
.compile (r
'^\t\t=== (.*) tests ===$')
118 self
.result_re
= re
.compile (r
'^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED'
119 r
'|WARNING|ERROR|UNSUPPORTED|UNTESTED'
121 self
.completed_re
= re
.compile (r
'.* completed at (.*)')
122 # Pieces of text to write at the head of the output.
123 # start_line is a pair in which the first element is a datetime
124 # and in which the second is the associated 'Test Run By' line.
125 self
.start_line
= None
126 self
.native_line
= ''
127 self
.target_line
= ''
129 self
.acats_premable
= ''
130 # Pieces of text to write at the end of the output.
131 # end_line is like start_line but for the 'runtest completed' line.
132 self
.acats_failures
= []
133 self
.version_output
= ''
135 # Known summary types.
137 '# of DejaGnu errors\t\t',
138 '# of expected passes\t\t',
139 '# of unexpected failures\t',
140 '# of unexpected successes\t',
141 '# of expected failures\t\t',
142 '# of unknown successes\t\t',
143 '# of known failures\t\t',
144 '# of untested testcases\t\t',
145 '# of unresolved testcases\t',
146 '# of unsupported tests\t\t'
152 sys
.stderr
.write ('Usage: ' + name
153 + ''' [-t tool] [-l variant-list] [-L] log-or-sum-file ...
155 tool The tool (e.g. g++, libffi) for which to create a
156 new test summary file. If not specified then output
157 is created for all tools.
158 variant-list One or more test variant names. If the list is
159 not specified then one is constructed from all
160 variants in the files for <tool>.
161 sum-file A test summary file with the format of those
162 created by runtest from DejaGnu.
163 If -L is used, merge *.log files instead of *.sum. In this
164 mode the exact order of lines may not be preserved, just different
165 Running *.exp chunks should be in correct order.
169 def fatal (self
, what
, string
):
172 sys
.stderr
.write (what
+ ': ' + string
+ '\n')
175 # Parse the command-line arguments.
176 def parse_cmdline (self
):
178 (options
, self
.files
) = getopt
.getopt (sys
.argv
[1:], 'l:t:L')
179 if len (self
.files
) == 0:
181 for (option
, value
) in options
:
183 self
.variations
.append (value
)
185 self
.tools
.append (value
)
188 except getopt
.GetoptError
as e
:
189 self
.fatal (None, e
.msg
)
191 # Try to parse time string TIME, returning an arbitrary time on failure.
192 # Getting this right is just a nice-to-have so failures should be silent.
193 def parse_time (self
, time
):
195 return datetime
.strptime (time
, '%c')
197 return datetime
.now()
199 # Parse an integer and abort on failure.
200 def parse_int (self
, filename
, value
):
204 self
.fatal (filename
, 'expected an integer, got: ' + value
)
206 # Return a list that represents no test results.
207 def zero_counts (self
):
208 return [0 for x
in self
.count_names
]
210 # Return the ToolRun for tool NAME.
211 def get_tool (self
, name
):
212 if name
not in self
.runs
:
213 self
.runs
[name
] = ToolRun (name
)
214 return self
.runs
[name
]
216 # Add the result counts in list FROMC to TOC.
217 def accumulate_counts (self
, toc
, fromc
):
218 for i
in range (len (self
.count_names
)):
221 # Parse the list of variations after 'Schedule of variations:'.
222 # Return the number seen.
223 def parse_variations (self
, filename
, file):
226 line
= file.readline()
228 self
.fatal (filename
, 'could not parse variation list')
231 self
.known_variations
.add (line
.strip())
233 return num_variations
235 # Parse from the first line after 'Running target ...' to the end
236 # of the run's summary.
237 def parse_run (self
, filename
, file, tool
, variation
, num_variations
):
243 # If this is the first run for this variation, add any text before
244 # the first harness to the header.
245 if not variation
.header
:
246 segment
= Segment (filename
, file.tell())
247 variation
.header
= segment
249 # Parse the rest of the summary (the '# of ' lines).
250 if len (variation
.counts
) == 0:
251 variation
.counts
= self
.zero_counts()
253 # Parse up until the first line of the summary.
254 if num_variations
== 1:
255 end
= '\t\t=== ' + tool
.name
+ ' Summary ===\n'
257 end
= ('\t\t=== ' + tool
.name
+ ' Summary for '
258 + variation
.name
+ ' ===\n')
260 line
= file.readline()
262 self
.fatal (filename
, 'no recognised summary line')
266 # Look for the start of a new harness.
267 if line
.startswith ('Running ') and line
.endswith (' ...\n'):
268 # Close off the current harness segment, if any.
270 segment
.lines
-= final_using
271 harness
.add_segment (first_key
, segment
)
272 name
= line
[len ('Running '):-len(' ...\n')]
273 harness
= variation
.get_harness (name
)
274 segment
= Segment (filename
, file.tell())
279 # Record test results. Associate the first test result with
280 # the harness segment, so that if a run for a particular harness
281 # has been split up, we can reassemble the individual segments
282 # in a sensible order.
284 # dejagnu sometimes issues warnings about the testing environment
285 # before running any tests. Treat them as part of the header
286 # rather than as a test result.
287 match
= self
.result_re
.match (line
)
288 if match
and (harness
or not line
.startswith ('WARNING:')):
290 self
.fatal (filename
, 'saw test result before harness name')
291 name
= match
.group (2)
292 # Ugly hack to get the right order for gfortran.
293 if name
.startswith ('gfortran.dg/g77/'):
295 key
= (name
, len (harness
.results
))
296 harness
.results
.append ((key
, line
))
297 if not first_key
and sort_logs
:
299 if line
.startswith ('ERROR: (DejaGnu)'):
300 for i
in range (len (self
.count_names
)):
301 if 'DejaGnu errors' in self
.count_names
[i
]:
302 variation
.counts
[i
] += 1
305 # 'Using ...' lines are only interesting in a header. Splitting
306 # the test up into parallel runs leads to more 'Using ...' lines
307 # than there would be in a single log.
308 if line
.startswith ('Using '):
313 # Add other text to the current segment, if any.
317 # Close off the final harness segment, if any.
319 segment
.lines
-= final_using
320 harness
.add_segment (first_key
, segment
)
324 line
= file.readline()
329 if not line
.startswith ('# '):
333 for i
in range (len (self
.count_names
)):
334 if line
.startswith (self
.count_names
[i
]):
335 count
= line
[len (self
.count_names
[i
]):-1].strip()
336 variation
.counts
[i
] += self
.parse_int (filename
, count
)
340 self
.fatal (filename
, 'unknown test result: ' + line
[:-1])
342 # Parse an acats run, which uses a different format from dejagnu.
343 # We have just skipped over '=== acats configuration ==='.
344 def parse_acats_run (self
, filename
, file):
345 # Parse the preamble, which describes the configuration and logs
346 # the creation of support files.
347 record
= (self
.acats_premable
== '')
349 self
.acats_premable
= '\t\t=== acats configuration ===\n'
351 line
= file.readline()
353 self
.fatal (filename
, 'could not parse acats preamble')
354 if line
== '\t\t=== acats tests ===\n':
357 self
.acats_premable
+= line
359 # Parse the test results themselves, using a dummy variation name.
360 tool
= self
.get_tool ('acats')
361 variation
= tool
.get_variation ('none')
362 self
.parse_run (filename
, file, tool
, variation
, 1)
364 # Parse the failure list.
367 line
= file.readline()
368 if line
.startswith ('*** FAILURES: '):
369 self
.acats_failures
.append (line
[len ('*** FAILURES: '):-1])
374 # Parse the final summary at the end of a log in order to capture
375 # the version output that follows it.
376 def parse_final_summary (self
, filename
, file):
377 record
= (self
.version_output
== '')
379 line
= file.readline()
382 if line
.startswith ('# of '):
385 self
.version_output
+= line
389 # Parse a .log or .sum file.
390 def parse_file (self
, filename
, file):
395 line
= file.readline()
399 # Parse the list of variations, which comes before the test
401 if line
.startswith ('Schedule of variations:'):
402 num_variations
= self
.parse_variations (filename
, file)
405 # Parse a testsuite run for one tool/variation combination.
406 if line
.startswith ('Running target '):
407 name
= line
[len ('Running target '):-1]
409 self
.fatal (filename
, 'could not parse tool name')
410 if name
not in self
.known_variations
:
411 self
.fatal (filename
, 'unknown target: ' + name
)
412 self
.parse_run (filename
, file, tool
,
413 tool
.get_variation (name
),
415 # If there is only one variation then there is no separate
416 # summary for it. Record any following version output.
417 if num_variations
== 1:
418 self
.parse_final_summary (filename
, file)
421 # Parse the start line. In the case where several files are being
422 # parsed, pick the one with the earliest time.
423 match
= self
.test_run_re
.match (line
)
425 time
= self
.parse_time (match
.group (2))
426 if not self
.start_line
or self
.start_line
[0] > time
:
427 self
.start_line
= (time
, line
)
430 # Parse the form used for native testing.
431 if line
.startswith ('Native configuration is '):
432 self
.native_line
= line
435 # Parse the target triplet.
436 if line
.startswith ('Target is '):
437 self
.target_line
= line
440 # Parse the host triplet.
441 if line
.startswith ('Host is '):
442 self
.host_line
= line
445 # Parse the acats premable.
446 if line
== '\t\t=== acats configuration ===\n':
447 self
.parse_acats_run (filename
, file)
450 # Parse the tool name.
451 match
= self
.tool_re
.match (line
)
453 tool
= self
.get_tool (match
.group (1))
456 # Skip over the final summary (which we instead create from
457 # individual runs) and parse the version output.
458 if tool
and line
== '\t\t=== ' + tool
.name
+ ' Summary ===\n':
459 if file.readline() != '\n':
460 self
.fatal (filename
, 'expected blank line after summary')
461 self
.parse_final_summary (filename
, file)
464 # Parse the completion line. In the case where several files
465 # are being parsed, pick the one with the latest time.
466 match
= self
.completed_re
.match (line
)
468 time
= self
.parse_time (match
.group (1))
469 if not self
.end_line
or self
.end_line
[0] < time
:
470 self
.end_line
= (time
, line
)
473 # Sanity check to make sure that important text doesn't get
474 # dropped accidentally.
475 if strict
and line
.strip() != '':
476 self
.fatal (filename
, 'unrecognised line: ' + line
[:-1])
478 # Output a segment of text.
479 def output_segment (self
, segment
):
480 with
safe_open (segment
.filename
) as file:
481 file.seek (segment
.start
)
482 for i
in range (segment
.lines
):
483 sys
.stdout
.write (file.readline())
485 # Output a summary giving the number of times each type of result has
487 def output_summary (self
, tool
, counts
):
488 for i
in range (len (self
.count_names
)):
489 name
= self
.count_names
[i
]
490 # dejagnu only prints result types that were seen at least once,
491 # but acats always prints a number of unexpected failures.
493 or (tool
.name
== 'acats'
494 and name
.startswith ('# of unexpected failures'))):
495 sys
.stdout
.write ('%s%d\n' % (name
, counts
[i
]))
497 # Output unified .log or .sum information for a particular variation,
498 # with a summary at the end.
499 def output_variation (self
, tool
, variation
):
500 self
.output_segment (variation
.header
)
501 for harness
in sorted (variation
.harnesses
.values(),
502 key
= attrgetter ('name')):
503 sys
.stdout
.write ('Running ' + harness
.name
+ ' ...\n')
505 harness
.results
.sort()
506 for (key
, line
) in harness
.results
:
507 sys
.stdout
.write (line
)
509 # Rearrange the log segments into test order (but without
510 # rearranging text within those segments).
511 for key
in sorted (harness
.segments
.keys()):
512 self
.output_segment (harness
.segments
[key
])
513 for segment
in harness
.empty
:
514 self
.output_segment (segment
)
515 if len (self
.variations
) > 1:
516 sys
.stdout
.write ('\t\t=== ' + tool
.name
+ ' Summary for '
517 + variation
.name
+ ' ===\n\n')
518 self
.output_summary (tool
, variation
.counts
)
520 # Output unified .log or .sum information for a particular tool,
521 # with a summary at the end.
522 def output_tool (self
, tool
):
523 counts
= self
.zero_counts()
524 if tool
.name
== 'acats':
525 # acats doesn't use variations, so just output everything.
526 # It also has a different approach to whitespace.
527 sys
.stdout
.write ('\t\t=== ' + tool
.name
+ ' tests ===\n')
528 for variation
in tool
.variations
.values():
529 self
.output_variation (tool
, variation
)
530 self
.accumulate_counts (counts
, variation
.counts
)
531 sys
.stdout
.write ('\t\t=== ' + tool
.name
+ ' Summary ===\n')
533 # Output the results in the usual dejagnu runtest format.
534 sys
.stdout
.write ('\n\t\t=== ' + tool
.name
+ ' tests ===\n\n'
535 'Schedule of variations:\n')
536 for name
in self
.variations
:
537 if name
in tool
.variations
:
538 sys
.stdout
.write (' ' + name
+ '\n')
539 sys
.stdout
.write ('\n')
540 for name
in self
.variations
:
541 if name
in tool
.variations
:
542 variation
= tool
.variations
[name
]
543 sys
.stdout
.write ('Running target '
544 + variation
.name
+ '\n')
545 self
.output_variation (tool
, variation
)
546 self
.accumulate_counts (counts
, variation
.counts
)
547 sys
.stdout
.write ('\n\t\t=== ' + tool
.name
+ ' Summary ===\n\n')
548 self
.output_summary (tool
, counts
)
553 # Parse the input files.
554 for filename
in self
.files
:
555 with
safe_open (filename
) as file:
556 self
.parse_file (filename
, file)
558 # Decide what to output.
559 if len (self
.variations
) == 0:
560 self
.variations
= sorted (self
.known_variations
)
562 for name
in self
.variations
:
563 if name
not in self
.known_variations
:
564 self
.fatal (None, 'no results for ' + name
)
565 if len (self
.tools
) == 0:
566 self
.tools
= sorted (self
.runs
.keys())
570 sys
.stdout
.write (self
.start_line
[1])
571 sys
.stdout
.write (self
.native_line
)
572 sys
.stdout
.write (self
.target_line
)
573 sys
.stdout
.write (self
.host_line
)
574 sys
.stdout
.write (self
.acats_premable
)
576 # Output the main body.
577 for name
in self
.tools
:
578 if name
not in self
.runs
:
579 self
.fatal (None, 'no results for ' + name
)
580 self
.output_tool (self
.runs
[name
])
583 if len (self
.acats_failures
) > 0:
584 sys
.stdout
.write ('*** FAILURES: '
585 + ' '.join (self
.acats_failures
) + '\n')
586 sys
.stdout
.write (self
.version_output
)
588 sys
.stdout
.write (self
.end_line
[1])
590 self
.fatal (e
.filename
, e
.strerror
)
This page took 0.048866 seconds and 4 git commands to generate.