3 # Copyright (C) 2016 Free Software Foundation, Inc.
5 # This file is part of GDB.
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 # This program is used to analyze the test results (i.e., *.sum files)
22 # generated by GDB's testsuite, and print the testcases that are found
25 # Racy testcases are considered as being testcases which can
26 # intermittently FAIL (or PASS) when run two or more times
27 # consecutively, i.e., tests whose results are not deterministic.
29 # This program is invoked when the user runs "make check" and
30 # specifies the RACY_ITER environment variable.
36 # The (global) dictionary that stores the associations between a *.sum
37 # file and its results. The data inside it will be stored as:
39 # files_and_tests = { 'file1.sum' : { 'PASS' : { 'test1', 'test2' ... },
40 # 'FAIL' : { 'test5', 'test6' ... },
43 # { 'file2.sum' : { 'PASS' : { 'test1', 'test3' ... },
48 files_and_tests
= dict ()
50 # The relatioships between various states of the same tests that
51 # should be ignored. For example, if the same test PASSes on a
52 # testcase run but KFAILs on another, this test should be considered
53 # racy because a known-failure is... known.
55 ignore_relations
= { 'PASS' : 'KFAIL' }
57 # We are interested in lines that start with '.?(PASS|FAIL)'. In
58 # other words, we don't process errors (maybe we should).
60 sum_matcher
= re
.compile('^(.?(PASS|FAIL)): (.*)$')
62 def parse_sum_line (line
, dic
):
63 """Parse a single LINE from a sumfile, and store the results in the
64 dictionary referenced by DIC."""
68 m
= re
.match (sum_matcher
, line
)
71 test_name
= m
.group (3)
72 # Remove tail parentheses. These are likely to be '(timeout)'
73 # and other extra information that will only confuse us.
74 test_name
= re
.sub ('(\s+)?\(.*$', '', test_name
)
75 if result
not in dic
.keys ():
77 if test_name
in dic
[result
]:
78 # If the line is already present in the dictionary, then
79 # we include a unique identifier in the end of it, in the
80 # form or '<<N>>' (where N is a number >= 2). This is
81 # useful because the GDB testsuite is full of non-unique
82 # test messages; however, if you process the racy summary
83 # file you will also need to perform this same operation
84 # in order to identify the racy test.
87 nname
= test_name
+ ' <<' + str (i
) + '>>'
88 if nname
not in dic
[result
]:
92 dic
[result
].add (test_name
)
94 def read_sum_files (files
):
95 """Read the sumfiles (passed as a list in the FILES variable), and
96 process each one, filling the FILES_AND_TESTS global dictionary with
97 information about them. """
98 global files_and_tests
101 with
open (x
, 'r') as f
:
102 files_and_tests
[x
] = dict ()
103 for line
in f
.readlines ():
104 parse_sum_line (line
, files_and_tests
[x
])
106 def identify_racy_tests ():
107 """Identify and print the racy tests. This function basically works
108 on sets, and the idea behind it is simple. It takes all the sets that
109 refer to the same result (for example, all the sets that contain PASS
110 tests), and compare them. If a test is present in all PASS sets, then
111 it is not racy. Otherwise, it is.
113 This function does that for all sets (PASS, FAIL, KPASS, KFAIL, etc.),
114 and then print a sorted list (without duplicates) of all the tests
115 that were found to be racy."""
116 global files_and_tests
118 # First, construct two dictionaries that will hold one set of
119 # testcases for each state (PASS, FAIL, etc.).
121 # Each set in NONRACY_TESTS will contain only the non-racy
122 # testcases for that state. A non-racy testcase is a testcase
123 # that has the same state in all test runs.
125 # Each set in ALL_TESTS will contain all tests, racy or not, for
127 nonracy_tests
= dict ()
129 for f
in files_and_tests
:
130 for state
in files_and_tests
[f
]:
132 nonracy_tests
[state
] &= files_and_tests
[f
][state
].copy ()
134 nonracy_tests
[state
] = files_and_tests
[f
][state
].copy ()
137 all_tests
[state
] |
= files_and_tests
[f
][state
].copy ()
139 all_tests
[state
] = files_and_tests
[f
][state
].copy ()
141 # Now, we eliminate the tests that are present in states that need
142 # to be ignored. For example, tests both in the PASS and KFAIL
143 # states should not be considered racy.
144 ignored_tests
= set ()
145 for s1
, s2
in ignore_relations
.iteritems ():
147 ignored_tests |
= (all_tests
[s1
] & all_tests
[s2
])
152 for f
in files_and_tests
:
153 for state
in files_and_tests
[f
]:
154 racy_tests |
= files_and_tests
[f
][state
] - nonracy_tests
[state
]
156 racy_tests
= racy_tests
- ignored_tests
159 print "\t\t=== gdb racy tests ===\n"
162 for line
in sorted (racy_tests
):
167 print "\t\t=== gdb Summary ===\n"
168 print "# of racy tests:\t\t%d" % len (racy_tests
)
170 if __name__
== '__main__':
171 if len (sys
.argv
) < 3:
172 # It only makes sense to invoke this program if you pass two
173 # or more files to be analyzed.
174 sys
.exit ("Usage: %s [FILE] [FILE] ..." % sys
.argv
[0])
175 read_sum_files (sys
.argv
[1:])
176 identify_racy_tests ()
This page took 0.03276 seconds and 4 git commands to generate.