Commit | Line | Data |
---|---|---|
fb6a751f SDJ |
1 | #!/usr/bin/env python |
2 | ||
61baf725 | 3 | # Copyright (C) 2016-2017 Free Software Foundation, Inc. |
fb6a751f SDJ |
4 | # |
5 | # This file is part of GDB. | |
6 | # | |
7 | # This program is free software; you can redistribute it and/or modify | |
8 | # it under the terms of the GNU General Public License as published by | |
9 | # the Free Software Foundation; either version 3 of the License, or | |
10 | # (at your option) any later version. | |
11 | # | |
12 | # This program is distributed in the hope that it will be useful, | |
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | # GNU General Public License for more details. | |
16 | # | |
17 | # You should have received a copy of the GNU General Public License | |
18 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
19 | ||
20 | ||
21 | # This program is used to analyze the test results (i.e., *.sum files) | |
22 | # generated by GDB's testsuite, and print the testcases that are found | |
23 | # to be racy. | |
24 | # | |
25 | # Racy testcases are considered as being testcases which can | |
26 | # intermittently FAIL (or PASS) when run two or more times | |
27 | # consecutively, i.e., tests whose results are not deterministic. | |
28 | # | |
29 | # This program is invoked when the user runs "make check" and | |
30 | # specifies the RACY_ITER environment variable. | |
31 | ||
32 | import sys | |
33 | import os | |
34 | import re | |
35 | ||
36 | # The (global) dictionary that stores the associations between a *.sum | |
37 | # file and its results. The data inside it will be stored as: | |
38 | # | |
39 | # files_and_tests = { 'file1.sum' : { 'PASS' : { 'test1', 'test2' ... }, | |
40 | # 'FAIL' : { 'test5', 'test6' ... }, | |
41 | # ... | |
42 | # }, | |
43 | # { 'file2.sum' : { 'PASS' : { 'test1', 'test3' ... }, | |
44 | # ... | |
45 | # } | |
46 | # } | |
47 | ||
48 | files_and_tests = dict () | |
49 | ||
50 | # The relatioships between various states of the same tests that | |
51 | # should be ignored. For example, if the same test PASSes on a | |
52 | # testcase run but KFAILs on another, this test should be considered | |
53 | # racy because a known-failure is... known. | |
54 | ||
55 | ignore_relations = { 'PASS' : 'KFAIL' } | |
56 | ||
57 | # We are interested in lines that start with '.?(PASS|FAIL)'. In | |
58 | # other words, we don't process errors (maybe we should). | |
59 | ||
60 | sum_matcher = re.compile('^(.?(PASS|FAIL)): (.*)$') | |
61 | ||
62 | def parse_sum_line (line, dic): | |
63 | """Parse a single LINE from a sumfile, and store the results in the | |
64 | dictionary referenced by DIC.""" | |
65 | global sum_matcher | |
66 | ||
67 | line = line.rstrip () | |
68 | m = re.match (sum_matcher, line) | |
69 | if m: | |
70 | result = m.group (1) | |
71 | test_name = m.group (3) | |
72 | # Remove tail parentheses. These are likely to be '(timeout)' | |
73 | # and other extra information that will only confuse us. | |
74 | test_name = re.sub ('(\s+)?\(.*$', '', test_name) | |
75 | if result not in dic.keys (): | |
76 | dic[result] = set () | |
77 | if test_name in dic[result]: | |
78 | # If the line is already present in the dictionary, then | |
79 | # we include a unique identifier in the end of it, in the | |
80 | # form or '<<N>>' (where N is a number >= 2). This is | |
81 | # useful because the GDB testsuite is full of non-unique | |
82 | # test messages; however, if you process the racy summary | |
83 | # file you will also need to perform this same operation | |
84 | # in order to identify the racy test. | |
85 | i = 2 | |
86 | while True: | |
87 | nname = test_name + ' <<' + str (i) + '>>' | |
88 | if nname not in dic[result]: | |
89 | break | |
90 | i += 1 | |
91 | test_name = nname | |
92 | dic[result].add (test_name) | |
93 | ||
94 | def read_sum_files (files): | |
95 | """Read the sumfiles (passed as a list in the FILES variable), and | |
96 | process each one, filling the FILES_AND_TESTS global dictionary with | |
97 | information about them. """ | |
98 | global files_and_tests | |
99 | ||
100 | for x in files: | |
101 | with open (x, 'r') as f: | |
102 | files_and_tests[x] = dict () | |
103 | for line in f.readlines (): | |
104 | parse_sum_line (line, files_and_tests[x]) | |
105 | ||
106 | def identify_racy_tests (): | |
107 | """Identify and print the racy tests. This function basically works | |
108 | on sets, and the idea behind it is simple. It takes all the sets that | |
109 | refer to the same result (for example, all the sets that contain PASS | |
110 | tests), and compare them. If a test is present in all PASS sets, then | |
111 | it is not racy. Otherwise, it is. | |
112 | ||
113 | This function does that for all sets (PASS, FAIL, KPASS, KFAIL, etc.), | |
114 | and then print a sorted list (without duplicates) of all the tests | |
115 | that were found to be racy.""" | |
116 | global files_and_tests | |
117 | ||
118 | # First, construct two dictionaries that will hold one set of | |
119 | # testcases for each state (PASS, FAIL, etc.). | |
120 | # | |
121 | # Each set in NONRACY_TESTS will contain only the non-racy | |
122 | # testcases for that state. A non-racy testcase is a testcase | |
123 | # that has the same state in all test runs. | |
124 | # | |
125 | # Each set in ALL_TESTS will contain all tests, racy or not, for | |
126 | # that state. | |
127 | nonracy_tests = dict () | |
128 | all_tests = dict () | |
129 | for f in files_and_tests: | |
130 | for state in files_and_tests[f]: | |
131 | try: | |
132 | nonracy_tests[state] &= files_and_tests[f][state].copy () | |
133 | except KeyError: | |
134 | nonracy_tests[state] = files_and_tests[f][state].copy () | |
135 | ||
136 | try: | |
137 | all_tests[state] |= files_and_tests[f][state].copy () | |
138 | except KeyError: | |
139 | all_tests[state] = files_and_tests[f][state].copy () | |
140 | ||
141 | # Now, we eliminate the tests that are present in states that need | |
142 | # to be ignored. For example, tests both in the PASS and KFAIL | |
143 | # states should not be considered racy. | |
144 | ignored_tests = set () | |
145 | for s1, s2 in ignore_relations.iteritems (): | |
146 | try: | |
147 | ignored_tests |= (all_tests[s1] & all_tests[s2]) | |
148 | except: | |
149 | continue | |
150 | ||
151 | racy_tests = set () | |
152 | for f in files_and_tests: | |
153 | for state in files_and_tests[f]: | |
154 | racy_tests |= files_and_tests[f][state] - nonracy_tests[state] | |
155 | ||
156 | racy_tests = racy_tests - ignored_tests | |
157 | ||
158 | # Print the header. | |
159 | print "\t\t=== gdb racy tests ===\n" | |
160 | ||
161 | # Print each test. | |
162 | for line in sorted (racy_tests): | |
163 | print line | |
164 | ||
165 | # Print the summary. | |
166 | print "\n" | |
167 | print "\t\t=== gdb Summary ===\n" | |
168 | print "# of racy tests:\t\t%d" % len (racy_tests) | |
169 | ||
170 | if __name__ == '__main__': | |
171 | if len (sys.argv) < 3: | |
172 | # It only makes sense to invoke this program if you pass two | |
173 | # or more files to be analyzed. | |
174 | sys.exit ("Usage: %s [FILE] [FILE] ..." % sys.argv[0]) | |
175 | read_sum_files (sys.argv[1:]) | |
176 | identify_racy_tests () | |
177 | exit (0) |