Commit | Line | Data |
---|---|---|
970ed795 EL |
1 | ############################################################################### |
2 | # Copyright (c) 2000-2014 Ericsson Telecom AB | |
3 | # All rights reserved. This program and the accompanying materials | |
4 | # are made available under the terms of the Eclipse Public License v1.0 | |
5 | # which accompanies this distribution, and is available at | |
6 | # http://www.eclipse.org/legal/epl-v10.html | |
7 | ############################################################################### | |
8 | import os, re | |
9 | import titan_publisher, utils | |
10 | ||
11 | MAKEFILE_PATCH = 'makefile_patch.sh' | |
12 | ||
13 | class product_handler: | |
14 | """ It is assumed that the VOB is reachable (e.g. through an SSHFS mounted | |
15 | directory) on the master machine. The source package is copied to all | |
16 | slaves with SCP. It is assumed that the logger is always set. | |
17 | ||
18 | Saving stdout and stderr of product actions is optional. During | |
19 | publishing these files should be linked or copied to somewhere under | |
20 | `public_html'. | |
21 | """ | |
22 | def __init__(self, logger = None, config = None): | |
23 | self._logger = logger | |
24 | self._config = config | |
25 | ||
26 | def config_products(self, target_path): | |
27 | """ Localize first in the following order: `test', `demo'. If neither of | |
28 | these directories are found the `src' will be copied, it's a must | |
29 | have. If there's a Makefile or .prj in `test', `demo', the list of | |
30 | files will be gathered from there. Otherwise the contents of these | |
31 | directories excluding the Makefile will be copied to the target | |
32 | directory. A new Makefile will be generated for the copied files. | |
33 | Configspec 129 is used. It's assumed that the ClearCase access is | |
34 | working and all files are accessible through SSHFS etc. The `test' | |
35 | directory always has higher priority than `demo'. However, `demo' is | |
36 | the one released to the customer and not `test', but we're using the | |
37 | TCC_Common VOB anyway. | |
38 | ||
39 | We generate single mode Makefiles here for simplicity. We can grep | |
40 | through the sources for a `create' call, but it's so ugly. It is | |
41 | assumed that the files enumerated in Makefiles or .prj files are all | |
42 | relative to the current directory. The source and target paths are | |
43 | class parameters. It is assumed that the Makefile and the project | |
44 | file are all in the `test' directory. The Makefile will be ignored if | |
45 | there's a project file in the same directory. | |
46 | ||
47 | The distribution of the VOB package is the job of the master. | |
48 | Configuring the products with `ttcn3_makefilegen' is the job of the | |
49 | slaves. | |
50 | ||
51 | Returns: | |
52 | 0 if everything went fine and the VOB package is ready for | |
53 | distribution. 1 otherwise. | |
54 | """ | |
55 | utils.run_cmd('rm -rf ' + target_path) | |
56 | for kind, products in self._config.products.iteritems(): | |
57 | for product in products: | |
58 | product_name = product['name'].strip() | |
59 | local_src_path = os.path.join(os.path.join(self._config.common['vob'], kind), product_name) | |
60 | src_dir = os.path.join(local_src_path, 'src') | |
61 | test_dir = os.path.join(local_src_path, 'test') | |
62 | demo_dir = os.path.join(local_src_path, 'demo') | |
63 | if not os.path.isdir(src_dir): | |
64 | self._logger.error('Missing `src\' directory for product `%s\' ' \ | |
65 | 'in %s, skipping product' % (product_name, local_src_path)) | |
66 | continue | |
67 | else: | |
68 | dirs_to_copy = [] | |
69 | files_to_copy = [] | |
70 | if os.path.isdir(test_dir): | |
71 | dirs_to_copy.append(test_dir) | |
72 | elif os.path.isdir(demo_dir): | |
73 | dirs_to_copy.append(demo_dir) | |
74 | else: | |
75 | # No `demo' or `test'. The `src' is copied only if the other | |
76 | # directories are missing. There can be junk files as well. The | |
77 | # Makefile patch script must have a fixed name | |
78 | # `makefile_patch.sh'. | |
79 | dirs_to_copy.append(src_dir) | |
80 | self._logger.debug('Product `%s\' in %s doesn\'t have the `demo\' or `test\' directories' | |
81 | % (product_name, local_src_path)) | |
82 | product_target_path = os.path.join(os.path.join(target_path, kind), product_name) | |
83 | utils.run_cmd('mkdir -p ' + product_target_path) | |
84 | has_prj_file = False | |
85 | for dir in dirs_to_copy: | |
86 | for dir_path, dir_names, file_names in os.walk(dir): | |
87 | if not has_prj_file and \ | |
88 | len([file_name for file_name in file_names \ | |
89 | if file_name.endswith('.prj')]) > 0: has_prj_file = True | |
90 | for file_name in file_names: | |
91 | if not has_prj_file: # Trust the project file if we have one. | |
92 | files_to_copy.append(os.path.join(dir_path, file_name)) | |
93 | if (file_name == 'Makefile' and not has_prj_file) or file_name.endswith('.prj'): | |
94 | (makefile_patch, extracted_files) = \ | |
95 | self.extract_files(dir_path, os.path.join(dir_path, file_name)) | |
96 | files_to_copy.extend(extracted_files) | |
97 | if makefile_patch: | |
98 | utils.run_cmd('cp -Lf %s %s' \ | |
99 | % (makefile_patch, os.path.join(product_target_path, MAKEFILE_PATCH))) | |
100 | utils.run_cmd('cp -Lf %s %s' % (' '.join(files_to_copy), product_target_path)) | |
101 | utils.run_cmd('chmod 644 * ; chmod 755 *.sh', product_target_path) | |
102 | self._logger.debug('Product `%s\' was configured successfully ' \ | |
103 | 'with %d files' % (product_name, len(files_to_copy))) | |
104 | return 0 | |
105 | ||
106 | def build_products(self, proddir, logdir, config, rt2 = False): | |
107 | """ Build the products provided in the list. Simple `compiler -s' etc. | |
108 | commands are executed from the directories of the products. The | |
109 | actions must be synchronized with the product configuration files. | |
110 | The stderr and stdout of actions is captured here, but it's optional. | |
111 | ||
112 | Arguments: | |
113 | The directory of the products, the actual build configuration and | |
114 | runtime. | |
115 | ||
116 | Returns: | |
117 | The build results in the following format: | |
118 | ||
119 | results['kind1'] = [ | |
120 | {'name1':{'action1':(1, o1, e1), 'action2':-1}}, | |
121 | {'name2':{'action1':(1, o1, e1), 'action2':-1}}, | |
122 | {'name3':-1} | |
123 | ] | |
124 | ||
125 | The standard output and error channels are returned for each action | |
126 | with the return value. The return value is usually the exit status | |
127 | of `make' or the `compiler'. If the element is a simple integer | |
128 | value the action was disabled for the current product. The output | |
129 | of this function is intended to be used by the presentation layer. | |
130 | """ | |
131 | results = {} | |
132 | if not os.path.isdir(logdir): | |
133 | utils.run_cmd('mkdir -p %s' % logdir) | |
134 | for kind, products in self._config.products.iteritems(): | |
135 | results[kind] = [] | |
136 | for product in products: | |
137 | product_name = product['name'].strip() | |
138 | product_dir = os.path.join(proddir, os.path.join(kind, product_name)) | |
139 | if not os.path.isdir(product_dir): | |
140 | # No `src' was found for the product. Maybe a list would be better | |
141 | # instead. | |
142 | results[kind].append({product_name:-1}) | |
143 | continue | |
144 | info = {product_name:{}} | |
145 | for product_key in product.iterkeys(): | |
146 | files = ' '.join(filter(lambda file: file.endswith('.ttcn') \ | |
147 | or file.endswith('.asn'), \ | |
148 | os.listdir(product_dir))) | |
149 | cmd = None | |
150 | if product_key == 'semantic': | |
151 | if product[product_key]: | |
152 | cmd = '%s/bin/compiler -s %s %s' % (config['installdir'], rt2 and '-R' or '', files) | |
153 | else: | |
154 | info[product_name][product_key] = -1 | |
155 | continue | |
156 | elif product_key == 'translate': | |
157 | if product[product_key]: | |
158 | cmd = '%s/bin/compiler %s %s' % (config['installdir'], rt2 and '-R' or '', files) | |
159 | else: | |
160 | info[product_name][product_key] = -1 | |
161 | continue | |
162 | elif product_key == 'compile' or product_key == 'run': | |
163 | if product[product_key]: | |
164 | utils.run_cmd('cd %s && %s/bin/ttcn3_makefilegen ' \ | |
165 | '-fp %s *' % (product_dir, config['installdir'], rt2 and '-R' or '')) | |
166 | if os.path.isfile(os.path.join(product_dir, MAKEFILE_PATCH)): | |
167 | self._logger.debug('Patching Makefile of product `%s\' for the %s runtime' | |
168 | % (product_name, rt2 and 'function-test' or 'load-test')) | |
169 | utils.run_cmd('cd %s && mv Makefile Makefile.bak' % product_dir) | |
170 | utils.run_cmd('cd %s && ./%s Makefile.bak Makefile' % (product_dir, MAKEFILE_PATCH)) | |
171 | cmd = 'make clean ; make dep ; make -j4 ; %s' % (product_key == 'run' and 'make run' or '') | |
172 | else: | |
173 | info[product_name][product_key] = -1 | |
174 | continue | |
175 | else: | |
176 | # Skip `name' or other things. | |
177 | continue | |
178 | (retval, stdout, stderr) = utils.run_cmd(cmd, product_dir, 900) | |
179 | prod_stdout = os.path.join(logdir, '%s_%s_%s_stdout.%s' \ | |
180 | % (kind, product_name, product_key, rt2 and 'rt2' or 'rt1')) | |
181 | prod_stderr = os.path.join(logdir, '%s_%s_%s_stderr.%s' \ | |
182 | % (kind, product_name, product_key, rt2 and 'rt2' or 'rt1')) | |
183 | output_files = (prod_stdout, prod_stderr) | |
184 | try: | |
185 | out_file = open(prod_stdout, 'wt') | |
186 | err_file = open(prod_stderr, 'wt') | |
187 | if 'vobtest_logs' not in config or config['vobtest_logs']: | |
188 | out_file.write(' '.join(stdout)) | |
189 | err_file.write(' '.join(stderr)) | |
190 | out_file.close() | |
191 | err_file.close() | |
192 | except IOError, (errno, strerror): | |
193 | self._logger.error('Error while dumping product results: %s (%s)' \ | |
194 | % (strerror, errno)) | |
195 | info[product_name][product_key] = (retval, output_files, stdout, stderr) | |
196 | results[kind].append(info) | |
197 | return results | |
198 | ||
199 | def extract_files(self, path, filename): | |
200 | """ Extract the files need to be copied all around from a given Makefile | |
201 | or .prj file. It handles wrapped lines (i.e. '\') in Makefiles. """ | |
202 | ||
203 | # Interesting patterns in Makefiles and .prj files. Tuples are faster | |
204 | # than lists, use them for storing constants. | |
205 | prj_matches = ( \ | |
206 | '<Module>\s*(.+)\s*</Module>', \ | |
207 | '<TestPort>\s*(.+)\s*</TestPort>', \ | |
208 | '<Config>\s*(.+)\s*</Config>', \ | |
209 | '<Other>\s*(.+)\s*</Other>', \ | |
210 | '<Other_Source>\s*(.+)\s*</Other_Source>', \ | |
211 | '<File path="\s*(.+)\s*"', \ | |
212 | '<File_Group path="\s*(.+)\s*"' ) | |
213 | ||
214 | makefile_matches = ( \ | |
215 | '^TTCN3_MODULES =\s*(.+)', \ | |
216 | '^ASN1_MODULES =\s*(.+)', \ | |
217 | '^USER_SOURCES =\s*(.+)', \ | |
218 | '^USER_HEADERS =\s*(.+)', \ | |
219 | '^OTHER_FILES =\s*(.+)' ) | |
220 | ||
221 | try: | |
222 | file = open(filename, 'rt') | |
223 | except IOError: | |
224 | self._logger.error('File `%s\' cannot be opened for reading' % filename) | |
225 | return (None, []) | |
226 | ||
227 | files = [] | |
228 | makefile_patch = None | |
229 | if re.search('.*[Mm]akefile$', filename): | |
230 | multi_line = False | |
231 | for line in file: | |
232 | line = line.strip() | |
233 | if multi_line: | |
234 | files.extend(map(lambda f: os.path.join(path, f), line.split())) | |
235 | multi_line = line.endswith('\\') | |
236 | if multi_line: | |
237 | files.pop() | |
238 | else: | |
239 | for line_match in makefile_matches: | |
240 | matched = re.search(line_match, line) | |
241 | if matched: | |
242 | files.extend(map(lambda f: os.path.join(path, f), | |
243 | matched.group(1).split())) | |
244 | multi_line = line.endswith('\\') | |
245 | if multi_line: | |
246 | files.pop() | |
247 | elif re.search('.*\.prj$', filename) or re.search('.*\.grp', filename): | |
248 | files_to_exclude = [] | |
249 | for line in file: | |
250 | # Only basic support for Makefile patching, since it doesn't have a | |
251 | # bright future in its current form... | |
252 | matched = re.search('<ScriptFile_AfterMake>\s*(.+)\s*</ScriptFile_AfterMake>', line) | |
253 | if matched: | |
254 | makefile_patch = os.path.join(path, matched.group(1)) | |
255 | continue | |
256 | matched = re.search('<UnUsed_List>\s*(.+)\s*</UnUsed_List>', line) | |
257 | if matched: | |
258 | files_to_exclude.extend(matched.group(1).split(',')) | |
259 | continue | |
260 | for line_match in prj_matches: | |
261 | matched = re.search(line_match, line) | |
262 | if matched and matched.group(1) not in files_to_exclude: | |
263 | file_to_append = os.path.join(path, matched.group(1)) | |
264 | files_to_append = [] | |
265 | if file_to_append != filename and file_to_append.endswith('.grp'): | |
266 | self._logger.debug('Group file `%s\' found' % file_to_append) | |
267 | last_slash = file_to_append.rfind('/') | |
268 | if last_slash != -1: | |
269 | grp_dir = file_to_append[:last_slash] | |
270 | if path.startswith('/'): | |
271 | grp_dir = os.path.join(path, grp_dir) | |
272 | (not_used, files_to_append) = self.extract_files(grp_dir, file_to_append) | |
273 | else: | |
274 | self._logger.warning('Skipping contents of `%s\', check ' \ | |
275 | 'this file by hand' % file_to_append) | |
276 | files.append(file_to_append) | |
277 | files.extend(files_to_append) | |
278 | break | |
279 | else: | |
280 | self._logger.error('Unsupported project description file: %s\n' % filename) | |
281 | file.close() | |
282 | return (makefile_patch, files) |