Commit | Line | Data |
---|---|---|
970ed795 EL |
1 | #!/bin/bash |
2 | ############################################################################### | |
3abe9331 | 3 | # Copyright (c) 2000-2015 Ericsson Telecom AB |
970ed795 EL |
4 | # All rights reserved. This program and the accompanying materials |
5 | # are made available under the terms of the Eclipse Public License v1.0 | |
6 | # which accompanies this distribution, and is available at | |
7 | # http://www.eclipse.org/legal/epl-v10.html | |
8 | ############################################################################### | |
9 | ||
10 | MYEXE="EmergencyLogTest" | |
11 | LOGDIR="logs" | |
12 | SUCCESS=0 | |
13 | LIST_OF_FAILED="failed_testcases.txt" | |
14 | DEBUG="true" | |
15 | ||
16 | ##################################### | |
17 | #init: | |
18 | # sets TTCN3_DIR, if is needed | |
19 | # makes executable | |
20 | ##################################### | |
21 | init() { | |
22 | if [ "$TTCN3_DIR" == "" ] | |
23 | then | |
24 | echo "TTCN3_DIR should be set" | |
25 | exit | |
26 | fi | |
27 | ||
28 | make | |
29 | chmod +x ./$MYEXE | |
30 | success=0 | |
31 | failed=0 | |
32 | echo "Failed tests:" > ${LIST_OF_FAILED} | |
33 | rm -f logs/* | |
34 | #TODO: set cut off length according to TimeStampFormat | |
35 | } | |
36 | ||
37 | ##################################### | |
38 | # debug | |
39 | # $1 : log file | |
40 | ##################################### | |
41 | debug(){ | |
42 | if [ "$DEBUG" == "true" ] ; then | |
43 | echo "$1" | |
44 | fi | |
45 | } | |
46 | ||
47 | ##################################### | |
48 | # modify_logfile | |
49 | # $1 : log file | |
50 | ##################################### | |
51 | modify_logfile() { | |
52 | #remove timestamps, data different for run by run | |
53 | debug "Modifying file $1" | |
54 | cmd=' | |
55 | s/\ (No such file or directory)//g | |
56 | s/^EXECUTOR_RUNTIME - MTC was created. Process id: [0-9]*\./EXECUTOR_RUNTIME - MTC was created. Process id:/g | |
57 | s/rocess id: [0-9][0-9]*/rocess id: X/g | |
58 | s/I\/O: [0-9][0-9]*/I\/O: Y/g | |
59 | s/switches: [0-9][0-9]*/switches: S/g | |
60 | s/block output operations: [0-9]*/block output operations: B/g | |
61 | s/block input operations: [0-9]*/block input operations: I/g | |
62 | s/system time: [0-9]*\.[0-9]*/system time: SYS/g | |
63 | s/user time: [0-9]*\.[0-9]*/user time: T/g | |
64 | s/seed [0-9][0-9]*[.][0-9]*/seed /g | |
65 | s/returned [0-9][0-9]*[.][0-9]*/returned R/g | |
66 | s/srand48.[\-]*[0-9]*.\./srand X/g | |
67 | s/t1: [0-9]e-[0-9]* s/t1: T s/g | |
68 | s/t1: [0-9e.-]* s/t1: T s/g | |
69 | s/t: [0-9e.-]* s/t: T s/g | |
70 | s/Mytime: [0-9.]*e-[0-9]* s/Mytime: T s/g | |
71 | s/Mytime: [0-9.]* s/Mytime: T s/g | |
72 | s/reference [0-9]* finished/reference R finished/g | |
73 | s/started on [a-zA-Z0-9._-]*. Version: .*/started on X. Version: V/g | |
74 | s/^[0-9.:]* //g | |
75 | s/[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*/a\.b\.c\.d/g | |
76 | s/The address of MC was set to [a-zA-Z0-9._-]*/The address of MC was set to X/g | |
77 | s/resident set size: [0-9]*/resident set size: S/g | |
78 | s/Action: Finish:[0-9][0-9]*[.][0-9]*/Action: Finish: F/g | |
79 | s/Action: Elapsed time:[0-9][0-9]*[.][0-9]*/Action: Elapsed time:E/g | |
80 | s/Read timer t: [0-9e.-]* s/Read timer t: T s/g | |
81 | s/Read timer t1: [0-9e.-]* s/Read timer t1: T s/g | |
82 | s/Action: Start:[0-9e.-]* s/Action: Start: S s/g | |
83 | s/^PARALLEL_PTC.*//g | |
84 | s/UNIX pathname .*$/UNIX pathname /g | |
85 | s/^EXECUTOR_COMPONENT.*$//g | |
86 | s/^PORTEVENT_MQUEUE.*$//g | |
87 | s/^$//g | |
88 | ' | |
89 | sed -e "$cmd" < "$1" > "${LOGDIR}/tmp" | |
90 | ||
91 | sed -e "/^$/d" < "${LOGDIR}/tmp" > "$1" | |
92 | #mv "${LOGDIR}/tmp" "$1" | |
93 | } | |
94 | ||
95 | ##################################### | |
96 | # run_and_modify_logfile | |
97 | # $1 : cfg file | |
98 | # $2 : cfg file base without extension | |
99 | ##################################### | |
100 | run_and_modify_logfile() { | |
3f84031e | 101 | echo $TTCN3_DIR/bin/ttcn3_start $MYEXE $1 |
970ed795 EL |
102 | $TTCN3_DIR/bin/ttcn3_start $MYEXE $1 |
103 | cp $LOGDIR/${MYEXE}-mtc.log $LOGDIR/${2}-mtc.log | |
104 | cp $LOGDIR/${MYEXE}-hc.log $LOGDIR/${2}-hc.log | |
105 | ||
106 | #remove timestamp, diff data: | |
107 | modify_logfile "${LOGDIR}/${2}-mtc.log" | |
108 | modify_logfile "${LOGDIR}/${2}-hc.log" | |
109 | ||
110 | #emergency log handling | |
111 | if [ -e "$LOGDIR/${MYEXE}-mtc.log_emergency" ] | |
112 | then | |
113 | debug "Emergency log file name: $LOGDIR/${2}-mtc.log_emergency" | |
114 | cp "$LOGDIR/${MYEXE}-mtc.log_emergency" "$LOGDIR/${2}-mtc.log_emergency" | |
115 | modify_logfile "$LOGDIR/${2}-mtc.log_emergency" | |
116 | fi | |
117 | ||
118 | i=3 | |
119 | while [ -e "$LOGDIR/${MYEXE}-$i.log" ] | |
120 | do | |
121 | cp "$LOGDIR/${MYEXE}-$i.log" "$LOGDIR/${2}-$i.log" | |
122 | modify_logfile "$LOGDIR/${2}-$i.log" | |
123 | let "i = $i + 1" | |
124 | done | |
125 | } | |
126 | ##################################### | |
127 | # create EmergencyLogCommentedOut (ELCO) file from the original log file | |
128 | # $1:original config file | |
129 | # output "${1}_ELCO.cfg" | |
130 | ##################################### | |
131 | create_ELCO_config() { | |
132 | elco_cfg=`basename $1 ".cfg"` | |
133 | elco_cfg="${elco_cfg}_ELCO.cfg" | |
134 | cp $1 "${elco_cfg}" | |
135 | cmd=' | |
136 | s/^\*\.Emergency/#\*\.Emergency/g' | |
137 | sed -e "$cmd" < "${elco_cfg}" > tmp | |
138 | mv tmp "${elco_cfg}" | |
139 | debug "OLD cfg name: $1" | |
140 | debug "ELCO cfg name: ${elco_cfg}" | |
141 | } | |
142 | ##################################### | |
143 | # compare_with_ELCO | |
144 | # $1: componenent id (mtc, hc, 3, 4 etc) | |
145 | ##################################### | |
146 | compare_with_ELCO() { | |
147 | diff -u "$LOGDIR/${elco_cfg_base}-$1.log" "$LOGDIR/${orig_cfg_base}-$1.log" > "${LOGDIR}/diff_${orig_cfg_base}_$1.log" | |
148 | if [ "$?" -eq $SUCCESS ] | |
149 | then | |
150 | debug ">>>tc $orig_cfg_base $1 part success<<<" | |
151 | let "success = $success + 1" | |
152 | else | |
153 | debug ">>>tc $orig_cfg_base $1 part failed<<<" | |
154 | echo "tc $orig_cfg_base $1 part failed" >> ${LIST_OF_FAILED} | |
155 | let "failed = $failed + 1" | |
156 | fi | |
157 | debug "success: ${success} failed: ${failed}" | |
158 | } | |
159 | ||
160 | ##################################### | |
161 | # compare_with_expected | |
162 | # $1: componenent id (mtc, hc, 3, 4 etc) | |
163 | # $2: expected log | |
164 | ##################################### | |
165 | compare_with_expected() { | |
166 | diff -u "$2" "${LOGDIR}/${orig_cfg_base}-$1.log" > "${LOGDIR}/diff_${orig_cfg_base}_$1.log" | |
167 | if [ "$?" -eq $SUCCESS ] | |
168 | then | |
169 | debug ">>>tc $orig_cfg_base $1 part success<<<" | |
170 | let "success = $success + 1" | |
171 | else | |
172 | debug ">>>tc $orig_cfg_base $1 part failed<<<" | |
173 | echo "tc $orig_cfg_base $1 part failed" >> ${LIST_OF_FAILED} | |
174 | let "failed = $failed + 1" | |
175 | fi | |
176 | debug "success: ${success} failed: ${failed}" | |
177 | } | |
178 | ||
179 | ##################################### | |
180 | # compare_with_expected_emergency | |
181 | # $1: componenent id (mtc, hc, 3, 4 etc) | |
182 | # $2: expected log | |
183 | ##################################### | |
184 | compare_with_expected_emergency() { | |
185 | #debug "==>diff -u $4 ${LOGDIR}/${orig_cfg_base}-$1.log_emergency > ${LOGDIR}/diff_${orig_cfg_base}_$1.log_emergency" | |
186 | diff -u "$2" "${LOGDIR}/${orig_cfg_base}-$1.log_emergency" > "${LOGDIR}/diff_${orig_cfg_base}_$1.log_emergency" | |
187 | if [ "$?" -eq $SUCCESS ] | |
188 | then | |
189 | debug ">>>tc $orig_cfg_base $1 part success<<<" | |
190 | let "success = $success + 1" | |
191 | else | |
192 | debug ">>>tc $orig_cfg_base $1 part failed<<<" | |
193 | echo "tc $orig_cfg_base $1 emergency part failed" >> ${LIST_OF_FAILED} | |
194 | let "failed = $failed + 1" | |
195 | fi | |
196 | debug "success: ${success} failed: ${failed}" | |
197 | } | |
198 | ##################################### | |
199 | # run_and_compare_log_files | |
200 | # $1 first config file | |
201 | # $2 2nd config file | |
202 | # $3 output file (diff_) | |
203 | ##################################### | |
204 | run_and_compare_log_files() { | |
205 | #running with orig cfg: | |
206 | rm ${LOGDIR}/${MYEXE}*.log | |
207 | orig_cfg_base=`basename $1 ".cfg"` | |
208 | run_and_modify_logfile "$1" "$orig_cfg_base" | |
209 | ||
210 | if [ "$2" == "" ]; then | |
211 | #running with modified cfg: | |
212 | create_ELCO_config "$1" | |
213 | else | |
214 | elco_cfg=$2 | |
215 | fi | |
216 | ||
217 | elco_cfg_base=`basename "$elco_cfg" ".cfg"` | |
218 | debug "ELCO cfg name: ${elco_cfg}" | |
219 | run_and_modify_logfile "${elco_cfg}" "$elco_cfg_base" | |
220 | ||
221 | #diff: | |
222 | compare_with_ELCO "mtc" | |
223 | compare_with_ELCO "hc" | |
224 | ||
225 | i=3 | |
226 | while [ -e "$LOGDIR/${elco_cfg_base}-$i.log" -a -e "$LOGDIR/${orig_cfg_base}-$i.log" ] | |
227 | do | |
228 | compare_with_ELCO "$i" | |
229 | let "i = $i + 1" | |
230 | done | |
231 | ||
232 | } | |
233 | ##################################### | |
234 | # run_and_compare_log_file_with_expected | |
235 | # $1 first config file | |
236 | # $2 expected mtc log file | |
237 | # $3 expected hc log file | |
238 | # $4 expected emergency log file | |
239 | ##################################### | |
240 | run_and_compare_log_file_with_expected() { | |
241 | rm -f ${LOGDIR}/${MYEXE}*.log* | |
242 | #running with orig cfg: | |
243 | orig_cfg_base=`basename $1 ".cfg"` | |
244 | run_and_modify_logfile "$1" "$orig_cfg_base" | |
245 | ||
246 | ||
247 | debug "cfg: $1" | |
248 | debug "mtclog: $2" | |
249 | debug "hclog: $3" | |
250 | #diff mtc with the expected: | |
251 | if [ -e "$2" ]; then | |
252 | modify_logfile "$2" | |
253 | compare_with_expected "mtc" "$2" | |
254 | else | |
255 | debug ">>>tc $orig_cfg_base mtc part failed, not existing expected log file $2<<<" | |
256 | echo "tc $orig_cfg_base mtc part failed" >> ${LIST_OF_FAILED} | |
257 | let "failed = $failed + 1" | |
258 | fi | |
259 | ||
260 | #diff: | |
261 | ||
262 | if [ "$3" != "" ]; then | |
263 | if [ -e "$3" ]; then | |
264 | modify_logfile "$3" | |
265 | compare_with_expected "hc" "$3" | |
266 | else | |
267 | debug ">>>tc $orig_cfg_base mtc part failed, not existing expected log file $3<<<" | |
268 | echo "tc $orig_cfg_base mtc part failed" >> ${LIST_OF_FAILED} | |
269 | let "failed = $failed + 1" | |
270 | fi | |
271 | fi | |
272 | ||
273 | #mtc emergency log kezeles | |
274 | if [ "$4" != "" ]; then | |
275 | if [ -e "$4" ]; then | |
276 | modify_logfile "$4" | |
277 | compare_with_expected_emergency "mtc" "$4" | |
278 | else | |
279 | debug ">>>tc $orig_cfg_base mtc part failed, not existing expected log file $4<<<" | |
280 | echo "tc $orig_cfg_base mtc part failed, not existing expected log file $4" >> ${LIST_OF_FAILED} | |
281 | let "failed = $failed + 1" | |
282 | fi | |
283 | fi | |
284 | } | |
285 | ||
286 | ##################################### | |
287 | # evaluate | |
288 | ##################################### | |
289 | evaluate() { | |
290 | echo "Summary: success: ${success}, failed: ${failed} " | |
291 | if [ $failed -eq 0 ] | |
292 | then | |
293 | echo "Overall verdict: success" | |
294 | exit_code=0 | |
295 | else | |
296 | echo "Overall verdict: failed" | |
297 | cat ${LIST_OF_FAILED} | | |
298 | while read line | |
299 | do | |
300 | echo $line | |
301 | done | |
302 | exit_code=1 | |
303 | fi | |
304 | ||
305 | exit $exit_code | |
306 | } | |
307 | ||
308 | #### MAIN ### | |
309 | ||
310 | # see the files "logs/diff*.log" ! | |
311 | ||
312 | init | |
313 | #======= Buffer All ======== | |
314 | run_and_compare_log_files EL_BufferAll_1.cfg #compatibility test with minimal coverage (compare logs with/without EL setting if EL-event does not happen | |
315 | run_and_compare_log_files EL_BufferAll_2.cfg #compatibility test with coverage "tc_parallel_portconn" (compare logs with/without EL setting if EL-event does not happen) - NOT STABILE, hc FAILED | |
316 | run_and_compare_log_files EL_BufferAll_3.cfg EL_BufferAll_3_NOEL.cfg #the same log expected with two different log files: EL and NOEL, EL-event occurs in the first case - PASSED | |
317 | run_and_compare_log_files EL_BufferAll_4.cfg #compatibility test with "tc_timer" (compare logs with/without EL setting if EL-event does not happen) - hc FAILED | |
318 | run_and_compare_log_files EL_BufferAll_5.cfg #compatibility test "with Titan_LogTest.tc_encdec" - NOT STABILE, mtc PASSED, hc PASSED | |
319 | run_and_compare_log_files EL_BufferAll_6.cfg #compatibility test with "tc_function_rnd and tc_encdec" - mtc PASSED, hc PASSED | |
320 | run_and_compare_log_file_with_expected "EL_BufferAll_7.cfg" "EL_BufferAll_7_mtc_expected.log" # - mtc PASSED | |
321 | run_and_compare_log_file_with_expected "EL_BufferAll_7A.cfg" "EL_BufferAll_7A_mtc_expected.log" # - mtc PASSED | |
322 | run_and_compare_log_files EL_BufferAll_8.cfg #compatibility test with MAXIMAL coverage - NOT STABILE; mtc PASSED hc FAILED (1 line order problem, | |
323 | run_and_compare_log_file_with_expected "EL_BufferAll_9.cfg" "EL_BufferAll_9_mtc_expected.log" # - More EL-event can be logged after each other - mtc FAILED ( 1 line missing), hc FAILED | |
324 | run_and_compare_log_file_with_expected "EL_BufferAll_10.cfg" "EL_BufferAll_10_mtc_expected.log" # - More EL-event with big buffer - mtc PASSED | |
325 | run_and_compare_log_file_with_expected "EL_BufferAll_11.cfg" "EL_BufferAll_11_mtc_expected.log" # - More EL-event with big buffer - mtc PASSED | |
326 | run_and_compare_log_file_with_expected "EL_BufferAll_12.cfg" "EL_BufferAll_12_mtc_expected.log" # - More EL-event with very small buffer - (buffer size=2) - mtc FAILED, one line missing | |
327 | run_and_compare_log_file_with_expected "EL_BufferAll_13.cfg" "EL_BufferAll_13_mtc_expected.log" # - More EL-event with very small buffer - (buffer size=3) - mtc FAILED, one line missing | |
328 | #======= Buffer Masked ======== | |
329 | run_and_compare_log_files EL_BufferMasked_1.cfg #compatibility test with minimal coverage (compare logs with/without EL setting if EL-event does not happen | |
330 | run_and_compare_log_files EL_BufferMasked_2.cfg #compatibility test with coverage "tc_parallel_portconn" (compare logs with/without EL setting if EL-event does not happen) - NOT STABILE | |
331 | run_and_compare_log_files EL_BufferMasked_4.cfg #compatibility test with "tc_timer" (compare logs with/without EL setting if EL-event does not happen) - PASSED | |
332 | run_and_compare_log_files EL_BufferMasked_5.cfg #compatibility test "with Titan_LogTest.tc_encdec" - NOT STABILE | |
333 | run_and_compare_log_files EL_BufferMasked_3.cfg EL_BufferMasked_3_NOEL.cfg #the same log expected with two different log files: EL and NOEL, EL-event occurs in the first case - PASSED | |
334 | run_and_compare_log_files EL_BufferMasked_6.cfg #compatibility test with "tc_function_rnd and tc_encdec" - PASSED | |
335 | run_and_compare_log_file_with_expected "EL_BufferMasked_7.cfg" "EL_BufferMasked_7_mtc_expected.log" "" "EL_BufferMasked_7_mtc_expected.log_emergency" # - PASSED | |
336 | run_and_compare_log_files EL_BufferMasked_8.cfg #compatibility test with MAXIMAL coverage - NOT STABILE | |
337 | run_and_compare_log_file_with_expected "EL_BufferMasked_9.cfg" "EL_BufferMasked_9_mtc_expected.log" "" "EL_BufferMasked_9_mtc_expected.log_emergency" # - More EL-event can be logged after each other - PASSED | |
338 | run_and_compare_log_file_with_expected "EL_BufferMasked_10.cfg" "EL_BufferMasked_10_mtc_expected.log" "" "EL_BufferMasked_10_mtc_expected.log_emergency" # - More EL-event with big buffer - PASSED | |
339 | run_and_compare_log_file_with_expected "EL_BufferMasked_11.cfg" "EL_BufferMasked_11_mtc_expected.log" "" "EL_BufferMasked_11_mtc_expected.log_emergency" # - More EL-event with big buffer - PASSED | |
340 | run_and_compare_log_file_with_expected "EL_BufferMasked_12.cfg" "EL_BufferMasked_12_mtc_expected.log" "" "EL_BufferMasked_12_mtc_expected.log_emergency" # - More EL-event with very small buffer - (buffer size=2) - mtc.log_emergency FAILED (wrong order) | |
341 | run_and_compare_log_file_with_expected "EL_BufferMasked_13.cfg" "EL_BufferMasked_13_mtc_expected.log" "" "EL_BufferMasked_13_mtc_expected.log_emergency" # - More EL-event with very small buffer - (buffer size=3) - mtc.log_emergency FAILED (wrong order) | |
342 | #Stat: 63 success/ 0 failed | |
343 | evaluate |