regression_test/logger/emergency_logging/EmergencyLogTest.sh

   1 #!/bin/bash
   2 ###############################################################################
   3 # Copyright (c) 2000-2015 Ericsson Telecom AB
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Eclipse Public License v1.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.eclipse.org/legal/epl-v10.html
   8 ###############################################################################
   9
  10 MYEXE="EmergencyLogTest"
  11 LOGDIR="logs"
  12 SUCCESS=0
  13 LIST_OF_FAILED="failed_testcases.txt"
  14 DEBUG="true"
  15
  16 #####################################
  17 #init:
  18 # sets TTCN3_DIR, if is needed
  19 # makes executable
  20 #####################################
  21 init() {
  22   if [ "$TTCN3_DIR" == "" ]
  23   then
  24      echo "TTCN3_DIR should be set"
  25      exit
  26   fi
  27
  28   make
  29   chmod +x ./$MYEXE
  30   success=0
  31   failed=0
  32   echo "Failed tests:" > ${LIST_OF_FAILED}
  33   rm -f logs/*
  34   #TODO: set cut off length according to TimeStampFormat
  35 }
  36
  37 #####################################
  38 # debug
  39 # $1 : log file
  40 #####################################
  41 debug(){
  42   if [ "$DEBUG" == "true" ] ; then
  43    echo "$1"
  44   fi
  45 }
  46
  47 #####################################
  48 # modify_logfile
  49 # $1 : log file
  50 #####################################
  51 modify_logfile() {
  52   #remove timestamps, data different for run by run
  53   debug "Modifying file $1"
  54   cmd='
  55 s/\ (No such file or directory)//g
  56 s/^EXECUTOR_RUNTIME - MTC was created. Process id: [0-9]*\./EXECUTOR_RUNTIME - MTC was created. Process id:/g
  57 s/rocess id: [0-9][0-9]*/rocess id: X/g
  58 s/I\/O: [0-9][0-9]*/I\/O: Y/g
  59 s/switches: [0-9][0-9]*/switches: S/g
  60 s/block output operations: [0-9]*/block output operations: B/g
  61 s/block input operations: [0-9]*/block input operations: I/g
  62 s/system time: [0-9]*\.[0-9]*/system time: SYS/g
  63 s/user time: [0-9]*\.[0-9]*/user time: T/g
  64 s/seed [0-9][0-9]*[.][0-9]*/seed /g
  65 s/returned [0-9][0-9]*[.][0-9]*/returned R/g
  66 s/srand48.[\-]*[0-9]*.\./srand X/g
  67 s/t1: [0-9]e-[0-9]* s/t1: T s/g
  68 s/t1: [0-9e.-]* s/t1: T s/g
  69 s/t: [0-9e.-]* s/t: T s/g
  70 s/Mytime: [0-9.]*e-[0-9]* s/Mytime: T s/g
  71 s/Mytime: [0-9.]* s/Mytime: T s/g
  72 s/reference [0-9]* finished/reference R finished/g
  73 s/started on [a-zA-Z0-9._-]*. Version: .*/started on X. Version: V/g
  74 s/^[0-9.:]* //g
  75 s/[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*/a\.b\.c\.d/g
  76 s/The address of MC was set to [a-zA-Z0-9._-]*/The address of MC was set to X/g
  77 s/resident set size: [0-9]*/resident set size: S/g
  78 s/Action: Finish:[0-9][0-9]*[.][0-9]*/Action: Finish: F/g
  79 s/Action: Elapsed time:[0-9][0-9]*[.][0-9]*/Action: Elapsed time:E/g
  80 s/Read timer t: [0-9e.-]* s/Read timer t: T s/g
  81 s/Read timer t1: [0-9e.-]* s/Read timer t1: T s/g
  82 s/Action: Start:[0-9e.-]* s/Action: Start: S s/g
  83 s/^PARALLEL_PTC.*//g
  84 s/UNIX pathname .*$/UNIX pathname /g
  85 s/^EXECUTOR_COMPONENT.*$//g
  86 s/^PORTEVENT_MQUEUE.*$//g
  87 s/^$//g
  88 '
  89   sed -e "$cmd" < "$1" > "${LOGDIR}/tmp"
  90
  91   sed -e "/^$/d" < "${LOGDIR}/tmp" > "$1"
  92   #mv "${LOGDIR}/tmp" "$1"
  93 }
  94
  95 #####################################
  96 # run_and_modify_logfile
  97 # $1 : cfg file
  98 # $2 : cfg file base without extension
  99 #####################################
 100 run_and_modify_logfile() {
 101   echo $TTCN3_DIR/bin/ttcn3_start $MYEXE $1
 102   $TTCN3_DIR/bin/ttcn3_start $MYEXE $1
 103   cp $LOGDIR/${MYEXE}-mtc.log $LOGDIR/${2}-mtc.log
 104   cp $LOGDIR/${MYEXE}-hc.log $LOGDIR/${2}-hc.log
 105
 106   #remove timestamp, diff data:
 107   modify_logfile "${LOGDIR}/${2}-mtc.log"
 108   modify_logfile "${LOGDIR}/${2}-hc.log"
 109
 110   #emergency log handling
 111   if [ -e "$LOGDIR/${MYEXE}-mtc.log_emergency" ]
 112   then
 113    debug "Emergency log file name: $LOGDIR/${2}-mtc.log_emergency"
 114    cp "$LOGDIR/${MYEXE}-mtc.log_emergency" "$LOGDIR/${2}-mtc.log_emergency"
 115    modify_logfile "$LOGDIR/${2}-mtc.log_emergency"
 116   fi
 117
 118   i=3
 119   while [ -e "$LOGDIR/${MYEXE}-$i.log" ]
 120   do
 121        cp "$LOGDIR/${MYEXE}-$i.log" "$LOGDIR/${2}-$i.log"
 122        modify_logfile "$LOGDIR/${2}-$i.log"
 123        let "i = $i + 1"
 124   done
 125 }
 126 #####################################
 127 # create EmergencyLogCommentedOut (ELCO) file from the original log file
 128 # $1:original config file
 129 # output "${1}_ELCO.cfg"
 130 #####################################
 131 create_ELCO_config() {
 132   elco_cfg=`basename $1 ".cfg"`
 133   elco_cfg="${elco_cfg}_ELCO.cfg"
 134   cp $1 "${elco_cfg}"
 135   cmd='
 136 s/^\*\.Emergency/#\*\.Emergency/g'
 137   sed -e "$cmd" < "${elco_cfg}" > tmp
 138   mv tmp "${elco_cfg}"
 139   debug "OLD  cfg name: $1"
 140   debug "ELCO cfg name: ${elco_cfg}"
 141 }
 142 #####################################
 143 # compare_with_ELCO
 144 # $1: componenent id (mtc, hc, 3, 4 etc)
 145 #####################################
 146 compare_with_ELCO() {
 147  diff -u "$LOGDIR/${elco_cfg_base}-$1.log" "$LOGDIR/${orig_cfg_base}-$1.log"  > "${LOGDIR}/diff_${orig_cfg_base}_$1.log"
 148  if [ "$?" -eq $SUCCESS ]
 149  then
 150    debug ">>>tc $orig_cfg_base $1 part success<<<"
 151    let "success = $success + 1"
 152  else
 153    debug ">>>tc $orig_cfg_base $1 part failed<<<"
 154    echo "tc $orig_cfg_base $1 part failed" >> ${LIST_OF_FAILED}
 155    let "failed = $failed + 1"
 156  fi
 157  debug "success: ${success} failed: ${failed}"
 158 }
 159
 160 #####################################
 161 # compare_with_expected
 162 # $1: componenent id (mtc, hc, 3, 4 etc)
 163 # $2: expected log
 164 #####################################
 165 compare_with_expected() {
 166  diff -u "$2" "${LOGDIR}/${orig_cfg_base}-$1.log" > "${LOGDIR}/diff_${orig_cfg_base}_$1.log"
 167  if [ "$?" -eq $SUCCESS ]
 168  then
 169    debug ">>>tc $orig_cfg_base $1 part success<<<"
 170    let "success = $success + 1"
 171  else
 172    debug ">>>tc $orig_cfg_base $1 part failed<<<"
 173    echo "tc $orig_cfg_base $1 part failed" >> ${LIST_OF_FAILED}
 174    let "failed = $failed + 1"
 175  fi
 176  debug "success: ${success} failed: ${failed}"
 177 }
 178
 179 #####################################
 180 # compare_with_expected_emergency
 181 # $1: componenent id (mtc, hc, 3, 4 etc)
 182 # $2: expected log
 183 #####################################
 184 compare_with_expected_emergency() {
 185  #debug "==>diff -u $4 ${LOGDIR}/${orig_cfg_base}-$1.log_emergency  > ${LOGDIR}/diff_${orig_cfg_base}_$1.log_emergency"
 186  diff -u "$2" "${LOGDIR}/${orig_cfg_base}-$1.log_emergency"  > "${LOGDIR}/diff_${orig_cfg_base}_$1.log_emergency"
 187  if [ "$?" -eq $SUCCESS ]
 188  then
 189    debug ">>>tc $orig_cfg_base $1 part success<<<"
 190    let "success = $success + 1"
 191  else
 192    debug ">>>tc $orig_cfg_base $1 part failed<<<"
 193    echo "tc $orig_cfg_base $1 emergency part failed" >> ${LIST_OF_FAILED}
 194    let "failed = $failed + 1"
 195  fi
 196  debug "success: ${success} failed: ${failed}"
 197 }
 198 #####################################
 199 # run_and_compare_log_files
 200 # $1 first config file
 201 # $2 2nd config file
 202 # $3 output file (diff_)
 203 #####################################
 204 run_and_compare_log_files() {
 205  #running with orig cfg:
 206  rm  ${LOGDIR}/${MYEXE}*.log
 207  orig_cfg_base=`basename $1 ".cfg"`
 208  run_and_modify_logfile "$1" "$orig_cfg_base"
 209
 210  if [ "$2" == "" ]; then
 211    #running with modified cfg:
 212    create_ELCO_config "$1"
 213  else
 214    elco_cfg=$2
 215  fi
 216
 217  elco_cfg_base=`basename "$elco_cfg" ".cfg"`
 218  debug "ELCO cfg name: ${elco_cfg}"
 219  run_and_modify_logfile "${elco_cfg}"  "$elco_cfg_base"
 220
 221  #diff:
 222  compare_with_ELCO "mtc"
 223  compare_with_ELCO "hc"
 224
 225  i=3
 226  while [ -e "$LOGDIR/${elco_cfg_base}-$i.log" -a -e "$LOGDIR/${orig_cfg_base}-$i.log" ]
 227  do
 228    compare_with_ELCO "$i"
 229    let "i = $i + 1"
 230  done
 231
 232 }
 233 #####################################
 234 # run_and_compare_log_file_with_expected
 235 # $1 first config file
 236 # $2 expected mtc log file
 237 # $3 expected hc log file
 238 # $4 expected emergency log file
 239 #####################################
 240 run_and_compare_log_file_with_expected() {
 241   rm -f ${LOGDIR}/${MYEXE}*.log*
 242   #running with orig cfg:
 243   orig_cfg_base=`basename $1 ".cfg"`
 244   run_and_modify_logfile "$1" "$orig_cfg_base"
 245
 246
 247   debug "cfg:  $1"
 248   debug "mtclog:  $2"
 249   debug "hclog: $3"
 250   #diff mtc with the expected:
 251   if [ -e "$2" ]; then
 252      modify_logfile "$2"
 253      compare_with_expected "mtc" "$2"
 254   else
 255     debug ">>>tc $orig_cfg_base mtc part failed, not existing expected log file $2<<<"
 256     echo "tc $orig_cfg_base mtc part failed" >> ${LIST_OF_FAILED}
 257     let "failed = $failed + 1"
 258   fi
 259
 260   #diff:
 261
 262   if [ "$3" != "" ]; then
 263    if [ -e "$3" ]; then
 264       modify_logfile "$3"
 265       compare_with_expected "hc" "$3"
 266    else
 267      debug ">>>tc $orig_cfg_base mtc part failed, not existing expected log file $3<<<"
 268      echo "tc $orig_cfg_base mtc part failed" >> ${LIST_OF_FAILED}
 269      let "failed = $failed + 1"
 270    fi
 271   fi
 272
 273   #mtc emergency log kezeles
 274   if [ "$4" != "" ]; then
 275    if [ -e "$4" ]; then
 276       modify_logfile "$4"
 277       compare_with_expected_emergency "mtc" "$4"
 278    else
 279      debug ">>>tc $orig_cfg_base mtc part failed, not existing expected log file $4<<<"
 280      echo "tc $orig_cfg_base mtc part failed, not existing expected log file $4" >> ${LIST_OF_FAILED}
 281      let "failed = $failed + 1"
 282    fi
 283   fi
 284 }
 285
 286 #####################################
 287 # evaluate
 288 #####################################
 289 evaluate() {
 290   echo "Summary: success: ${success}, failed: ${failed} "
 291   if [ $failed -eq 0 ]
 292   then
 293    echo "Overall verdict: success"
 294    exit_code=0
 295   else
 296    echo "Overall verdict: failed"
 297    cat ${LIST_OF_FAILED} |
 298    while read  line
 299    do
 300     echo $line
 301    done
 302    exit_code=1
 303   fi
 304
 305   exit $exit_code
 306 }
 307
 308 #### MAIN ###
 309
 310 # see the files "logs/diff*.log" !
 311
 312 init
 313 #======= Buffer All ========
 314 run_and_compare_log_files EL_BufferAll_1.cfg  #compatibility test with minimal coverage (compare logs with/without EL setting if EL-event does not happen
 315 run_and_compare_log_files EL_BufferAll_2.cfg  #compatibility test with coverage "tc_parallel_portconn" (compare logs with/without EL setting if EL-event does not happen) - NOT STABILE, hc FAILED
 316 run_and_compare_log_files EL_BufferAll_3.cfg EL_BufferAll_3_NOEL.cfg #the same log expected with two different log files: EL and NOEL, EL-event occurs in the first case - PASSED
 317 run_and_compare_log_files EL_BufferAll_4.cfg  #compatibility test with "tc_timer" (compare logs with/without EL setting if EL-event does not happen) - hc FAILED
 318 run_and_compare_log_files EL_BufferAll_5.cfg  #compatibility test "with Titan_LogTest.tc_encdec" - NOT STABILE, mtc PASSED, hc PASSED
 319 run_and_compare_log_files EL_BufferAll_6.cfg  #compatibility test with "tc_function_rnd and tc_encdec" - mtc PASSED, hc PASSED
 320 run_and_compare_log_file_with_expected "EL_BufferAll_7.cfg"  "EL_BufferAll_7_mtc_expected.log"  # - mtc PASSED
 321 run_and_compare_log_file_with_expected "EL_BufferAll_7A.cfg"  "EL_BufferAll_7A_mtc_expected.log"  # - mtc PASSED
 322 run_and_compare_log_files EL_BufferAll_8.cfg   #compatibility test with MAXIMAL coverage - NOT STABILE; mtc PASSED hc FAILED (1 line order problem,
 323 run_and_compare_log_file_with_expected "EL_BufferAll_9.cfg"  "EL_BufferAll_9_mtc_expected.log"  # - More EL-event can be logged after each other - mtc FAILED ( 1 line missing), hc FAILED
 324 run_and_compare_log_file_with_expected "EL_BufferAll_10.cfg"  "EL_BufferAll_10_mtc_expected.log"  # - More EL-event with big buffer - mtc PASSED
 325 run_and_compare_log_file_with_expected "EL_BufferAll_11.cfg"  "EL_BufferAll_11_mtc_expected.log"  # - More EL-event with big buffer - mtc PASSED
 326 run_and_compare_log_file_with_expected "EL_BufferAll_12.cfg"  "EL_BufferAll_12_mtc_expected.log"  # - More EL-event with very small buffer - (buffer size=2) - mtc FAILED, one line missing
 327 run_and_compare_log_file_with_expected "EL_BufferAll_13.cfg"  "EL_BufferAll_13_mtc_expected.log"  # - More EL-event with very small buffer - (buffer size=3) - mtc FAILED, one line missing
 328 #======= Buffer Masked ========
 329 run_and_compare_log_files EL_BufferMasked_1.cfg  #compatibility test with minimal coverage (compare logs with/without EL setting if EL-event does not happen
 330 run_and_compare_log_files EL_BufferMasked_2.cfg  #compatibility test with coverage "tc_parallel_portconn" (compare logs with/without EL setting if EL-event does not happen) - NOT STABILE
 331 run_and_compare_log_files EL_BufferMasked_4.cfg  #compatibility test with "tc_timer" (compare logs with/without EL setting if EL-event does not happen) - PASSED
 332 run_and_compare_log_files EL_BufferMasked_5.cfg  #compatibility test "with Titan_LogTest.tc_encdec" - NOT STABILE
 333 run_and_compare_log_files EL_BufferMasked_3.cfg EL_BufferMasked_3_NOEL.cfg #the same log expected with two different log files: EL and NOEL, EL-event occurs in the first case - PASSED
 334 run_and_compare_log_files EL_BufferMasked_6.cfg  #compatibility test with "tc_function_rnd and tc_encdec" - PASSED
 335 run_and_compare_log_file_with_expected "EL_BufferMasked_7.cfg"  "EL_BufferMasked_7_mtc_expected.log" "" "EL_BufferMasked_7_mtc_expected.log_emergency" # - PASSED
 336 run_and_compare_log_files EL_BufferMasked_8.cfg  #compatibility test with MAXIMAL coverage - NOT STABILE
 337 run_and_compare_log_file_with_expected "EL_BufferMasked_9.cfg"  "EL_BufferMasked_9_mtc_expected.log" ""  "EL_BufferMasked_9_mtc_expected.log_emergency" # - More EL-event can be logged after each other - PASSED
 338 run_and_compare_log_file_with_expected "EL_BufferMasked_10.cfg"  "EL_BufferMasked_10_mtc_expected.log" ""  "EL_BufferMasked_10_mtc_expected.log_emergency" # - More EL-event with big buffer - PASSED
 339 run_and_compare_log_file_with_expected "EL_BufferMasked_11.cfg"  "EL_BufferMasked_11_mtc_expected.log"  ""  "EL_BufferMasked_11_mtc_expected.log_emergency" # - More EL-event with big buffer - PASSED
 340 run_and_compare_log_file_with_expected "EL_BufferMasked_12.cfg"  "EL_BufferMasked_12_mtc_expected.log"  "" "EL_BufferMasked_12_mtc_expected.log_emergency" # - More EL-event with very small buffer - (buffer size=2) - mtc.log_emergency FAILED (wrong order)
 341 run_and_compare_log_file_with_expected "EL_BufferMasked_13.cfg"  "EL_BufferMasked_13_mtc_expected.log"  "" "EL_BufferMasked_13_mtc_expected.log_emergency" # - More EL-event with very small buffer - (buffer size=3) - mtc.log_emergency FAILED (wrong order)
 342 #Stat: 63 success/ 0 failed
 343 evaluate