Sync with 5.4.0

[deliverable/titan.core.git] / regression_test / profiler / Testcases.ttcn

/******************************************************************************
 * Copyright (c) 2000-2015 Ericsson Telecom AB
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 ******************************************************************************/

module Testcases {

import from Shell all;

type record of charstring CharstringList;

function f_compare_output_files(in CharstringList p_output_files,
  in CharstringList p_exp_output_files) runs on Shell_CT
{
  var integer i;
  for (i := 0; i < sizeof(p_output_files); i := i + 1) {
    f_compareFiles(p_output_files[i], p_exp_output_files[i], 0);
    if (getverdict != pass) {
      action("Output file '" & p_output_files[i] & "' does not match the expected file '" &
        p_exp_output_files[i] & "'");
    }
  }
}

function f_test_profiler(in charstring p_target_file, in charstring p_config_file,
  in CharstringList p_output_files, in CharstringList p_exp_output_files) runs on Shell_CT
{
  // make the target (the makefile must have a rule for it)
  f_shellCommandWithVerdict("make " & p_target_file, "", c_shell_successWithoutError);

  if (getverdict == pass) {
    // run the executable with the specified configuration file
    f_shellCommandWithVerdict("ttcn3_start " & p_target_file & " " & p_config_file, "",
      c_shell_successWithoutWarningAndError);
    
    if (getverdict == pass) {
      f_compare_output_files(p_output_files, p_exp_output_files);
    }
    else {
      action("Failed to run target '", p_target_file, "', with configuration file '", p_config_file, "'");
    }
  }
  else {
    action("Failed to make target '", p_target_file, "'");
  }
}

function f_test_profmerge(in charstring p_arguments, in CharstringList p_output_files,
  in CharstringList p_exp_output_files) runs on Shell_CT
{
  // run the profmerge command
  f_shellCommandWithVerdict("ttcn3_profmerge " & p_arguments, "",
    c_shell_successWithoutWarningAndError);
  if (getverdict == pass) {
    f_compare_output_files(p_output_files, p_exp_output_files);
  }
  else {
    action("Failed to run profmerge with arguments '", p_arguments, "'");
  }
}

testcase tc_coverage() runs on Shell_CT
{
  // only code coverage is done in this case, since that is exact, and the results can be checked with
  // a simple file comparison
  f_test_profiler("prof.exe", "prof1.cfg", { "data.json", "prof1.stats" }, { "data_e.json", "prof1_e.stats" } );
}

testcase tc_profiling() runs on Shell_CT
{
  // only profiling is done in this case
  // the results cannot be checked, since the call times will vary in each run
  f_test_profiler("prof.exe", "prof2.cfg", { }, { } );
}

testcase tc_profiling_and_coverage() runs on Shell_CT
{
  // both profiling and code coverage is activated in this case
  // although the call times cannot be checked, the statistics filter is set to generate an 
  // empty statistics file, which can be cheked
  f_test_profiler("prof.exe", "prof3.cfg", { "empty.stats" }, { "empty_e.stats" } );
}

testcase tc_profmerge_double() runs on Shell_CT
{
  // the database file of the coverage-only test is merged with itself (values are doubled)
  // the results can be compared, since they only contain coverage data and zero times
  // all average time related statistics are filtered out (since they're all zeros),
  // as well as all sorted statistics (since those are platform dependent)
  f_test_profmerge("-s double.stats -o double.json -f 1800007 data.json data.json", 
    { "double.json", "double.stats" }, { "double_e.json", "double_e.stats" } );
}

testcase tc_profmerge_all() runs on Shell_CT
{
  // the database files of the 3 profiler tests are merged (values are again doubled)
  // profiler data is discarded (-p), so the results can be compared
  // (the output database is the same as the previous test, the statistics file is not,
  // since it contains no time data instead of zero times)
  f_test_profmerge("-p -o merged.json -s merged.stats -f 1800007 data.json data2.json data3.json",
    { "merged.json", "merged.stats" }, { "double_e.json", "merged_e.stats" } );
}

control {
  // the profmerge tests require the output files generated by the profiler tests
  var verdicttype vt1 := execute(tc_coverage());
  var verdicttype vt2 := execute(tc_profiling());
  var verdicttype vt3 := execute(tc_profiling_and_coverage());
  if (vt1 == pass) {
    execute(tc_profmerge_double());
  }
  if (vt1 == pass and vt2 == pass and vt3 == pass) {
    execute(tc_profmerge_all());
  }
}

}