org.eclipse.tracecompass.ctf.core.tests/src/org/eclipse/tracecompass/ctf/core/tests/ctftestsuite/CtfTestSuiteTests.java

   1 /*******************************************************************************
   2  * Copyright (c) 2013, 2014 Ericsson
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *   Alexandre Montplaisir - Initial API and implementation
  11  *******************************************************************************/
  12
  13 package org.eclipse.tracecompass.ctf.core.tests.ctftestsuite;
  14
  15 import static org.junit.Assert.assertNotNull;
  16 import static org.junit.Assert.fail;
  17
  18 import java.io.IOException;
  19 import java.nio.file.DirectoryStream;
  20 import java.nio.file.Files;
  21 import java.nio.file.Path;
  22 import java.nio.file.Paths;
  23 import java.util.LinkedList;
  24 import java.util.List;
  25
  26 import org.eclipse.tracecompass.ctf.core.trace.CTFReaderException;
  27 import org.eclipse.tracecompass.ctf.core.trace.CTFTrace;
  28 import org.eclipse.tracecompass.ctf.core.trace.CTFTraceReader;
  29 import org.junit.Rule;
  30 import org.junit.Test;
  31 import org.junit.rules.TestRule;
  32 import org.junit.rules.Timeout;
  33 import org.junit.runner.RunWith;
  34 import org.junit.runners.Parameterized;
  35 import org.junit.runners.Parameterized.Parameters;
  36
  37 /**
  38  * Parameterized test class running the CTF Test Suite
  39  *
  40  * (from https://github.com/efficios/ctf-testsuite).
  41  *
  42  * @author Alexandre Montplaisir
  43  */
  44 @RunWith(Parameterized.class)
  45 public class CtfTestSuiteTests {
  46
  47     /** Time-out tests after 10 seconds. */
  48     @Rule
  49     public TestRule globalTimeout = new Timeout(10000);
  50
  51     private static final Path BASE_PATH = Paths.get("traces", "ctf-testsuite", "tests", "1.8");
  52
  53     /**
  54      * Test we know are currently failing. Ignore them so we can at least run
  55      * the others.
  56      *
  57      * TODO Actually fix them!
  58      */
  59     private static final Path[] IGNORED_TESTS = {
  60             BASE_PATH.resolve(Paths.get("regression", "metadata", "pass", "sequence-typedef-length")),
  61             BASE_PATH.resolve(Paths.get("regression", "metadata", "pass", "array-of-struct"))
  62     };
  63
  64     private final String fTracePath;
  65     private final boolean fExpectSuccess;
  66
  67     // ------------------------------------------------------------------------
  68     // Methods for the Parametrized runner
  69     // ------------------------------------------------------------------------
  70
  71     /**
  72      * Get the existing trace paths in the CTF-Testsuite git tree.
  73      *
  74      * @return The list of CTF traces (directories) to test
  75      */
  76     @Parameters(name = "{index}: {0}")
  77     public static Iterable<Object[]> getTracePaths() {
  78         final List<Object[]> dirs = new LinkedList<>();
  79
  80         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("fuzzing", "metadata", "fail")), false);
  81         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("fuzzing", "metadata", "pass")), true);
  82         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("fuzzing", "stream", "fail")), false);
  83         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("fuzzing", "stream", "pass")), true);
  84
  85         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("regression", "metadata", "fail")), false);
  86         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("regression", "metadata", "pass")), true);
  87         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("regression", "stream", "fail")), false);
  88         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("regression", "stream", "pass")), true);
  89
  90         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("stress", "metadata", "fail")), false);
  91         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("stress", "metadata", "pass")), true);
  92         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("stress", "stream", "fail")), false);
  93
  94         //FIXME Layout of the following directory is now different
  95         //addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("stress", "stream", "pass")), true);
  96
  97         return dirs;
  98     }
  99
 100     private static void addDirsFrom(List<Object[]> dirs, Path path, boolean expectSuccess) {
 101         if (!Files.exists(path)) {
 102             /* Some planned directories may not exist yet in the test suite */
 103             return;
 104         }
 105         try (DirectoryStream<Path> ds = Files.newDirectoryStream(path, DIR_FILTER);) {
 106             for (Path p : ds) {
 107                 /* Add this test case to the list of tests to run */
 108                 Object array[] = new Object[] { p.toString(), expectSuccess };
 109                 dirs.add(array);
 110             }
 111         } catch (IOException e) {
 112             /* Something is wrong with the layout of the test suite? */
 113             e.printStackTrace();
 114         }
 115     }
 116
 117     private static final DirectoryStream.Filter<Path> DIR_FILTER =
 118             new DirectoryStream.Filter<Path>() {
 119                 @Override
 120                 public boolean accept(Path entry) {
 121                     /* Only accept directories and non-blacklisted tests */
 122                     if (!Files.isDirectory(entry)) {
 123                         return false;
 124                     }
 125                     for (Path ignoredTestPath : IGNORED_TESTS) {
 126                         if (entry.equals(ignoredTestPath)) {
 127                             System.err.println("Skipping test " + entry.toString() + " as requested.");
 128                             return false;
 129                         }
 130                     }
 131                     return true;
 132                 }
 133             };
 134
 135     // ------------------------------------------------------------------------
 136     // Test constructor
 137     // ------------------------------------------------------------------------
 138
 139     /**
 140      * Constructor for the parametrized tests
 141      *
 142      * @param tracePath
 143      *            The complete path to the trace to test
 144      * @param expectSuccess
 145      *            Should this trace parse successfully, or not.
 146      */
 147     public CtfTestSuiteTests(String tracePath, boolean expectSuccess) {
 148         fTracePath = tracePath;
 149         fExpectSuccess = expectSuccess;
 150     }
 151
 152     // ------------------------------------------------------------------------
 153     // Test methods
 154     // ------------------------------------------------------------------------
 155
 156     /**
 157      * Test opening and reading the trace
 158      */
 159     @Test
 160     public void testTrace() {
 161         try (/* Instantiate the trace (which implies parsing the metadata) */
 162                 CTFTrace trace = new CTFTrace(fTracePath);
 163                 /* Read the trace until the end */
 164                 CTFTraceReader reader = new CTFTraceReader(trace);) {
 165
 166             reader.getCurrentEventDef();
 167             while (reader.advance()) {
 168                 assertNotNull(reader.getCurrentEventDef());
 169             }
 170
 171             checkIfWeShoudlSucceed();
 172         } catch (CTFReaderException e) {
 173             checkIfWeShouldFail(e);
 174         } catch (OutOfMemoryError e) {
 175             checkIfWeShouldFail(e);
 176         }
 177     }
 178
 179     private void checkIfWeShoudlSucceed() {
 180         if (!fExpectSuccess) {
 181             fail("Trace was expected to fail parsing: " + fTracePath);
 182         }
 183     }
 184
 185     private void checkIfWeShouldFail(Throwable e) {
 186         if (fExpectSuccess) {
 187             fail("Trace was expected to succeed, but failed parsing: " +
 188                     fTracePath + " (" + e.getMessage() + ")");
 189         }
 190     }
 191 }