TMF: Fix trace synchronization memory performance test, to reduce variability

And document a bit more how to use the USED_JAVA_HEAP metric, so that it gives
accurate results.

Change-Id: Id776ab4cbd70f1f22bcfa9fc5efe95b97b6c4b77
Signed-off-by: Geneviève Bastien <gbastien+lttng@versatic.net>
Reviewed-on: https://git.eclipse.org/r/30999
Tested-by: Hudson CI
Reviewed-by: Bernd Hufmann <bernd.hufmann@ericsson.com>
Tested-by: Bernd Hufmann <bernd.hufmann@ericsson.com>

diff --git a/org.eclipse.linuxtools.lttng2.kernel.core.tests/perf/org/eclipse/linuxtools/lttng2/kernel/core/tests/perf/event/matching/TraceSynchronizationBenchmark.java b/org.eclipse.linuxtools.lttng2.kernel.core.tests/perf/org/eclipse/linuxtools/lttng2/kernel/core/tests/perf/event/matching/TraceSynchronizationBenchmark.java
index 5fe6fd25d415e0f907de04eb634f0056454ea4d1..968ccb8da49c1d5252d859478ea5adc55dd9962d 100644 (file)
--- a/org.eclipse.linuxtools.lttng2.kernel.core.tests/perf/org/eclipse/linuxtools/lttng2/kernel/core/tests/perf/event/matching/TraceSynchronizationBenchmark.java
+++ b/org.eclipse.linuxtools.lttng2.kernel.core.tests/perf/org/eclipse/linuxtools/lttng2/kernel/core/tests/perf/event/matching/TraceSynchronizationBenchmark.java
@@ -12,16 +12,17 @@
 
 package org.eclipse.linuxtools.lttng2.kernel.core.tests.perf.event.matching;
 
-import static org.junit.Assert.fail;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assume.assumeTrue;
 
+import java.util.Arrays;
+
 import org.eclipse.linuxtools.lttng2.kernel.core.event.matching.TcpEventMatching;
 import org.eclipse.linuxtools.lttng2.kernel.core.event.matching.TcpLttngEventMatching;
 import org.eclipse.linuxtools.tmf.core.event.matching.TmfEventMatching;
-import org.eclipse.linuxtools.tmf.core.exceptions.TmfTraceException;
+import org.eclipse.linuxtools.tmf.core.synchronization.SynchronizationAlgorithm;
+import org.eclipse.linuxtools.tmf.core.synchronization.SynchronizationManager;
 import org.eclipse.linuxtools.tmf.core.trace.ITmfTrace;
-import org.eclipse.linuxtools.tmf.core.trace.TmfExperiment;
-import org.eclipse.linuxtools.tmf.ctf.core.CtfTmfEvent;
 import org.eclipse.linuxtools.tmf.ctf.core.CtfTmfTrace;
 import org.eclipse.linuxtools.tmf.ctf.core.tests.shared.CtfTmfTestTrace;
 import org.eclipse.test.performance.Dimension;
@@ -42,7 +43,6 @@ public class TraceSynchronizationBenchmark {
     private static final String TIME = " (time)";
     private static final String MEMORY = " (memory usage)";
     private static final String TEST_SUMMARY = "Trace synchronization";
-    private static int BLOCK_SIZE = 1000;
 
     /**
      * Initialize some data
@@ -94,16 +94,9 @@ public class TraceSynchronizationBenchmark {
         perf.tagAsSummary(pm, TEST_SUMMARY + ':' + testName + TIME, Dimension.CPU_TIME);
 
         for (int i = 0; i < loop_count; i++) {
-            TmfExperiment experiment = new TmfExperiment(CtfTmfEvent.class, "Test experiment", testTraces, BLOCK_SIZE);
-
             pm.start();
-            try {
-                experiment.synchronizeTraces(true);
-            } catch (TmfTraceException e) {
-                fail("Failed at iteration " + i + " with message: " + e.getMessage());
-            }
+            SynchronizationManager.synchronizeTraces(null, Arrays.asList(testTraces), true);
             pm.stop();
-
         }
         pm.commit();
 
@@ -116,18 +109,14 @@ public class TraceSynchronizationBenchmark {
         perf.tagAsSummary(pm, TEST_SUMMARY + ':' + testName + MEMORY, Dimension.USED_JAVA_HEAP);
 
         for (int i = 0; i < loop_count; i++) {
-            TmfExperiment experiment = new TmfExperiment(CtfTmfEvent.class, "Test experiment", testTraces, BLOCK_SIZE);
 
             System.gc();
             pm.start();
-            try {
-                experiment.synchronizeTraces(true);
-            } catch (TmfTraceException e) {
-                fail("Failed at iteration " + i + " with message: " + e.getMessage());
-            }
+            SynchronizationAlgorithm algo = SynchronizationManager.synchronizeTraces(null, Arrays.asList(testTraces), true);
+            assertNotNull(algo);
+
             System.gc();
             pm.stop();
-
         }
         pm.commit();
     }

diff --git a/org.eclipse.linuxtools.tmf.help/doc/Developer-Guide.mediawiki b/org.eclipse.linuxtools.tmf.help/doc/Developer-Guide.mediawiki
index 6b975200d71e7e9a3ae557ef576f7015c791318e..217f8ca0aa8063dbaff0b7cce17a1fc51d5007fa 100644 (file)
--- a/org.eclipse.linuxtools.tmf.help/doc/Developer-Guide.mediawiki
+++ b/org.eclipse.linuxtools.tmf.help/doc/Developer-Guide.mediawiki
@@ -3670,4 +3670,4 @@ System time (Elapsed time): The time between the start and the end of the execut
 
 Kernel time: Time spent in kernel mode
 
-Used Java Heap: It is the difference between the memory used at the beginning of the execution and at the end. This metric may be useful to calculate the overall size occupied by the data generated by the test run, by forcing a garbage collection before taking the metrics at the beginning and at the end of the execution. But it will not show the memory used throughout the execution.
+Used Java Heap: It is the difference between the memory used at the beginning of the execution and at the end. This metric may be useful to calculate the overall size occupied by the data generated by the test run, by forcing a garbage collection before taking the metrics at the beginning and at the end of the execution. But it will not show the memory used throughout the execution. There can be a large standard deviation. The reason for this is  that when benchmarking methods that trigger tasks in different threads, like signals and/or analysis, these other threads might be in various states at each run of the test, which will impact the memory usage calculated. When using this metric, either make sure the method to test does not trigger external threads or make sure you wait for them to finish.