From 4d2a4a2cef3360c2b2436b365088d0536f43978c Mon Sep 17 00:00:00 2001 From: =?utf8?q?Genevi=C3=A8ve=20Bastien?= Date: Wed, 4 Jun 2014 09:55:08 -0400 Subject: [PATCH 1/1] TMF: Synchronize an experiment instead of collection of traces MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit That will make it compatible with streaming since one full trace does not have to be read before the others. It will also accelerate the time necessary to get results, as matches will be found earlier. The overhead of reading the experiment instead of the traces decreases performance though. Change-Id: I7c94c21e63781b2cae0faead915be8959170a992 Signed-off-by: Geneviève Bastien Reviewed-on: https://git.eclipse.org/r/35350 Tested-by: Hudson CI Reviewed-by: Alexandre Montplaisir --- .../TraceSynchronizationBenchmark.java | 20 +++++--- .../event/matchandsync/MatchAndSyncTest.java | 44 ++++++++++++---- .../core/event/matching/TmfEventMatches.java | 1 - .../core/event/matching/TmfEventMatching.java | 50 ++++++++++++------- .../matching/TmfNetworkEventMatching.java | 8 +-- .../core/trace/experiment/TmfExperiment.java | 43 ++++++++++------ 6 files changed, 112 insertions(+), 54 deletions(-) diff --git a/org.eclipse.tracecompass.lttng2.kernel.core.tests/perf/org/eclipse/tracecompass/lttng2/kernel/core/tests/perf/event/matching/TraceSynchronizationBenchmark.java b/org.eclipse.tracecompass.lttng2.kernel.core.tests/perf/org/eclipse/tracecompass/lttng2/kernel/core/tests/perf/event/matching/TraceSynchronizationBenchmark.java index 61e6bb7f5a..2e124b1fa5 100644 --- a/org.eclipse.tracecompass.lttng2.kernel.core.tests/perf/org/eclipse/tracecompass/lttng2/kernel/core/tests/perf/event/matching/TraceSynchronizationBenchmark.java +++ b/org.eclipse.tracecompass.lttng2.kernel.core.tests/perf/org/eclipse/tracecompass/lttng2/kernel/core/tests/perf/event/matching/TraceSynchronizationBenchmark.java @@ -15,7 +15,7 @@ package org.eclipse.tracecompass.lttng2.kernel.core.tests.perf.event.matching; import static org.junit.Assert.assertNotNull; import static org.junit.Assume.assumeTrue; -import java.util.Arrays; +import java.util.Collections; import org.eclipse.test.performance.Dimension; import org.eclipse.test.performance.Performance; @@ -26,6 +26,8 @@ import org.eclipse.tracecompass.tmf.core.event.matching.TmfEventMatching; import org.eclipse.tracecompass.tmf.core.synchronization.SynchronizationAlgorithm; import org.eclipse.tracecompass.tmf.core.synchronization.SynchronizationManager; import org.eclipse.tracecompass.tmf.core.trace.ITmfTrace; +import org.eclipse.tracecompass.tmf.core.trace.experiment.TmfExperiment; +import org.eclipse.tracecompass.tmf.ctf.core.event.CtfTmfEvent; import org.eclipse.tracecompass.tmf.ctf.core.tests.shared.CtfTmfTestTrace; import org.eclipse.tracecompass.tmf.ctf.core.trace.CtfTmfTrace; import org.junit.BeforeClass; @@ -62,7 +64,8 @@ public class TraceSynchronizationBenchmark { try (CtfTmfTrace trace1 = CtfTmfTestTrace.SYNC_SRC.getTrace(); CtfTmfTrace trace2 = CtfTmfTestTrace.SYNC_DEST.getTrace();) { ITmfTrace[] traces = { trace1, trace2 }; - runCpuTest(traces, "Match TCP events", 40); + TmfExperiment experiment = new TmfExperiment(CtfTmfEvent.class, "Test experiment", traces, TmfExperiment.DEFAULT_INDEX_PAGE_SIZE, null); + runCpuTest(experiment, "Match TCP events", 40); } } @@ -78,19 +81,20 @@ public class TraceSynchronizationBenchmark { CtfTmfTrace trace2 = CtfTmfTestTrace.DJANGO_DB.getTrace(); CtfTmfTrace trace3 = CtfTmfTestTrace.DJANGO_HTTPD.getTrace();) { ITmfTrace[] traces = { trace1, trace2, trace3 }; - runCpuTest(traces, "Django traces", 10); - runMemoryTest(traces, "Django traces", 10); + TmfExperiment experiment = new TmfExperiment(CtfTmfEvent.class, "Test experiment", traces, TmfExperiment.DEFAULT_INDEX_PAGE_SIZE, null); + runCpuTest(experiment, "Django traces", 10); + runMemoryTest(experiment, "Django traces", 10); } } - private static void runCpuTest(ITmfTrace[] testTraces, String testName, int loop_count) { + private static void runCpuTest(TmfExperiment experiment, String testName, int loop_count) { Performance perf = Performance.getDefault(); PerformanceMeter pm = perf.createPerformanceMeter(TEST_ID + testName + TIME); perf.tagAsSummary(pm, TEST_SUMMARY + ':' + testName + TIME, Dimension.CPU_TIME); for (int i = 0; i < loop_count; i++) { pm.start(); - SynchronizationManager.synchronizeTraces(null, Arrays.asList(testTraces), true); + SynchronizationManager.synchronizeTraces(null, Collections. singleton(experiment), true); pm.stop(); } pm.commit(); @@ -98,7 +102,7 @@ public class TraceSynchronizationBenchmark { } /* Benchmark memory used by the algorithm */ - private static void runMemoryTest(ITmfTrace[] testTraces, String testName, int loop_count) { + private static void runMemoryTest(TmfExperiment experiment, String testName, int loop_count) { Performance perf = Performance.getDefault(); PerformanceMeter pm = perf.createPerformanceMeter(TEST_ID + testName + MEMORY); perf.tagAsSummary(pm, TEST_SUMMARY + ':' + testName + MEMORY, Dimension.USED_JAVA_HEAP); @@ -107,7 +111,7 @@ public class TraceSynchronizationBenchmark { System.gc(); pm.start(); - SynchronizationAlgorithm algo = SynchronizationManager.synchronizeTraces(null, Arrays.asList(testTraces), true); + SynchronizationAlgorithm algo = SynchronizationManager.synchronizeTraces(null, Collections. singleton(experiment), true); assertNotNull(algo); System.gc(); diff --git a/org.eclipse.tracecompass.lttng2.kernel.core.tests/src/org/eclipse/tracecompass/lttng2/kernel/core/tests/event/matchandsync/MatchAndSyncTest.java b/org.eclipse.tracecompass.lttng2.kernel.core.tests/src/org/eclipse/tracecompass/lttng2/kernel/core/tests/event/matchandsync/MatchAndSyncTest.java index c6bdc822ac..6caf09cbf6 100644 --- a/org.eclipse.tracecompass.lttng2.kernel.core.tests/src/org/eclipse/tracecompass/lttng2/kernel/core/tests/event/matchandsync/MatchAndSyncTest.java +++ b/org.eclipse.tracecompass.lttng2.kernel.core.tests/src/org/eclipse/tracecompass/lttng2/kernel/core/tests/event/matchandsync/MatchAndSyncTest.java @@ -14,13 +14,19 @@ package org.eclipse.tracecompass.lttng2.kernel.core.tests.event.matchandsync; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.junit.Assume.assumeTrue; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.LinkedList; import java.util.List; +import java.util.Map; import org.eclipse.tracecompass.lttng2.kernel.core.event.matching.TcpEventMatching; import org.eclipse.tracecompass.lttng2.kernel.core.event.matching.TcpLttngEventMatching; +import org.eclipse.tracecompass.tmf.core.event.matching.IMatchProcessingUnit; import org.eclipse.tracecompass.tmf.core.event.matching.TmfEventMatching; import org.eclipse.tracecompass.tmf.core.event.matching.TmfNetworkEventMatching; import org.eclipse.tracecompass.tmf.core.trace.ITmfTrace; @@ -41,7 +47,6 @@ public class MatchAndSyncTest { */ @Test public void testMatching() { - final String cr = System.getProperty("line.separator"); assumeTrue(CtfTmfTestTrace.SYNC_SRC.exists()); assumeTrue(CtfTmfTestTrace.SYNC_DEST.exists()); try (CtfTmfTrace trace1 = CtfTmfTestTrace.SYNC_SRC.getTrace(); @@ -57,14 +62,35 @@ public class MatchAndSyncTest { TmfNetworkEventMatching twoTraceMatch = new TmfNetworkEventMatching(tracearr); assertTrue(twoTraceMatch.matchEvents()); - String stats = twoTraceMatch.toString(); - assertEquals("TmfEventMatches [ Number of matches found: 46 ]" + - "Trace 0:" + cr + - " 3 unmatched incoming events" + cr + - " 2 unmatched outgoing events" + cr + - "Trace 1:" + cr + - " 2 unmatched incoming events" + cr + - " 1 unmatched outgoing events" + cr, stats); + /* Set method and fields accessible to make sure the counts are ok */ + try { + /* Verify number of matches */ + Method method = TmfEventMatching.class.getDeclaredMethod("getProcessingUnit"); + method.setAccessible(true); + IMatchProcessingUnit procUnit = (IMatchProcessingUnit) method.invoke(twoTraceMatch); + assertEquals(46, procUnit.countMatches()); + + /* Verify unmatched incoming */ + Field fieldIn = twoTraceMatch.getClass().getDeclaredField("fUnmatchedIn"); + fieldIn.setAccessible(true); + Map unmatched = (Map ) fieldIn.get(twoTraceMatch); + Map unmatchedTrace = (Map) unmatched.get(trace1); + assertEquals(3, unmatchedTrace.size()); + unmatchedTrace = (Map) unmatched.get(trace2); + assertEquals(2, unmatchedTrace.size()); + + /* Verify unmatched outgoing */ + Field fieldOut = twoTraceMatch.getClass().getDeclaredField("fUnmatchedOut"); + fieldOut.setAccessible(true); + unmatched = (Map ) fieldOut.get(twoTraceMatch); + unmatchedTrace = (Map) unmatched.get(trace1); + assertEquals(2, unmatchedTrace.size()); + unmatchedTrace = (Map) unmatched.get(trace2); + assertEquals(1, unmatchedTrace.size()); + + } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchFieldException e) { + fail(e.getMessage()); + } } } diff --git a/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/event/matching/TmfEventMatches.java b/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/event/matching/TmfEventMatches.java index c512ee26bb..b32ba7e1ec 100644 --- a/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/event/matching/TmfEventMatches.java +++ b/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/event/matching/TmfEventMatches.java @@ -38,7 +38,6 @@ public class TmfEventMatches implements IMatchProcessingUnit { /** * IMatchProcessingUnit overrides */ - @Override public void init(Collection fTraces) { fMatchCount = 0; diff --git a/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/event/matching/TmfEventMatching.java b/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/event/matching/TmfEventMatching.java index 7b7f1837b8..4f992f1789 100644 --- a/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/event/matching/TmfEventMatching.java +++ b/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/event/matching/TmfEventMatching.java @@ -12,11 +12,10 @@ package org.eclipse.tracecompass.tmf.core.event.matching; -import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.HashSet; +import java.util.Set; import org.eclipse.tracecompass.internal.tmf.core.Activator; import org.eclipse.tracecompass.tmf.core.event.ITmfEvent; @@ -24,6 +23,8 @@ import org.eclipse.tracecompass.tmf.core.request.ITmfEventRequest; import org.eclipse.tracecompass.tmf.core.request.TmfEventRequest; import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimeRange; import org.eclipse.tracecompass.tmf.core.trace.ITmfTrace; +import org.eclipse.tracecompass.tmf.core.trace.TmfTraceManager; +import org.eclipse.tracecompass.tmf.core.trace.experiment.TmfExperiment; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; @@ -50,6 +51,8 @@ public abstract class TmfEventMatching implements ITmfEventMatching { NETWORK } + private static final Multimap MATCH_DEFINITIONS = HashMultimap.create(); + /** * The array of traces to match */ @@ -60,8 +63,6 @@ public abstract class TmfEventMatching implements ITmfEventMatching { */ private final IMatchProcessingUnit fMatches; - private static final Map> fMatchDefinitions = new HashMap<>(); - private final Multimap fMatchMap = HashMultimap.create(); /** @@ -76,17 +77,35 @@ public abstract class TmfEventMatching implements ITmfEventMatching { if (tmfEventMatches == null) { throw new IllegalArgumentException(); } - fTraces = traces; + fTraces = new HashSet<>(traces); fMatches = tmfEventMatches; } /** - * Returns the traces to process + * Returns the traces to synchronize. These are the traces that were + * specified in the constructor, they may contain either traces or + * experiment. * - * @return The traces + * @return The traces to synchronize */ - protected Collection getTraces() { - return fTraces; + protected Collection getTraces() { + return new HashSet<>(fTraces); + } + + /** + * Returns the individual traces to process. If some of the traces specified + * to synchronize in the constructor were experiments, only the traces + * contained in this experiment will be returned. No {@link TmfExperiment} + * are returned by this method. + * + * @return The individual traces to synchronize, no experiments + */ + protected Collection getIndividualTraces() { + Set traces = new HashSet<>(); + for (ITmfTrace trace : fTraces) { + traces.addAll(Arrays.asList(TmfTraceManager.getTraceSet(trace))); + } + return traces; } /** @@ -116,11 +135,11 @@ public abstract class TmfEventMatching implements ITmfEventMatching { */ protected void initMatching() { fMatches.init(fTraces); - List deflist = fMatchDefinitions.get(getMatchingType()); + Collection deflist = MATCH_DEFINITIONS.get(getMatchingType()); if (deflist == null) { return; } - for (ITmfTrace trace : fTraces) { + for (ITmfTrace trace : getIndividualTraces()) { for (ITmfMatchEventDefinition def : deflist) { if (def.canMatchTrace(trace)) { fMatchMap.put(trace, def); @@ -227,10 +246,7 @@ public abstract class TmfEventMatching implements ITmfEventMatching { */ public static void registerMatchObject(ITmfMatchEventDefinition match) { for (MatchingType type : match.getApplicableMatchingTypes()) { - if (!fMatchDefinitions.containsKey(type)) { - fMatchDefinitions.put(type, new ArrayList()); - } - fMatchDefinitions.get(type).add(match); + MATCH_DEFINITIONS.put(type, match); } } diff --git a/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/event/matching/TmfNetworkEventMatching.java b/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/event/matching/TmfNetworkEventMatching.java index bb57160ab6..2cd8385711 100644 --- a/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/event/matching/TmfNetworkEventMatching.java +++ b/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/event/matching/TmfNetworkEventMatching.java @@ -84,7 +84,7 @@ public class TmfNetworkEventMatching extends TmfEventMatching { // Initialize the matching infrastructure (unmatched event lists) fUnmatchedIn.clear(); fUnmatchedOut.clear(); - for (ITmfTrace trace : getTraces()) { + for (ITmfTrace trace : getIndividualTraces()) { fUnmatchedIn.put(trace, new HashMap, ITmfEvent>()); fUnmatchedOut.put(trace, new HashMap, ITmfEvent>()); } @@ -187,8 +187,8 @@ public class TmfNetworkEventMatching extends TmfEventMatching { * events as value for the unmatched table. Not necessary right now * though */ - if (!unmatchedTbl.get(trace).containsKey(eventKey)) { - unmatchedTbl.get(trace).put(eventKey, event); + if (!unmatchedTbl.get(event.getTrace()).containsKey(eventKey)) { + unmatchedTbl.get(event.getTrace()).put(eventKey, event); } } @@ -206,7 +206,7 @@ public class TmfNetworkEventMatching extends TmfEventMatching { StringBuilder b = new StringBuilder(); b.append(getProcessingUnit()); int i = 0; - for (ITmfTrace trace : getTraces()) { + for (ITmfTrace trace : getIndividualTraces()) { b.append("Trace " + i++ + ":" + cr + " " + countEvents(fUnmatchedIn.get(trace)) + " unmatched incoming events" + cr + " " + countEvents(fUnmatchedOut.get(trace)) + " unmatched outgoing events" + cr); diff --git a/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/trace/experiment/TmfExperiment.java b/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/trace/experiment/TmfExperiment.java index 29bbe6d76e..f6de77f1e0 100644 --- a/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/trace/experiment/TmfExperiment.java +++ b/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/trace/experiment/TmfExperiment.java @@ -19,7 +19,9 @@ package org.eclipse.tracecompass.tmf.core.trace.experiment; import java.io.File; import java.nio.ByteBuffer; -import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import org.eclipse.core.resources.IProject; import org.eclipse.core.resources.IResource; @@ -107,6 +109,12 @@ public class TmfExperiment extends TmfTrace implements ITmfEventParser, ITmfPers */ private boolean fInitialized = false; + /** + * Lock for synchronization methods. These methods cannot be 'synchronized' + * since it makes it impossible to use an event request on the experiment + * during synchronization (the request thread would block) + */ + private final Lock fSyncLock = new ReentrantLock(); // ------------------------------------------------------------------------ // Construction @@ -520,7 +528,7 @@ public class TmfExperiment extends TmfTrace implements ITmfEventParser, ITmfPers * @return The synchronization object * @since 3.0 */ - public synchronized SynchronizationAlgorithm synchronizeTraces() { + public SynchronizationAlgorithm synchronizeTraces() { return synchronizeTraces(false); } @@ -533,25 +541,30 @@ public class TmfExperiment extends TmfTrace implements ITmfEventParser, ITmfPers * @return The synchronization object * @since 3.0 */ - public synchronized SynchronizationAlgorithm synchronizeTraces(boolean doSync) { + public SynchronizationAlgorithm synchronizeTraces(boolean doSync) { + fSyncLock.lock(); - String syncDirectory = getSynchronizationFolder(true); + try { + String syncDirectory = getSynchronizationFolder(true); - final File syncFile = (syncDirectory != null) ? new File(syncDirectory + File.separator + SYNCHRONIZATION_FILE_NAME) : null; + final File syncFile = (syncDirectory != null) ? new File(syncDirectory + File.separator + SYNCHRONIZATION_FILE_NAME) : null; - final SynchronizationAlgorithm syncAlgo = SynchronizationManager.synchronizeTraces(syncFile, Arrays.asList(fTraces), doSync); + final SynchronizationAlgorithm syncAlgo = SynchronizationManager.synchronizeTraces(syncFile, Collections. singleton(this), doSync); - final TmfTraceSynchronizedSignal signal = new TmfTraceSynchronizedSignal(this, syncAlgo); + final TmfTraceSynchronizedSignal signal = new TmfTraceSynchronizedSignal(this, syncAlgo); - /* Broadcast in separate thread to prevent deadlock */ - new Thread() { - @Override - public void run() { - broadcast(signal); - } - }.start(); + /* Broadcast in separate thread to prevent deadlock */ + new Thread() { + @Override + public void run() { + broadcast(signal); + } + }.start(); - return syncAlgo; + return syncAlgo; + } finally { + fSyncLock.unlock(); + } } @Override -- 2.34.1