1 /*******************************************************************************
2 * Copyright (c) 2011, 2014 Ericsson, Ecole Polytechnique de Montreal and others
4 * All rights reserved. This program and the accompanying materials are made
5 * available under the terms of the Eclipse Public License v1.0 which
6 * accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * Matthew Khouzam - Initial API and implementation
11 * Alexandre Montplaisir - Initial API and implementation
12 * Simon Delisle - Replace LinkedList by TreeSet in callsitesByName attribute
13 *******************************************************************************/
15 package org
.eclipse
.linuxtools
.ctf
.core
.trace
;
18 import java
.io
.FileFilter
;
19 import java
.io
.IOException
;
20 import java
.io
.Serializable
;
21 import java
.nio
.ByteBuffer
;
22 import java
.nio
.ByteOrder
;
23 import java
.nio
.channels
.FileChannel
;
24 import java
.nio
.channels
.FileChannel
.MapMode
;
25 import java
.nio
.file
.StandardOpenOption
;
26 import java
.util
.Arrays
;
27 import java
.util
.Collection
;
28 import java
.util
.Collections
;
29 import java
.util
.Comparator
;
30 import java
.util
.HashMap
;
33 import java
.util
.TreeSet
;
34 import java
.util
.UUID
;
36 import org
.eclipse
.linuxtools
.ctf
.core
.event
.CTFCallsite
;
37 import org
.eclipse
.linuxtools
.ctf
.core
.event
.CTFClock
;
38 import org
.eclipse
.linuxtools
.ctf
.core
.event
.IEventDeclaration
;
39 import org
.eclipse
.linuxtools
.ctf
.core
.event
.io
.BitBuffer
;
40 import org
.eclipse
.linuxtools
.ctf
.core
.event
.scope
.IDefinitionScope
;
41 import org
.eclipse
.linuxtools
.ctf
.core
.event
.scope
.LexicalScope
;
42 import org
.eclipse
.linuxtools
.ctf
.core
.event
.types
.Definition
;
43 import org
.eclipse
.linuxtools
.ctf
.core
.event
.types
.IDefinition
;
44 import org
.eclipse
.linuxtools
.ctf
.core
.event
.types
.IntegerDefinition
;
45 import org
.eclipse
.linuxtools
.ctf
.core
.event
.types
.StructDeclaration
;
46 import org
.eclipse
.linuxtools
.ctf
.core
.event
.types
.StructDefinition
;
47 import org
.eclipse
.linuxtools
.internal
.ctf
.core
.SafeMappedByteBuffer
;
48 import org
.eclipse
.linuxtools
.internal
.ctf
.core
.event
.CTFCallsiteComparator
;
49 import org
.eclipse
.linuxtools
.internal
.ctf
.core
.event
.metadata
.exceptions
.ParseException
;
50 import org
.eclipse
.linuxtools
.internal
.ctf
.core
.event
.types
.ArrayDefinition
;
53 * A CTF trace on the file system.
55 * Represents a trace on the filesystem. It is responsible of parsing the
56 * metadata, creating declarations data structures, indexing the event packets
57 * (in other words, all the work that can be shared between readers), but the
58 * actual reading of events is left to TraceReader.
60 * @author Matthew Khouzam
61 * @version $Revision: 1.0 $
63 public class CTFTrace
implements IDefinitionScope
, AutoCloseable
{
66 public String
toString() {
67 /* Only for debugging, shouldn't be externalized */
68 return "CTFTrace [path=" + fPath
+ ", major=" + fMajor
+ ", minor=" //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
69 + fMinor
+ ", uuid=" + fUuid
+ "]"; //$NON-NLS-1$ //$NON-NLS-2$
73 * The trace directory on the filesystem.
75 private final File fPath
;
78 * Major CTF version number
83 * Minor CTF version number
95 private ByteOrder fByteOrder
;
98 * Packet header structure declaration
100 private StructDeclaration fPacketHeaderDecl
= null;
103 * The clock of the trace
105 private CTFClock fSingleClock
= null;
108 * Packet header structure definition
110 * This is only used when opening the trace files, to read the first packet
111 * header and see if they are valid trace files.
113 private StructDefinition fPacketHeaderDef
;
116 * Collection of streams contained in the trace.
118 private final Map
<Long
, CTFStream
> fStreams
= new HashMap
<>();
121 * Collection of environment variables set by the tracer
123 private final Map
<String
, String
> fEnvironment
= new HashMap
<>();
126 * Collection of all the clocks in a system.
128 private final Map
<String
, CTFClock
> fClocks
= new HashMap
<>();
130 /** Handlers for the metadata files */
131 private static final FileFilter METADATA_FILE_FILTER
= new MetadataFileFilter();
132 private static final Comparator
<File
> METADATA_COMPARATOR
= new MetadataComparator();
134 /** Callsite helpers */
135 private CTFCallsiteComparator fCtfCallsiteComparator
= new CTFCallsiteComparator();
137 private Map
<String
, TreeSet
<CTFCallsite
>> fCallsitesByName
= new HashMap
<>();
139 /** Callsite helpers */
140 private TreeSet
<CTFCallsite
> fCallsitesByIP
= new TreeSet
<>();
142 // ------------------------------------------------------------------------
144 // ------------------------------------------------------------------------
150 * Filesystem path of the trace directory
151 * @throws CTFReaderException
152 * If no CTF trace was found at the path
154 public CTFTrace(String path
) throws CTFReaderException
{
155 this(new File(path
));
163 * Filesystem path of the trace directory.
164 * @throws CTFReaderException
165 * If no CTF trace was found at the path
167 public CTFTrace(File path
) throws CTFReaderException
{
169 final Metadata metadata
= new Metadata(this);
171 /* Set up the internal containers for this trace */
172 if (!fPath
.exists()) {
173 throw new CTFReaderException("Trace (" + path
.getPath() + ") doesn't exist. Deleted or moved?"); //$NON-NLS-1$ //$NON-NLS-2$
176 if (!fPath
.isDirectory()) {
177 throw new CTFReaderException("Path must be a valid directory"); //$NON-NLS-1$
180 /* Open and parse the metadata file */
181 metadata
.parseFile();
187 * Streamed constructor
195 private void init(File path
) throws CTFReaderException
{
197 /* Open all the trace files */
199 /* List files not called metadata and not hidden. */
200 File
[] files
= path
.listFiles(METADATA_FILE_FILTER
);
201 Arrays
.sort(files
, METADATA_COMPARATOR
);
203 /* Try to open each file */
204 for (File streamFile
: files
) {
205 openStreamInput(streamFile
);
208 /* Create their index */
209 for (CTFStream stream
: getStreams()) {
210 Set
<CTFStreamInput
> inputs
= stream
.getStreamInputs();
211 for (CTFStreamInput s
: inputs
) {
220 * FIXME Not needed anymore, class doesn't need to be AutoCloseable.
225 public void close() {
228 // ------------------------------------------------------------------------
229 // Getters/Setters/Predicates
230 // ------------------------------------------------------------------------
233 * Gets an event declaration hash map for a given streamID
236 * The ID of the stream from which to read
237 * @return The Hash map with the event declarations
239 * @deprecated use {@link CTFTrace#getEventDeclarations(Long)}
242 public Map
<Long
, IEventDeclaration
> getEvents(Long streamId
) {
243 return fStreams
.get(streamId
).getEvents();
247 * Gets an event declaration list for a given streamID
250 * The ID of the stream from which to read
251 * @return The list of event declarations
254 public Collection
<IEventDeclaration
> getEventDeclarations(Long streamId
) {
255 return fStreams
.get(streamId
).getEventDeclarations();
259 * Get an event by it's ID
262 * The ID of the stream from which to read
264 * the ID of the event
265 * @return the event declaration
267 * @deprecated use {@link CTFTrace#getEventType(long, int)} instead
270 public IEventDeclaration
getEventType(long streamId
, long id
) {
271 return getStream(streamId
).getEventDeclaration((int) id
);
275 * Get an event by it's ID
278 * The ID of the stream from which to read
280 * the ID of the event
281 * @return the event declaration
284 public IEventDeclaration
getEventType(long streamId
, int id
) {
285 return getEvents(streamId
).get(id
);
289 * Method getStream gets the stream for a given id
292 * Long the id of the stream
293 * @return Stream the stream that we need
296 public CTFStream
getStream(Long id
) {
297 return fStreams
.get(id
);
301 * Method nbStreams gets the number of available streams
303 * @return int the number of streams
305 public int nbStreams() {
306 return fStreams
.size();
310 * Method setMajor sets the major version of the trace (DO NOT USE)
313 * long the major version
315 public void setMajor(long major
) {
320 * Method setMinor sets the minor version of the trace (DO NOT USE)
323 * long the minor version
325 public void setMinor(long minor
) {
330 * Method setUUID sets the UUID of a trace
335 public void setUUID(UUID uuid
) {
340 * Method setByteOrder sets the byte order
343 * ByteOrder of the trace, can be little-endian or big-endian
345 public void setByteOrder(ByteOrder byteOrder
) {
346 fByteOrder
= byteOrder
;
350 * Method setPacketHeader sets the packet header of a trace (DO NOT USE)
352 * @param packetHeader
353 * StructDeclaration the header in structdeclaration form
355 public void setPacketHeader(StructDeclaration packetHeader
) {
356 fPacketHeaderDecl
= packetHeader
;
360 * Method majorIsSet is the major version number set?
362 * @return boolean is the major set?
365 public boolean majorIsSet() {
366 return fMajor
!= null;
370 * Method minorIsSet. is the minor version number set?
372 * @return boolean is the minor set?
374 public boolean minorIsSet() {
375 return fMinor
!= null;
379 * Method UUIDIsSet is the UUID set?
381 * @return boolean is the UUID set?
384 public boolean uuidIsSet() {
385 return fUuid
!= null;
389 * Method byteOrderIsSet is the byteorder set?
391 * @return boolean is the byteorder set?
393 public boolean byteOrderIsSet() {
394 return fByteOrder
!= null;
398 * Method packetHeaderIsSet is the packet header set?
400 * @return boolean is the packet header set?
402 public boolean packetHeaderIsSet() {
403 return fPacketHeaderDecl
!= null;
407 * Method getUUID gets the trace UUID
409 * @return UUID gets the trace UUID
411 public UUID
getUUID() {
416 * Method getMajor gets the trace major version
418 * @return long gets the trace major version
420 public long getMajor() {
425 * Method getMinor gets the trace minor version
427 * @return long gets the trace minor version
429 public long getMinor() {
434 * Method getByteOrder gets the trace byte order
436 * @return ByteOrder gets the trace byte order
438 public final ByteOrder
getByteOrder() {
443 * Method getPacketHeader gets the trace packet header
445 * @return StructDeclaration gets the trace packet header
447 public StructDeclaration
getPacketHeader() {
448 return fPacketHeaderDecl
;
452 * Method getTraceDirectory gets the trace directory
454 * @return File the path in "File" format.
456 public File
getTraceDirectory() {
461 * Get all the streams as an iterable.
463 * @return Iterable<Stream> an iterable over streams.
466 public Iterable
<CTFStream
> getStreams() {
467 return fStreams
.values();
471 * Method getPath gets the path of the trace directory
473 * @return String the path of the trace directory, in string format.
474 * @see java.io.File#getPath()
476 public String
getPath() {
477 return (fPath
!= null) ? fPath
.getPath() : ""; //$NON-NLS-1$
480 // ------------------------------------------------------------------------
482 // ------------------------------------------------------------------------
484 private void addStream(CTFStreamInput s
) {
489 CTFStream stream
= s
.getStream();
490 fStreams
.put(stream
.getId(), stream
);
499 * Tries to open the given file, reads the first packet header of the file
500 * and check its validity. This will add a file to a stream as a streaminput
503 * A trace file in the trace directory.
505 * Which index in the class' streamFileChannel array this file
507 * @throws CTFReaderException
508 * if there is a file error
510 private CTFStream
openStreamInput(File streamFile
) throws CTFReaderException
{
511 ByteBuffer byteBuffer
;
512 BitBuffer streamBitBuffer
;
515 if (!streamFile
.canRead()) {
516 throw new CTFReaderException("Unreadable file : " //$NON-NLS-1$
517 + streamFile
.getPath());
520 try (FileChannel fc
= FileChannel
.open(streamFile
.toPath(), StandardOpenOption
.READ
)) {
521 /* Map one memory page of 4 kiB */
522 byteBuffer
= SafeMappedByteBuffer
.map(fc
, MapMode
.READ_ONLY
, 0, (int) Math
.min(fc
.size(), 4096L));
523 if (byteBuffer
== null) {
524 throw new IllegalStateException("Failed to allocate memory"); //$NON-NLS-1$
526 /* Create a BitBuffer with this mapping and the trace byte order */
527 streamBitBuffer
= new BitBuffer(byteBuffer
, this.getByteOrder());
529 if (fPacketHeaderDecl
!= null) {
530 /* Read the packet header */
531 fPacketHeaderDef
= fPacketHeaderDecl
.createDefinition(this, LexicalScope
.PACKET_HEADER
, streamBitBuffer
);
533 } catch (IOException e
) {
534 /* Shouldn't happen at this stage if every other check passed */
535 throw new CTFReaderException(e
);
537 if (fPacketHeaderDef
!= null) {
538 validateMagicNumber(fPacketHeaderDef
);
540 validateUUID(fPacketHeaderDef
);
542 /* Read the stream ID */
543 IDefinition streamIDDef
= fPacketHeaderDef
.lookupDefinition("stream_id"); //$NON-NLS-1$
545 if (streamIDDef
instanceof IntegerDefinition
) {
546 /* This doubles as a null check */
547 long streamID
= ((IntegerDefinition
) streamIDDef
).getValue();
548 stream
= fStreams
.get(streamID
);
550 /* No stream_id in the packet header */
551 stream
= fStreams
.get(null);
555 /* No packet header, we suppose there is only one stream */
556 stream
= fStreams
.get(null);
559 if (stream
== null) {
560 throw new CTFReaderException("Unexpected end of stream"); //$NON-NLS-1$
564 * Create the stream input and add a reference to the streamInput in the
567 stream
.addInput(new CTFStreamInput(stream
, streamFile
));
571 private void validateUUID(StructDefinition packetHeaderDef
) throws CTFReaderException
{
572 IDefinition lookupDefinition
= packetHeaderDef
.lookupDefinition("uuid"); //$NON-NLS-1$
573 ArrayDefinition uuidDef
= (ArrayDefinition
) lookupDefinition
;
574 if (uuidDef
!= null) {
575 UUID otheruuid
= Utils
.getUUIDfromDefinition(uuidDef
);
576 if (!fUuid
.equals(otheruuid
)) {
577 throw new CTFReaderException("UUID mismatch"); //$NON-NLS-1$
582 private static void validateMagicNumber(StructDefinition packetHeaderDef
) throws CTFReaderException
{
583 IntegerDefinition magicDef
= (IntegerDefinition
) packetHeaderDef
.lookupDefinition("magic"); //$NON-NLS-1$
584 int magic
= (int) magicDef
.getValue();
585 if (magic
!= Utils
.CTF_MAGIC
) {
586 throw new CTFReaderException("CTF magic mismatch"); //$NON-NLS-1$
590 // ------------------------------------------------------------------------
592 // ------------------------------------------------------------------------
598 public LexicalScope
getScopePath() {
599 return LexicalScope
.TRACE
;
603 * Looks up a definition from packet
608 * @see org.eclipse.linuxtools.ctf.core.event.scope.IDefinitionScope#lookupDefinition(String)
611 public Definition
lookupDefinition(String lookupPath
) {
612 if (lookupPath
.equals(LexicalScope
.TRACE_PACKET_HEADER
.toString())) {
613 return fPacketHeaderDef
;
618 // ------------------------------------------------------------------------
619 // Live trace reading
620 // ------------------------------------------------------------------------
623 * Add a new stream file to support new streams while the trace is being
627 * the file of the stream
628 * @throws CTFReaderException
629 * A stream had an issue being read
632 public void addStreamFile(File streamFile
) throws CTFReaderException
{
633 openStreamInput(streamFile
);
637 * Registers a new stream to the trace.
641 * @throws ParseException
642 * If there was some problem reading the metadata
645 public void addStream(CTFStream stream
) throws ParseException
{
647 * If there is already a stream without id (the null key), it must be
650 if (fStreams
.get(null) != null) {
651 throw new ParseException("Stream without id with multiple streams"); //$NON-NLS-1$
655 * If the stream we try to add has the null key, it must be the only
656 * one. Thus, if the streams container is not empty, it is not valid.
658 if ((stream
.getId() == null) && (fStreams
.size() != 0)) {
659 throw new ParseException("Stream without id with multiple streams"); //$NON-NLS-1$
663 * If a stream with the same ID already exists, it is not valid.
665 CTFStream existingStream
= fStreams
.get(stream
.getId());
666 if (existingStream
!= null) {
667 throw new ParseException("Stream id already exists"); //$NON-NLS-1$
670 /* This stream is valid and has a unique id. */
671 fStreams
.put(stream
.getId(), stream
);
675 * Gets the Environment variables from the trace metadata (See CTF spec)
677 * @return The environment variables in the form of an unmodifiable map
681 public Map
<String
, String
> getEnvironment() {
682 return Collections
.unmodifiableMap(fEnvironment
);
686 * Add a variable to the environment variables
689 * the name of the variable
691 * the value of the variable
693 public void addEnvironmentVar(String varName
, String varValue
) {
694 fEnvironment
.put(varName
, varValue
);
698 * Add a clock to the clock list
701 * the name of the clock (full name with scope)
705 public void addClock(String nameValue
, CTFClock ctfClock
) {
706 fClocks
.put(nameValue
, ctfClock
);
710 * gets the clock with a specific name
713 * the name of the clock.
716 public CTFClock
getClock(String name
) {
717 return fClocks
.get(name
);
721 * gets the clock if there is only one. (this is 100% of the use cases as of
726 public final CTFClock
getClock() {
727 if (fSingleClock
!= null && fClocks
.size() == 1) {
730 if (fClocks
.size() == 1) {
731 fSingleClock
= fClocks
.get(fClocks
.keySet().iterator().next());
738 * gets the time offset of a clock with respect to UTC in nanoseconds
740 * @return the time offset of a clock with respect to UTC in nanoseconds
742 public final long getOffset() {
743 if (getClock() == null) {
746 return fSingleClock
.getClockOffset();
750 * gets the time offset of a clock with respect to UTC in nanoseconds
752 * @return the time offset of a clock with respect to UTC in nanoseconds
754 private double getTimeScale() {
755 if (getClock() == null) {
758 return fSingleClock
.getClockScale();
762 * Gets the current first packet start time
764 * @return the current start time
767 public long getCurrentStartTime() {
768 long currentStart
= Long
.MAX_VALUE
;
769 for (CTFStream stream
: fStreams
.values()) {
770 for (CTFStreamInput si
: stream
.getStreamInputs()) {
771 currentStart
= Math
.min(currentStart
, si
.getIndex().getEntries().get(0).getTimestampBegin());
774 return timestampCyclesToNanos(currentStart
);
778 * Gets the current last packet end time
780 * @return the current end time
783 public long getCurrentEndTime() {
784 long currentEnd
= Long
.MIN_VALUE
;
785 for (CTFStream stream
: fStreams
.values()) {
786 for (CTFStreamInput si
: stream
.getStreamInputs()) {
787 currentEnd
= Math
.max(currentEnd
, si
.getTimestampEnd());
790 return timestampCyclesToNanos(currentEnd
);
794 * Does the trace need to time scale?
796 * @return if the trace is in ns or cycles.
798 private boolean clockNeedsScale() {
799 if (getClock() == null) {
802 return fSingleClock
.isClockScaled();
806 * the inverse clock for returning to a scale.
808 * @return 1.0 / scale
810 private double getInverseTimeScale() {
811 if (getClock() == null) {
814 return fSingleClock
.getClockAntiScale();
819 * clock cycles since boot
820 * @return time in nanoseconds UTC offset
823 public long timestampCyclesToNanos(long cycles
) {
824 long retVal
= cycles
+ getOffset();
826 * this fix is since quite often the offset will be > than 53 bits and
827 * therefore the conversion will be lossy
829 if (clockNeedsScale()) {
830 retVal
= (long) (retVal
* getTimeScale());
837 * time in nanoseconds UTC offset
838 * @return clock cycles since boot.
841 public long timestampNanoToCycles(long nanos
) {
844 * this fix is since quite often the offset will be > than 53 bits and
845 * therefore the conversion will be lossy
847 if (clockNeedsScale()) {
848 retVal
= (long) (nanos
* getInverseTimeScale());
852 return retVal
- getOffset();
859 * the event name of the callsite
861 * the name of the callsite function
863 * the ip of the callsite
865 * the filename of the callsite
867 * the line number of the callsite
869 public void addCallsite(String eventName
, String funcName
, long ip
,
870 String fileName
, long lineNumber
) {
871 final CTFCallsite cs
= new CTFCallsite(eventName
, funcName
, ip
,
872 fileName
, lineNumber
);
873 TreeSet
<CTFCallsite
> csl
= fCallsitesByName
.get(eventName
);
875 csl
= new TreeSet
<>(fCtfCallsiteComparator
);
876 fCallsitesByName
.put(eventName
, csl
);
881 fCallsitesByIP
.add(cs
);
885 * Gets the set of callsites associated to an event name. O(1)
889 * @return the callsite set can be empty
892 public TreeSet
<CTFCallsite
> getCallsiteCandidates(String eventName
) {
893 TreeSet
<CTFCallsite
> retVal
= fCallsitesByName
.get(eventName
);
894 if (retVal
== null) {
895 retVal
= new TreeSet
<>(fCtfCallsiteComparator
);
901 * The I'm feeling lucky of getCallsiteCandidates O(1)
905 * @return the first callsite that has that event name, can be null
908 public CTFCallsite
getCallsite(String eventName
) {
909 TreeSet
<CTFCallsite
> callsites
= fCallsitesByName
.get(eventName
);
910 if (callsites
!= null) {
911 return callsites
.first();
917 * Gets a callsite from the instruction pointer O(log(n))
920 * the instruction pointer to lookup
921 * @return the callsite just before that IP in the list remember the IP is
922 * backwards on X86, can be null if no callsite is before the IP.
925 public CTFCallsite
getCallsite(long ip
) {
926 CTFCallsite cs
= new CTFCallsite(null, null, ip
, null, 0L);
927 return fCallsitesByIP
.ceiling(cs
);
931 * Gets a callsite using the event name and instruction pointer O(log(n))
934 * the name of the event
936 * the instruction pointer
937 * @return the closest matching callsite, can be null
939 public CTFCallsite
getCallsite(String eventName
, long ip
) {
940 final TreeSet
<CTFCallsite
> candidates
= fCallsitesByName
.get(eventName
);
941 if (candidates
== null) {
944 final CTFCallsite dummyCs
= new CTFCallsite(null, null, ip
, null, -1);
945 final CTFCallsite callsite
= candidates
.ceiling(dummyCs
);
946 if (callsite
== null) {
947 return candidates
.floor(dummyCs
);
956 * the ID of the stream
958 * new file in the stream
959 * @throws CTFReaderException
960 * The file must exist
963 // TODO: remove suppress warning
964 @SuppressWarnings("resource")
965 public void addStream(long id
, File streamFile
) throws CTFReaderException
{
966 CTFStream stream
= null;
967 final File file
= streamFile
;
969 throw new CTFReaderException("cannot create a stream with no file"); //$NON-NLS-1$
971 if (fStreams
.containsKey(id
)) {
972 stream
= fStreams
.get(id
);
974 stream
= new CTFStream(this);
975 fStreams
.put(id
, stream
);
977 stream
.addInput(new CTFStreamInput(stream
, file
));
981 class MetadataFileFilter
implements FileFilter
{
984 public boolean accept(File pathname
) {
985 if (pathname
.isDirectory()) {
988 if (pathname
.isHidden()) {
991 if (pathname
.getName().equals("metadata")) { //$NON-NLS-1$
999 class MetadataComparator
implements Comparator
<File
>, Serializable
{
1001 private static final long serialVersionUID
= 1L;
1004 public int compare(File o1
, File o2
) {
1005 return o1
.getName().compareTo(o2
.getName());