From 9bc251139ac56d68a844b61bd63598d0f2ca4534 Mon Sep 17 00:00:00 2001 From: Francis Deslauriers Date: Fri, 3 Nov 2023 19:15:49 +0000 Subject: [PATCH] src.ctf.fs: implement `MetadataStreamDecoder` class This class offers a `decode()` method that decodes a potentially packetized metadata stream as a `std::string`. After decoding a buffer, the `pktInfo()` method may return packet information of the packetized metadata stream. Signed-off-by: Francis Deslauriers Change-Id: Iee867271fe6493b19c4bd77151409886b7c01ed9 Reviewed-on: https://review.lttng.org/c/babeltrace/+/8050 Reviewed-by: Philippe Proulx Reviewed-on: https://review.lttng.org/c/babeltrace/+/12257 --- src/Makefile.am | 2 + src/compat/endian.h | 1 + .../metadata/tsdl/metadata-stream-decoder.cpp | 321 ++++++++++++++++++ .../metadata/tsdl/metadata-stream-decoder.hpp | 205 +++++++++++ 4 files changed, 529 insertions(+) create mode 100644 src/plugins/ctf/common/src/metadata/tsdl/metadata-stream-decoder.cpp create mode 100644 src/plugins/ctf/common/src/metadata/tsdl/metadata-stream-decoder.hpp diff --git a/src/Makefile.am b/src/Makefile.am index e42a4054..e8638eea 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -697,6 +697,8 @@ plugins_ctf_babeltrace_plugin_ctf_la_SOURCES = \ plugins/ctf/common/src/item-seq/medium.hpp \ plugins/ctf/common/src/metadata/ctf-ir.cpp \ plugins/ctf/common/src/metadata/ctf-ir.hpp \ + plugins/ctf/common/src/metadata/tsdl/metadata-stream-decoder.cpp \ + plugins/ctf/common/src/metadata/tsdl/metadata-stream-decoder.hpp \ plugins/ctf/common/src/msg-iter/msg-iter.cpp \ plugins/ctf/common/src/msg-iter/msg-iter.hpp \ plugins/ctf/common/src/null-cp-finder.hpp \ diff --git a/src/compat/endian.h b/src/compat/endian.h index 1de99d2d..cd2e93c8 100644 --- a/src/compat/endian.h +++ b/src/compat/endian.h @@ -49,6 +49,7 @@ #elif defined(__MINGW32__) #include +#include #ifndef __BIG_ENDIAN #define __BIG_ENDIAN 4321 diff --git a/src/plugins/ctf/common/src/metadata/tsdl/metadata-stream-decoder.cpp b/src/plugins/ctf/common/src/metadata/tsdl/metadata-stream-decoder.cpp new file mode 100644 index 00000000..8b75eeb2 --- /dev/null +++ b/src/plugins/ctf/common/src/metadata/tsdl/metadata-stream-decoder.cpp @@ -0,0 +1,321 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright 2022 Francis Deslauriers + * Copyright 2024 Philippe Proulx + */ + +#include + +#include "cpp-common/bt2c/aliases.hpp" +#include "cpp-common/bt2c/read-fixed-len-int.hpp" +#include "cpp-common/vendor/fmt/format.h" + +#include "metadata-stream-decoder.hpp" + +namespace ctf { +namespace src { + +const bt2c::DataLen MetadataStreamDecoder::_PktHeader::len = bt2c::DataLen::fromBytes(37); + +MetadataStreamDecoder::_PktHeader::_PktHeader( + const std::uint32_t magicParam, const bt2c::Uuid& uuidParam, const std::uint32_t checksumParam, + const bt2c::DataLen contentLenParam, const bt2c::DataLen totalLenParam, + const std::uint8_t compressionSchemeParam, const std::uint8_t encryptionSchemeParam, + const std::uint8_t checksumSchemeParam, const std::uint8_t majorVersionParam, + const std::uint8_t minorVersionParam) : + magic {magicParam}, + uuid {uuidParam}, checksum {checksumParam}, contentLen {contentLenParam}, + totalLen {totalLenParam}, compressionScheme {compressionSchemeParam}, + encryptionScheme {encryptionSchemeParam}, checksumScheme {checksumSchemeParam}, + majorVersion {majorVersionParam}, minorVersion {minorVersionParam} +{ +} + +void MetadataStreamDecoder::_validatePktHeader(const _PktHeader& header) const +{ + if (header.compressionScheme != 0) { + BT_CPPLOGE_APPEND_CAUSE_AND_THROW( + bt2c::Error, + "Metadata stream packet compression is not supported as of this version: " + "compression-scheme={}", + header.compressionScheme); + } + + if (header.encryptionScheme != 0) { + BT_CPPLOGE_APPEND_CAUSE_AND_THROW( + bt2c::Error, + "Metadata stream packet encryption is not supported as of this version: " + "encryption-scheme={}", + header.encryptionScheme); + } + + if (header.checksum != 0 || header.checksumScheme != 0) { + BT_CPPLOGE_APPEND_CAUSE_AND_THROW( + bt2c::Error, + "Metadata stream packet checksum verification is not supported as of this version: " + "checksum-scheme={}, checksum={:x}", + header.checksumScheme, header.checksum); + } + + if (!header.versionIsValid()) { + BT_CPPLOGE_APPEND_CAUSE_AND_THROW(bt2c::Error, + "Expecting metadata stream packet version 1.8: " + "actual-version={}.{}", + header.majorVersion, header.minorVersion); + } + + try { + if (header.contentLen < _PktHeader::len) { + BT_CPPLOGE_APPEND_CAUSE_AND_THROW( + bt2c::Error, "Packet content length is less than the header length."); + } + + if (header.contentLen.hasExtraBits()) { + BT_CPPLOGE_APPEND_CAUSE_AND_THROW(bt2c::Error, + "Packet content length is not a multiple of 8."); + } + } catch (const bt2c::Error&) { + BT_CPPLOGE_APPEND_CAUSE_AND_RETHROW( + "Invalid metadata stream packet content length: content-len-bits={}", + header.contentLen.bits()); + } + + try { + if (header.totalLen < header.contentLen) { + BT_CPPLOGE_APPEND_CAUSE_AND_THROW( + bt2c::Error, "Packet total length is less than packet content length."); + } + + if (header.totalLen.hasExtraBits()) { + BT_CPPLOGE_APPEND_CAUSE_AND_THROW(bt2c::Error, + "Packet total length is not a multiple of 8."); + } + } catch (const bt2c::Error&) { + BT_CPPLOGE_APPEND_CAUSE_AND_RETHROW( + "Invalid metadata stream packet total length: total-len-bits={}", + header.totalLen.bits()); + } +} + +bt2s::optional +MetadataStreamDecoder::_getByteOrder(const bt2c::ConstBytes buffer) const noexcept +{ + /* We need to read a 32-bit magic number */ + BT_ASSERT(buffer.size() >= sizeof(std::uint32_t)); + + static constexpr std::uint32_t expectedMagic = 0x75d11d57U; + static constexpr auto nativeByteOrder = + BYTE_ORDER == BIG_ENDIAN ? ByteOrder::Big : ByteOrder::Little; + + /* Read magic number */ + const auto magic = bt2c::readFixedLenInt(buffer.data()); + + /* Dedude byte order of the metadata stream packet header */ + if (magic == expectedMagic) { + return nativeByteOrder; + } else if (magic == GUINT32_SWAP_LE_BE(expectedMagic)) { + return nativeByteOrder == ByteOrder::Big ? ByteOrder::Little : ByteOrder::Big; + } else { + /* Doesn't look like a metadata stream packet */ + return bt2s::nullopt; + } +} + +namespace { + +/* + * Stateful reader of packet header fields. + */ +class PktHeaderReader final +{ +public: + explicit PktHeaderReader(const ByteOrder byteOrder, const std::uint8_t * const buf) : + _mByteOrder {byteOrder}, _mBuf {buf} + { + } + + std::uint32_t readNextUInt32Field() noexcept + { + return this->_readNextIntAndAdvance(); + } + + std::uint8_t readNextUInt8Field() noexcept + { + return this->_readNextIntAndAdvance(); + } + + bt2c::Uuid readNextUuidField() noexcept + { + const bt2c::Uuid uuid {_mBuf}; + + _mBuf += uuid.size(); + return uuid; + } + +private: + template + IntT _readNextInt() const noexcept + { + if (_mByteOrder == ByteOrder::Big) { + return bt2c::readFixedLenIntBe(_mBuf); + } else { + BT_ASSERT(_mByteOrder == ByteOrder::Little); + return bt2c::readFixedLenIntLe(_mBuf); + } + } + + template + IntT _readNextIntAndAdvance() noexcept + { + const auto res = this->_readNextInt(); + + _mBuf += sizeof(res); + return res; + } + + ByteOrder _mByteOrder; + const std::uint8_t *_mBuf; +}; + +} /* namespace */ + +MetadataStreamDecoder::_PktHeader +MetadataStreamDecoder::_readPktHeader(const std::uint8_t * const buf, const ByteOrder byteOrder, + const bt2c::DataLen curOffset) const +{ + BT_ASSERT(!curOffset.hasExtraBits()); + + PktHeaderReader reader {byteOrder, buf}; + + const auto magic = reader.readNextUInt32Field(); + const auto uuid = reader.readNextUuidField(); + const auto checksum = reader.readNextUInt32Field(); + const auto contentLen = bt2c::DataLen::fromBits(reader.readNextUInt32Field()); + const auto totalLen = bt2c::DataLen::fromBits(reader.readNextUInt32Field()); + const auto compressionScheme = reader.readNextUInt8Field(); + const auto encryptionScheme = reader.readNextUInt8Field(); + const auto checksumScheme = reader.readNextUInt8Field(); + const auto majorVersion = reader.readNextUInt8Field(); + const auto minorVersion = reader.readNextUInt8Field(); + + const _PktHeader header {magic, + uuid, + checksum, + contentLen, + totalLen, + compressionScheme, + encryptionScheme, + checksumScheme, + majorVersion, + minorVersion}; + + try { + this->_validatePktHeader(header); + } catch (const bt2c::Error&) { + BT_CPPLOGE_APPEND_CAUSE_AND_RETHROW("Invalid packet header: offset-bytes={}", + curOffset.bytes()); + } + + return header; +} + +MetadataStreamDecoder::MetadataStreamDecoder(const bt2c::Logger& parentLogger) noexcept : + _mLogger {parentLogger, "PLUGIN/CTF/META/DECODER"} +{ + BT_CPPLOGD("Creating TSDL metadata stream decoder."); +} + +std::string MetadataStreamDecoder::_textFromPacketizedMetadata(const bt2c::ConstBytes buffer) +{ + const auto byteOrder = this->_getByteOrder(buffer); + + /* It's a packetized metadata stream section */ + BT_ASSERT(byteOrder); + + std::string plainTextMetadata; + auto curOffset = bt2c::DataLen::fromBits(0); + + while (curOffset.bytes() < buffer.size()) { + try { + const auto pktData = buffer.data() + curOffset.bytes(); + + if (curOffset + _PktHeader::len > bt2c::DataLen::fromBytes(buffer.size())) { + BT_CPPLOGE_APPEND_CAUSE_AND_THROW( + bt2c::Error, "Remaining buffer isn't large enough to hold a packet header."); + } + + const auto header = this->_readPktHeader(pktData, *byteOrder, curOffset); + + if (_mPktInfo) { + if (_mPktInfo->uuid() != header.uuid) { + BT_CPPLOGE_APPEND_CAUSE_AND_THROW( + bt2c::Error, + "Metadata UUID mismatch between packets of the same metadata stream: " + "pkt-uuid=\"" BT_UUID_FMT "\", " + "expected-uuid=\"" BT_UUID_FMT "\"", + BT_UUID_FMT_VALUES(header.uuid), BT_UUID_FMT_VALUES(_mPktInfo->uuid())); + } + } else { + _mPktInfo = MetadataStreamPacketInfo {*byteOrder, header.majorVersion, + header.minorVersion, header.uuid}; + } + + /* Copy the packet payload */ + const auto payload = pktData + _PktHeader::len.bytes(); + const auto payloadLen = header.contentLen - _PktHeader::len; + + plainTextMetadata.append(reinterpret_cast(payload), payloadLen.bytes()); + + /* Advance offset to the next packet */ + curOffset += header.totalLen; + ++_mPktIdx; + } catch (const bt2c::Error&) { + BT_CPPLOGE_APPEND_CAUSE_AND_RETHROW( + "Failed to read a metadata stream packet: offset-bytes={}, pkt-idx={}", + curOffset.bytes(), _mPktIdx); + } + } + + return plainTextMetadata; +} + +void MetadataStreamDecoder::_maybeSetMetadataStreamType(const bt2c::ConstBytes buffer) +{ + if (this->_getByteOrder(buffer)) { + if (!_mStreamType) { + _mStreamType = _MetadataStreamType::Packetized; + } else if (*_mStreamType != _MetadataStreamType::Packetized) { + BT_CPPLOGE_APPEND_CAUSE_AND_THROW(bt2c::Error, + "Expecting a packetized metadata stream section."); + } + } else { + if (!_mStreamType) { + _mStreamType = _MetadataStreamType::PlainText; + } else if (*_mStreamType != _MetadataStreamType::PlainText) { + BT_CPPLOGE_APPEND_CAUSE_AND_THROW(bt2c::Error, + "Expecting a plain text metadata stream section."); + } + } +} + +std::string MetadataStreamDecoder::decode(const bt2c::ConstBytes buffer) +{ + this->_maybeSetMetadataStreamType(buffer); + + try { + if (*_mStreamType == _MetadataStreamType::Packetized) { + return this->_textFromPacketizedMetadata(buffer); + } else { + BT_ASSERT(*_mStreamType == _MetadataStreamType::PlainText); + return std::string {reinterpret_cast(buffer.data()), buffer.size()}; + } + } catch (const bt2c::Error&) { + BT_CPPLOGE_APPEND_CAUSE_AND_RETHROW( + "Failed to decode metadata stream section: data-ptr={}, data-len-bytes={}", + fmt::ptr(buffer.data()), buffer.size()); + } +} + +} /* namespace src */ +} /* namespace ctf */ diff --git a/src/plugins/ctf/common/src/metadata/tsdl/metadata-stream-decoder.hpp b/src/plugins/ctf/common/src/metadata/tsdl/metadata-stream-decoder.hpp new file mode 100644 index 00000000..5c51fadb --- /dev/null +++ b/src/plugins/ctf/common/src/metadata/tsdl/metadata-stream-decoder.hpp @@ -0,0 +1,205 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright 2022 Francis Deslauriers + * Copyright 2024 Philippe Proulx + */ + +#ifndef BABELTRACE_PLUGINS_CTF_COMMON_SRC_METADATA_TSDL_METADATA_STREAM_DECODER_HPP +#define BABELTRACE_PLUGINS_CTF_COMMON_SRC_METADATA_TSDL_METADATA_STREAM_DECODER_HPP + +#include "cpp-common/bt2c/aliases.hpp" +#include "cpp-common/bt2c/data-len.hpp" +#include "cpp-common/bt2c/logging.hpp" +#include "cpp-common/bt2s/optional.hpp" + +#include "../../metadata/ctf-ir.hpp" + +namespace ctf { +namespace src { + +/* + * Packet information of a metadata stream. + */ +class MetadataStreamPacketInfo final +{ +public: + explicit MetadataStreamPacketInfo(const ByteOrder byteOrder, const unsigned int major, + const unsigned int minor, const bt2c::Uuid& uuid) noexcept : + _mByteOrder {byteOrder}, + _mMajor {major}, _mMinor {minor}, _mUuid {uuid} + { + } + + bool operator==(const MetadataStreamPacketInfo& other) const noexcept + { + return _mByteOrder == other._mByteOrder && _mMajor == other._mMajor && + _mMinor == other._mMinor && _mUuid == other._mUuid; + } + + bool operator!=(const MetadataStreamPacketInfo& other) const noexcept + { + return !(*this == other); + } + + ByteOrder byteOrder() const noexcept + { + return _mByteOrder; + } + + unsigned int majorVersion() const noexcept + { + return _mMajor; + } + + unsigned int minorVersion() const noexcept + { + return _mMinor; + } + + const bt2c::Uuid& uuid() const noexcept + { + return _mUuid; + } + +private: + ByteOrder _mByteOrder; + unsigned int _mMajor; + unsigned int _mMinor; + bt2c::Uuid _mUuid; +}; + +/* + * A metadata stream decoder offers the decode() method to convert + * either a plain text metadata stream or multiple packets of a + * packetized metadata stream to plain text. + * + * The first call to decode() determines the permanent mode, based on + * the data, of the decoder amongst: + * + * Plain text mode: + * The next calls to decode() only accept plain text metadata stream + * data, throwing `bt2c::Error` otherwise. + * + * pktInfo() returns `bt2s::nullopt`. + * + * Packetized mode: + * The next calls to decode() only accept packetized metadata stream + * data, throwing `bt2c::Error` otherwise. + * + * Furthermore, the next calls to decode() validate that each + * metadata stream packet has the same UUID, again throwing + * `bt2c::Error` otherwise. + * + * pktInfo() returns a value. + */ +class MetadataStreamDecoder final +{ +public: + explicit MetadataStreamDecoder(const bt2c::Logger& parentLogger) noexcept; + + /* + * Decodes the next metadata stream section `buffer`, appending a + * cause to the error of the current thread and throwing + * `bt2c::Error` on error. + * + * `buffer.size()` must be greater than or equal to 4. + */ + std::string decode(bt2c::ConstBytes buffer); + + const bt2s::optional& pktInfo() const noexcept + { + return _mPktInfo; + } + +private: + /* + * Type of metadata stream. + */ + enum class _MetadataStreamType + { + Packetized, + PlainText, + }; + + /* + * Container of metadata stream packet header information. + * + * This structure is not to be used as a direct memory mapping of + * such a header. + */ + struct _PktHeader final + { + explicit _PktHeader(std::uint32_t magicParam, const bt2c::Uuid& uuidParam, + std::uint32_t checksumParam, bt2c::DataLen contentLenParam, + bt2c::DataLen totalLenParam, std::uint8_t compressionSchemeParam, + std::uint8_t encryptionSchemeParam, std::uint8_t checksumSchemeParam, + std::uint8_t majorVersionParam, std::uint8_t minorVersionParam); + + bool versionIsValid() const noexcept + { + return majorVersion == 1 && minorVersion == 8; + } + + static const bt2c::DataLen len; + std::uint32_t magic; + bt2c::Uuid uuid; + std::uint32_t checksum; + bt2c::DataLen contentLen; + bt2c::DataLen totalLen; + std::uint8_t compressionScheme; + std::uint8_t encryptionScheme; + std::uint8_t checksumScheme; + std::uint8_t majorVersion; + std::uint8_t minorVersion; + }; + + /* + * Returns the byte order of the metadata stream `buffer`, or + * `bt2s::nullopt` if `buffer` doesn't look like a packet header. + * + * `buffer.size()` must be greater than or equal to 4. + */ + bt2s::optional _getByteOrder(bt2c::ConstBytes buffer) const noexcept; + + /* + * Reads and returns one metadata stream packet header having the + * byte order `byteOrder` from `buf` at the offset `curOffset` + * within some metadata stream section. + * + * `buf` must offer at least `_PktHeader::len.bytes()` bytes + * of data. + * + * `curOffset.hasExtraBits()` must return false. + */ + _PktHeader _readPktHeader(const std::uint8_t *buf, ByteOrder byteOrder, + bt2c::DataLen curOffset) const; + + /* + * Validates the packet header `header`, throwing + * `bt2c::Error` if it's invalid. + */ + void _validatePktHeader(const _PktHeader& header) const; + + /* + * Returns the plain text data from the packetized metadata + * stream `buffer`. + */ + std::string _textFromPacketizedMetadata(bt2c::ConstBytes buffer); + + /* + * Sets the current metadata stream type from `buffer` if not + * already done. + */ + void _maybeSetMetadataStreamType(bt2c::ConstBytes buffer); + + bt2c::Logger _mLogger; + bt2s::optional _mPktInfo; + std::size_t _mPktIdx = 0; + bt2s::optional<_MetadataStreamType> _mStreamType; +}; + +} /* namespace src */ +} /* namespace ctf */ + +#endif /* BABELTRACE_PLUGINS_CTF_COMMON_SRC_METADATA_TSDL_METADATA_STREAM_DECODER_HPP */ -- 2.34.1