barectf/config_parse.py

   1 # The MIT License (MIT)
   2 #
   3 # Copyright (c) 2015-2020 Philippe Proulx <pproulx@efficios.com>
   4 #
   5 # Permission is hereby granted, free of charge, to any person obtaining
   6 # a copy of this software and associated documentation files (the
   7 # "Software"), to deal in the Software without restriction, including
   8 # without limitation the rights to use, copy, modify, merge, publish,
   9 # distribute, sublicense, and/or sell copies of the Software, and to
  10 # permit persons to whom the Software is furnished to do so, subject to
  11 # the following conditions:
  12 #
  13 # The above copyright notice and this permission notice shall be
  14 # included in all copies or substantial portions of the Software.
  15 #
  16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  17 # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  18 # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  19 # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
  20 # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  21 # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  22 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  23
  24 from barectf import metadata
  25 from barectf import config
  26 import pkg_resources
  27 import collections
  28 import jsonschema
  29 import os.path
  30 import enum
  31 import yaml
  32 import uuid
  33 import copy
  34 import os
  35
  36
  37 # The context of a configuration parsing error.
  38 #
  39 # Such a context object has a name and, optionally, a message.
  40 class _ConfigParseErrorCtx:
  41     def __init__(self, name, msg=None):
  42         self._name = name
  43         self._msg = msg
  44
  45     @property
  46     def name(self):
  47         return self._name
  48
  49     @property
  50     def msg(self):
  51         return self._msg
  52
  53
  54 # Appends the context having the object name `obj_name` and the
  55 # (optional) message `msg` to the `_ConfigParseError` exception `exc`
  56 # and then raises `exc` again.
  57 def _append_error_ctx(exc, obj_name, msg=None):
  58     exc.append_ctx(obj_name, msg)
  59     raise
  60
  61
  62 # A configuration parsing error.
  63 #
  64 # Such an error object contains a list of contexts (`ctx` property).
  65 #
  66 # The first context of this list is the most specific context, while the
  67 # last is the more general.
  68 #
  69 # Use append_ctx() to append a context to an existing configuration
  70 # parsing error when you catch it before raising it again. You can use
  71 # _append_error_ctx() to do exactly this in a single call.
  72 class _ConfigParseError(RuntimeError):
  73     def __init__(self, init_ctx_name, init_ctx_msg=None):
  74         self._ctx = []
  75         self.append_ctx(init_ctx_name, init_ctx_msg)
  76
  77     @property
  78     def ctx(self):
  79         return self._ctx
  80
  81     def append_ctx(self, name, msg=None):
  82         self._ctx.append(_ConfigParseErrorCtx(name, msg))
  83
  84
  85 def _opt_to_public(obj):
  86     if obj is None:
  87         return
  88
  89     return obj.to_public()
  90
  91
  92 # Pseudo object base class.
  93 #
  94 # A concrete pseudo object contains the same data as its public version,
  95 # but it's mutable.
  96 #
  97 # The to_public() method converts the pseudo object to an equivalent
  98 # public, immutable object, caching the result so as to always return
  99 # the same Python object.
 100 class _PseudoObj:
 101     def __init__(self):
 102         self._public = None
 103
 104     def to_public(self):
 105         if self._public is None:
 106             self._public = self._to_public()
 107
 108         return self._public
 109
 110     def _to_public(self):
 111         raise NotImplementedError
 112
 113
 114 class _PropertyMapping(_PseudoObj):
 115     def __init__(self):
 116         super().__init__()
 117         self.object = None
 118         self.prop = None
 119
 120     def _to_public(self):
 121         return metadata.PropertyMapping(self.object.to_public(), self.prop)
 122
 123
 124 class _Integer(_PseudoObj):
 125     def __init__(self):
 126         super().__init__()
 127         self.size = None
 128         self.byte_order = None
 129         self.align = None
 130         self.signed = False
 131         self.base = 10
 132         self.encoding = metadata.Encoding.NONE
 133         self.property_mappings = []
 134
 135     @property
 136     def real_align(self):
 137         if self.align is None:
 138             if self.size % 8 == 0:
 139                 return 8
 140             else:
 141                 return 1
 142         else:
 143             return self.align
 144
 145     def _to_public(self):
 146         prop_mappings = [pm.to_public() for pm in self.property_mappings]
 147         return metadata.Integer(self.size, self.byte_order, self.align,
 148                                 self.signed, self.base, self.encoding,
 149                                 prop_mappings)
 150
 151
 152 class _FloatingPoint(_PseudoObj):
 153     def __init__(self):
 154         super().__init__()
 155         self.exp_size = None
 156         self.mant_size = None
 157         self.byte_order = None
 158         self.align = 8
 159
 160     @property
 161     def real_align(self):
 162         return self.align
 163
 164     def _to_public(self):
 165         return metadata.FloatingPoint(self.exp_size, self.mant_size,
 166                                       self.byte_order, self.align)
 167
 168
 169 class _Enum(_PseudoObj):
 170     def __init__(self):
 171         super().__init__()
 172         self.value_type = None
 173         self.members = collections.OrderedDict()
 174
 175     @property
 176     def real_align(self):
 177         return self.value_type.real_align
 178
 179     def _to_public(self):
 180         return metadata.Enum(self.value_type.to_public(), self.members)
 181
 182
 183 class _String(_PseudoObj):
 184     def __init__(self):
 185         super().__init__()
 186         self.encoding = metadata.Encoding.UTF8
 187
 188     @property
 189     def real_align(self):
 190         return 8
 191
 192     def _to_public(self):
 193         return metadata.String(self.encoding)
 194
 195
 196 class _Array(_PseudoObj):
 197     def __init__(self):
 198         super().__init__()
 199         self.element_type = None
 200         self.length = None
 201
 202     @property
 203     def real_align(self):
 204         return self.element_type.real_align
 205
 206     def _to_public(self):
 207         return metadata.Array(self.element_type.to_public(), self.length)
 208
 209
 210 class _Struct(_PseudoObj):
 211     def __init__(self):
 212         super().__init__()
 213         self.min_align = 1
 214         self.fields = collections.OrderedDict()
 215
 216     @property
 217     def real_align(self):
 218         align = self.min_align
 219
 220         for pseudo_field in self.fields.values():
 221             if pseudo_field.real_align > align:
 222                 align = pseudo_field.real_align
 223
 224         return align
 225
 226     def _to_public(self):
 227         fields = []
 228
 229         for name, pseudo_field in self.fields.items():
 230             fields.append((name, pseudo_field.to_public()))
 231
 232         return metadata.Struct(self.min_align, collections.OrderedDict(fields))
 233
 234
 235 class _Trace(_PseudoObj):
 236     def __init__(self):
 237         super().__init__()
 238         self.byte_order = None
 239         self.uuid = None
 240         self.packet_header_type = None
 241
 242     def _to_public(self):
 243         return metadata.Trace(self.byte_order, self.uuid,
 244                               _opt_to_public(self.packet_header_type))
 245
 246
 247 class _Clock(_PseudoObj):
 248     def __init__(self):
 249         super().__init__()
 250         self.name = None
 251         self.uuid = None
 252         self.description = None
 253         self.freq = int(1e9)
 254         self.error_cycles = 0
 255         self.offset_seconds = 0
 256         self.offset_cycles = 0
 257         self.absolute = False
 258         self.return_ctype = 'uint32_t'
 259
 260     def _to_public(self):
 261         return metadata.Clock(self.name, self.uuid, self.description, self.freq,
 262                               self.error_cycles, self.offset_seconds,
 263                               self.offset_cycles, self.absolute,
 264                               self.return_ctype)
 265
 266
 267 class _Event(_PseudoObj):
 268     def __init__(self):
 269         super().__init__()
 270         self.id = None
 271         self.name = None
 272         self.log_level = None
 273         self.payload_type = None
 274         self.context_type = None
 275
 276     def _to_public(self):
 277         return metadata.Event(self.id, self.name, self.log_level,
 278                               _opt_to_public(self.payload_type),
 279                               _opt_to_public(self.context_type))
 280
 281
 282 class _Stream(_PseudoObj):
 283     def __init__(self):
 284         super().__init__()
 285         self.id = None
 286         self.name = None
 287         self.packet_context_type = None
 288         self.event_header_type = None
 289         self.event_context_type = None
 290         self.events = collections.OrderedDict()
 291
 292     def is_event_empty(self, event):
 293         total_fields = 0
 294
 295         if self.event_header_type is not None:
 296             total_fields += len(self.event_header_type.fields)
 297
 298         if self.event_context_type is not None:
 299             total_fields += len(self.event_context_type.fields)
 300
 301         if event.context_type is not None:
 302             total_fields += len(event.context_type.fields)
 303
 304         if event.payload_type is not None:
 305             total_fields += len(event.payload_type.fields)
 306
 307         return total_fields == 0
 308
 309     def _to_public(self):
 310         events = []
 311
 312         for name, pseudo_ev in self.events.items():
 313             events.append((name, pseudo_ev.to_public()))
 314
 315         return metadata.Stream(self.id, self.name,
 316                                _opt_to_public(self.packet_context_type),
 317                                _opt_to_public(self.event_header_type),
 318                                _opt_to_public(self.event_context_type),
 319                                collections.OrderedDict(events))
 320
 321
 322 class _Metadata(_PseudoObj):
 323     def __init__(self):
 324         super().__init__()
 325         self.trace = None
 326         self.env = None
 327         self.clocks = None
 328         self.streams = None
 329         self.default_stream_name = None
 330
 331     def _to_public(self):
 332         clocks = []
 333
 334         for name, pseudo_clock in self.clocks.items():
 335             clocks.append((name, pseudo_clock.to_public()))
 336
 337         streams = []
 338
 339         for name, pseudo_stream in self.streams.items():
 340             streams.append((name, pseudo_stream.to_public()))
 341
 342         return metadata.Metadata(self.trace.to_public(), self.env,
 343                                  collections.OrderedDict(clocks),
 344                                  collections.OrderedDict(streams),
 345                                  self.default_stream_name)
 346
 347
 348 # This JSON schema reference resolver only serves to detect when it
 349 # needs to resolve a remote URI.
 350 #
 351 # This must never happen in barectf because all our schemas are local;
 352 # it would mean a programming or schema error.
 353 class _RefResolver(jsonschema.RefResolver):
 354     def resolve_remote(self, uri):
 355         raise RuntimeError(f'Missing local schema with URI `{uri}`')
 356
 357
 358 # Schema validator which considers all the schemas found in the barectf
 359 # package's `schemas` directory.
 360 #
 361 # The only public method is validate() which accepts an instance to
 362 # validate as well as a schema short ID.
 363 class _SchemaValidator:
 364     def __init__(self):
 365         subdirs = ['config', os.path.join('2', 'config')]
 366         schemas_dir = pkg_resources.resource_filename(__name__, 'schemas')
 367         self._store = {}
 368
 369         for subdir in subdirs:
 370             dir = os.path.join(schemas_dir, subdir)
 371
 372             for file_name in os.listdir(dir):
 373                 if not file_name.endswith('.yaml'):
 374                     continue
 375
 376                 with open(os.path.join(dir, file_name)) as f:
 377                     schema = yaml.load(f, Loader=yaml.SafeLoader)
 378
 379                 assert '$id' in schema
 380                 schema_id = schema['$id']
 381                 assert schema_id not in self._store
 382                 self._store[schema_id] = schema
 383
 384     @staticmethod
 385     def _dict_from_ordered_dict(o_dict):
 386         dct = {}
 387
 388         for k, v in o_dict.items():
 389             new_v = v
 390
 391             if type(v) is collections.OrderedDict:
 392                 new_v = _SchemaValidator._dict_from_ordered_dict(v)
 393
 394             dct[k] = new_v
 395
 396         return dct
 397
 398     def _validate(self, instance, schema_short_id):
 399         # retrieve full schema ID from short ID
 400         schema_id = f'https://barectf.org/schemas/{schema_short_id}.json'
 401         assert schema_id in self._store
 402
 403         # retrieve full schema
 404         schema = self._store[schema_id]
 405
 406         # Create a reference resolver for this schema using this
 407         # validator's schema store.
 408         resolver = _RefResolver(base_uri=schema_id, referrer=schema,
 409                                 store=self._store)
 410
 411         # create a JSON schema validator using this reference resolver
 412         validator = jsonschema.Draft7Validator(schema, resolver=resolver)
 413
 414         # Validate the instance, converting its
 415         # `collections.OrderedDict` objects to `dict` objects so as to
 416         # make any error message easier to read (because
 417         # validator.validate() below uses str() for error messages, and
 418         # collections.OrderedDict.__str__() returns a somewhat bulky
 419         # representation).
 420         validator.validate(self._dict_from_ordered_dict(instance))
 421
 422     # Validates `instance` using the schema having the short ID
 423     # `schema_short_id`.
 424     #
 425     # A schema short ID is the part between `schemas/` and `.json` in
 426     # its URI.
 427     #
 428     # Raises a `_ConfigParseError` object, hiding any `jsonschema`
 429     # exception, on validation failure.
 430     def validate(self, instance, schema_short_id):
 431         try:
 432             self._validate(instance, schema_short_id)
 433         except jsonschema.ValidationError as exc:
 434             # convert to barectf `_ConfigParseError` exception
 435             contexts = ['Configuration object']
 436
 437             # Each element of the instance's absolute path is either an
 438             # integer (array element's index) or a string (object
 439             # property's name).
 440             for elem in exc.absolute_path:
 441                 if type(elem) is int:
 442                     ctx = f'Element {elem}'
 443                 else:
 444                     ctx = f'`{elem}` property'
 445
 446                 contexts.append(ctx)
 447
 448             schema_ctx = ''
 449
 450             if len(exc.context) > 0:
 451                 # According to the documentation of
 452                 # jsonschema.ValidationError.context(),
 453                 # the method returns a
 454                 #
 455                 # > list of errors from the subschemas
 456                 #
 457                 # This contains additional information about the
 458                 # validation failure which can help the user figure out
 459                 # what's wrong exactly.
 460                 #
 461                 # Join each message with `; ` and append this to our
 462                 # configuration parsing error's message.
 463                 msgs = '; '.join([e.message for e in exc.context])
 464                 schema_ctx = f': {msgs}'
 465
 466             new_exc = _ConfigParseError(contexts.pop(),
 467                                         f'{exc.message}{schema_ctx} (from schema `{schema_short_id}`)')
 468
 469             for ctx in reversed(contexts):
 470                 new_exc.append_ctx(ctx)
 471
 472             raise new_exc
 473
 474
 475 # Converts the byte order string `bo_str` to a `metadata.ByteOrder`
 476 # enumerator.
 477 def _byte_order_str_to_bo(bo_str):
 478     bo_str = bo_str.lower()
 479
 480     if bo_str == 'le':
 481         return metadata.ByteOrder.LE
 482     elif bo_str == 'be':
 483         return metadata.ByteOrder.BE
 484
 485
 486 # Converts the encoding string `encoding_str` to a `metadata.Encoding`
 487 # enumerator.
 488 def _encoding_str_to_encoding(encoding_str):
 489     encoding_str = encoding_str.lower()
 490
 491     if encoding_str == 'utf-8' or encoding_str == 'utf8':
 492         return metadata.Encoding.UTF8
 493     elif encoding_str == 'ascii':
 494         return metadata.Encoding.ASCII
 495     elif encoding_str == 'none':
 496         return metadata.Encoding.NONE
 497
 498
 499 # Validates the TSDL identifier `iden`, raising a `_ConfigParseError`
 500 # exception using `ctx_obj_name` and `prop` to format the message if
 501 # it's invalid.
 502 def _validate_identifier(iden, ctx_obj_name, prop):
 503     assert type(iden) is str
 504     ctf_keywords = {
 505         'align',
 506         'callsite',
 507         'clock',
 508         'enum',
 509         'env',
 510         'event',
 511         'floating_point',
 512         'integer',
 513         'stream',
 514         'string',
 515         'struct',
 516         'trace',
 517         'typealias',
 518         'typedef',
 519         'variant',
 520     }
 521
 522     if iden in ctf_keywords:
 523         msg = f'Invalid {prop} (not a valid identifier): `{iden}`'
 524         raise _ConfigParseError(ctx_obj_name, msg)
 525
 526
 527 # Validates the alignment `align`, raising a `_ConfigParseError`
 528 # exception using `ctx_obj_name` if it's invalid.
 529 def _validate_alignment(align, ctx_obj_name):
 530     assert align >= 1
 531
 532     if (align & (align - 1)) != 0:
 533         raise _ConfigParseError(ctx_obj_name,
 534                                 f'Invalid alignment (not a power of two): {align}')
 535
 536
 537 # Entities.
 538 #
 539 # Order of values is important here.
 540 @enum.unique
 541 class _Entity(enum.IntEnum):
 542     TRACE_PACKET_HEADER = 0
 543     STREAM_PACKET_CONTEXT = 1
 544     STREAM_EVENT_HEADER = 2
 545     STREAM_EVENT_CONTEXT = 3
 546     EVENT_CONTEXT = 4
 547     EVENT_PAYLOAD = 5
 548
 549
 550 # A validator which validates the configured metadata for barectf
 551 # specific needs.
 552 #
 553 # barectf needs:
 554 #
 555 # * The alignments of all header/context field types are at least 8.
 556 #
 557 # * There are no nested structure or array field types, except the
 558 #   packet header field type's `uuid` field
 559 #
 560 class _BarectfMetadataValidator:
 561     def __init__(self):
 562         self._type_to_validate_type_func = {
 563             _Struct: self._validate_struct_type,
 564             _Array: self._validate_array_type,
 565         }
 566
 567     def _validate_struct_type(self, t, entity_root):
 568         if not entity_root:
 569             raise _ConfigParseError('Structure field type',
 570                                     'Inner structure field types are not supported as of this version')
 571
 572         for field_name, field_type in t.fields.items():
 573             if entity_root and self._cur_entity is _Entity.TRACE_PACKET_HEADER:
 574                 if field_name == 'uuid':
 575                     # allow
 576                     continue
 577
 578             try:
 579                 self._validate_type(field_type, False)
 580             except _ConfigParseError as exc:
 581                 _append_error_ctx(exc,
 582                                   f'Structure field type\'s field `{field_name}`')
 583
 584     def _validate_array_type(self, t, entity_root):
 585         raise _ConfigParseError('Array field type',
 586                                 'Not supported as of this version')
 587
 588     def _validate_type(self, t, entity_root):
 589         func = self._type_to_validate_type_func.get(type(t))
 590
 591         if func is not None:
 592             func(t, entity_root)
 593
 594     def _validate_entity(self, t):
 595         if t is None:
 596             return
 597
 598         # make sure root field type has a real alignment of at least 8
 599         if t.real_align < 8:
 600             raise _ConfigParseError('Root field type',
 601                                     f'Effective alignment must be at least 8 (got {t.real_align})')
 602
 603         assert type(t) is _Struct
 604
 605         # validate field types
 606         self._validate_type(t, True)
 607
 608     def _validate_event_entities_and_names(self, stream, ev):
 609         try:
 610             _validate_identifier(ev.name, 'Event type', 'event type name')
 611
 612             self._cur_entity = _Entity.EVENT_CONTEXT
 613
 614             try:
 615                 self._validate_entity(ev.context_type)
 616             except _ConfigParseError as exc:
 617                 _append_error_ctx(exc, 'Event type',
 618                                   'Invalid context field type')
 619
 620             self._cur_entity = _Entity.EVENT_PAYLOAD
 621
 622             try:
 623                 self._validate_entity(ev.payload_type)
 624             except _ConfigParseError as exc:
 625                 _append_error_ctx(exc, 'Event type',
 626                                   'Invalid payload field type')
 627
 628             if stream.is_event_empty(ev):
 629                 raise _ConfigParseError('Event type', 'Empty')
 630         except _ConfigParseError as exc:
 631             _append_error_ctx(exc, f'Event type `{ev.name}`')
 632
 633     def _validate_stream_entities_and_names(self, stream):
 634         try:
 635             _validate_identifier(stream.name, 'Stream type', 'stream type name')
 636             self._cur_entity = _Entity.STREAM_PACKET_CONTEXT
 637
 638             try:
 639                 self._validate_entity(stream.packet_context_type)
 640             except _ConfigParseError as exc:
 641                 _append_error_ctx(exc, 'Stream type',
 642                                   'Invalid packet context field type')
 643
 644             self._cur_entity = _Entity.STREAM_EVENT_HEADER
 645
 646             try:
 647                 self._validate_entity(stream.event_header_type)
 648             except _ConfigParseError as exc:
 649                 _append_error_ctx(exc, 'Stream type',
 650                                   'Invalid event header field type')
 651
 652             self._cur_entity = _Entity.STREAM_EVENT_CONTEXT
 653
 654             try:
 655                 self._validate_entity(stream.event_context_type)
 656             except _ConfigParseError as exc:
 657                 _append_error_ctx(exc, 'Stream type',
 658                                   'Invalid event context field type')
 659
 660             for ev in stream.events.values():
 661                 self._validate_event_entities_and_names(stream, ev)
 662         except _ConfigParseError as exc:
 663             _append_error_ctx(exc, f'Stream type `{stream.name}`')
 664
 665     def _validate_entities_and_names(self, meta):
 666         self._cur_entity = _Entity.TRACE_PACKET_HEADER
 667
 668         try:
 669             self._validate_entity(meta.trace.packet_header_type)
 670         except _ConfigParseError as exc:
 671             _append_error_ctx(exc, 'Trace type',
 672                               'Invalid packet header field type')
 673
 674         for stream in meta.streams.values():
 675             self._validate_stream_entities_and_names(stream)
 676
 677     def _validate_default_stream(self, meta):
 678         if meta.default_stream_name is not None:
 679             if meta.default_stream_name not in meta.streams.keys():
 680                 msg = f'Default stream type name (`{meta.default_stream_name}`) does not name an existing stream type'
 681                 raise _ConfigParseError('Metadata', msg)
 682
 683     def validate(self, meta):
 684         try:
 685             self._validate_entities_and_names(meta)
 686             self._validate_default_stream(meta)
 687         except _ConfigParseError as exc:
 688             _append_error_ctx(exc, 'barectf metadata')
 689
 690
 691 # A validator which validates special fields of trace, stream, and event
 692 # types.
 693 class _MetadataSpecialFieldsValidator:
 694     # Validates the packet header field type `t`.
 695     def _validate_trace_packet_header_type(self, t):
 696         ctx_obj_name = '`packet-header-type` property'
 697
 698         # If there's more than one stream type, then the `stream_id`
 699         # (stream type ID) field is required.
 700         if len(self._meta.streams) > 1:
 701             if t is None:
 702                 raise _ConfigParseError('Trace type',
 703                                         '`stream_id` field is required (because there\'s more than one stream type), but packet header field type is missing')
 704
 705             if 'stream_id' not in t.fields:
 706                 raise _ConfigParseError(ctx_obj_name,
 707                                         '`stream_id` field is required (because there\'s more than one stream type)')
 708
 709         if t is None:
 710             return
 711
 712         # The `magic` field type must be the first one.
 713         #
 714         # The `stream_id` field type's size (bits) must be large enough
 715         # to accomodate any stream type ID.
 716         for i, (field_name, field_type) in enumerate(t.fields.items()):
 717             if field_name == 'magic':
 718                 if i != 0:
 719                     raise _ConfigParseError(ctx_obj_name,
 720                                             '`magic` field must be the first packet header field type\'s field')
 721             elif field_name == 'stream_id':
 722                 if len(self._meta.streams) > (1 << field_type.size):
 723                     raise _ConfigParseError(ctx_obj_name,
 724                                             f'`stream_id` field\'s size is too small to accomodate {len(self._meta.streams)} stream types')
 725
 726     # Validates the trace type of the metadata object `meta`.
 727     def _validate_trace(self, meta):
 728         self._validate_trace_packet_header_type(meta.trace.packet_header_type)
 729
 730     # Validates the packet context field type of the stream type
 731     # `stream`.
 732     def _validate_stream_packet_context(self, stream):
 733         ctx_obj_name = '`packet-context-type` property'
 734         t = stream.packet_context_type
 735         assert t is not None
 736
 737         # The `timestamp_begin` and `timestamp_end` field types must be
 738         # mapped to the `value` property of the same clock.
 739         ts_begin = t.fields.get('timestamp_begin')
 740         ts_end = t.fields.get('timestamp_end')
 741
 742         if ts_begin is not None and ts_end is not None:
 743             if ts_begin.property_mappings[0].object.name != ts_end.property_mappings[0].object.name:
 744                 raise _ConfigParseError(ctx_obj_name,
 745                                         '`timestamp_begin` and `timestamp_end` fields must be mapped to the same clock value')
 746
 747         # The `packet_size` field type's size must be greater than or
 748         # equal to the `content_size` field type's size.
 749         if t.fields['content_size'].size > t.fields['packet_size'].size:
 750             raise _ConfigParseError(ctx_obj_name,
 751                                     '`content_size` field\'s size must be less than or equal to `packet_size` field\'s size')
 752
 753     # Validates the event header field type of the stream type `stream`.
 754     def _validate_stream_event_header(self, stream):
 755         ctx_obj_name = '`event-header-type` property'
 756         t = stream.event_header_type
 757
 758         # If there's more than one event type, then the `id` (event type
 759         # ID) field is required.
 760         if len(stream.events) > 1:
 761             if t is None:
 762                 raise _ConfigParseError('Stream type',
 763                                         '`id` field is required (because there\'s more than one event type), but event header field type is missing')
 764
 765             if 'id' not in t.fields:
 766                 raise _ConfigParseError(ctx_obj_name,
 767                                         '`id` field is required (because there\'s more than one event type)')
 768
 769         if t is None:
 770             return
 771
 772         # The `id` field type's size (bits) must be large enough to
 773         # accomodate any event type ID.
 774         eid = t.fields.get('id')
 775
 776         if eid is not None:
 777             if len(stream.events) > (1 << eid.size):
 778                 raise _ConfigParseError(ctx_obj_name,
 779                                         f'`id` field\'s size is too small to accomodate {len(stream.events)} event types')
 780
 781     # Validates the stream type `stream`.
 782     def _validate_stream(self, stream):
 783         self._validate_stream_packet_context(stream)
 784         self._validate_stream_event_header(stream)
 785
 786     # Validates the trace and stream types of the metadata object
 787     # `meta`.
 788     def validate(self, meta):
 789         self._meta = meta
 790
 791         try:
 792             try:
 793                 self._validate_trace(meta)
 794             except _ConfigParseError as exc:
 795                 _append_error_ctx(exc, 'Trace type')
 796
 797             for stream in meta.streams.values():
 798                 try:
 799                     self._validate_stream(stream)
 800                 except _ConfigParseError as exc:
 801                     _append_error_ctx(exc, f'Stream type `{stream.name}`')
 802         except _ConfigParseError as exc:
 803             _append_error_ctx(exc, 'Metadata')
 804
 805
 806 # A barectf YAML configuration parser.
 807 #
 808 # When you build such a parser, it parses the configuration file and
 809 # creates a corresponding `config.Config` object which you can get with
 810 # the `config` property.
 811 #
 812 # See the comments of _parse() for more implementation details about the
 813 # parsing stages and general strategy.
 814 class _YamlConfigParser:
 815     # Builds a barectf YAML configuration parser and parses the
 816     # configuration file having the path `path`.
 817     #
 818     # The parser considers the inclusion directories `include_dirs`,
 819     # ignores nonexistent inclusion files if `ignore_include_not_found`
 820     # is `True`, and dumps the effective configuration (as YAML) if
 821     # `dump_config` is `True`.
 822     #
 823     # Raises `_ConfigParseError` on parsing error.
 824     def __init__(self, path, include_dirs, ignore_include_not_found,
 825                  dump_config):
 826         self._root_path = path
 827         self._class_name_to_create_field_type_func = {
 828             'int': self._create_integer_field_type,
 829             'integer': self._create_integer_field_type,
 830             'flt': self._create_float_field_type,
 831             'float': self._create_float_field_type,
 832             'floating-point': self._create_float_field_type,
 833             'enum': self._create_enum_field_type,
 834             'enumeration': self._create_enum_field_type,
 835             'str': self._create_string_field_type,
 836             'string': self._create_string_field_type,
 837             'struct': self._create_struct_field_type,
 838             'structure': self._create_struct_field_type,
 839             'array': self._create_array_field_type,
 840         }
 841         self._include_dirs = include_dirs
 842         self._ignore_include_not_found = ignore_include_not_found
 843         self._dump_config = dump_config
 844         self._schema_validator = _SchemaValidator()
 845         self._parse()
 846
 847     # Sets the default byte order as found in the `metadata_node` node.
 848     def _set_byte_order(self, metadata_node):
 849         self._bo = _byte_order_str_to_bo(metadata_node['trace']['byte-order'])
 850         assert self._bo is not None
 851
 852     # Sets the clock value property mapping of the pseudo integer field
 853     # type object `int_obj` as found in the `prop_mapping_node` node.
 854     def _set_int_clock_prop_mapping(self, int_obj, prop_mapping_node):
 855         clock_name = prop_mapping_node['name']
 856         clock = self._clocks.get(clock_name)
 857
 858         if clock is None:
 859             exc = _ConfigParseError('`property-mappings` property',
 860                                     f'Clock type `{clock_name}` does not exist')
 861             exc.append_ctx('Integer field type')
 862             raise exc
 863
 864         prop_mapping = _PropertyMapping()
 865         prop_mapping.object = clock
 866         prop_mapping.prop = 'value'
 867         int_obj.property_mappings.append(prop_mapping)
 868
 869     # Creates a pseudo integer field type from the node `node` and
 870     # returns it.
 871     def _create_integer_field_type(self, node):
 872         obj = _Integer()
 873         obj.size = node['size']
 874         align_node = node.get('align')
 875
 876         if align_node is not None:
 877             _validate_alignment(align_node, 'Integer field type')
 878             obj.align = align_node
 879
 880         signed_node = node.get('signed')
 881
 882         if signed_node is not None:
 883             obj.signed = signed_node
 884
 885         obj.byte_order = self._bo
 886         bo_node = node.get('byte-order')
 887
 888         if bo_node is not None:
 889             obj.byte_order = _byte_order_str_to_bo(bo_node)
 890
 891         base_node = node.get('base')
 892
 893         if base_node is not None:
 894             if base_node == 'bin':
 895                 obj.base = 2
 896             elif base_node == 'oct':
 897                 obj.base = 8
 898             elif base_node == 'dec':
 899                 obj.base = 10
 900             else:
 901                 assert base_node == 'hex'
 902                 obj.base = 16
 903
 904         encoding_node = node.get('encoding')
 905
 906         if encoding_node is not None:
 907             obj.encoding = _encoding_str_to_encoding(encoding_node)
 908
 909         pm_node = node.get('property-mappings')
 910
 911         if pm_node is not None:
 912             assert len(pm_node) == 1
 913             self._set_int_clock_prop_mapping(obj, pm_node[0])
 914
 915         return obj
 916
 917     # Creates a pseudo floating point number field type from the node
 918     # `node` and returns it.
 919     def _create_float_field_type(self, node):
 920         obj = _FloatingPoint()
 921         size_node = node['size']
 922         obj.exp_size = size_node['exp']
 923         obj.mant_size = size_node['mant']
 924         align_node = node.get('align')
 925
 926         if align_node is not None:
 927             _validate_alignment(align_node, 'Floating point number field type')
 928             obj.align = align_node
 929
 930         obj.byte_order = self._bo
 931         bo_node = node.get('byte-order')
 932
 933         if bo_node is not None:
 934             obj.byte_order = _byte_order_str_to_bo(bo_node)
 935
 936         return obj
 937
 938     # Creates a pseudo enumeration field type from the node `node` and
 939     # returns it.
 940     def _create_enum_field_type(self, node):
 941         ctx_obj_name = 'Enumeration field type'
 942         obj = _Enum()
 943
 944         # value (integer) field type
 945         try:
 946             obj.value_type = self._create_type(node['value-type'])
 947         except _ConfigParseError as exc:
 948             _append_error_ctx(exc, ctx_obj_name,
 949                               'Cannot create value (integer) field type')
 950
 951         # members
 952         members_node = node.get('members')
 953
 954         if members_node is not None:
 955             if obj.value_type.signed:
 956                 value_min = -(1 << obj.value_type.size - 1)
 957                 value_max = (1 << (obj.value_type.size - 1)) - 1
 958             else:
 959                 value_min = 0
 960                 value_max = (1 << obj.value_type.size) - 1
 961
 962             cur = 0
 963
 964             for m_node in members_node:
 965                 if type(m_node) is str:
 966                     label = m_node
 967                     value = (cur, cur)
 968                     cur += 1
 969                 else:
 970                     assert type(m_node) is collections.OrderedDict
 971                     label = m_node['label']
 972                     value = m_node['value']
 973
 974                     if type(value) is int:
 975                         cur = value + 1
 976                         value = (value, value)
 977                     else:
 978                         assert type(value) is list
 979                         assert len(value) == 2
 980                         mn = value[0]
 981                         mx = value[1]
 982
 983                         if mn > mx:
 984                             exc = _ConfigParseError(ctx_obj_name)
 985                             exc.append_ctx(f'Member `{label}`',
 986                                            f'Invalid integral range ({mn} > {mx})')
 987                             raise exc
 988
 989                         value = (mn, mx)
 990                         cur = mx + 1
 991
 992                 # Make sure that all the integral values of the range
 993                 # fits the enumeration field type's integer value field
 994                 # type depending on its size (bits).
 995                 member_obj_name = f'Member `{label}`'
 996                 msg = f'Value {value[0]} is outside the value type range [{value_min}, {value_max}]'
 997
 998                 try:
 999                     if value[0] < value_min or value[0] > value_max:
1000                         raise _ConfigParseError(member_obj_name, msg)
1001
1002                     if value[1] < value_min or value[1] > value_max:
1003                         raise _ConfigParseError(member_obj_name, msg)
1004                 except _ConfigParseError as exc:
1005                     _append_error_ctx(exc, ctx_obj_name)
1006
1007                 obj.members[label] = value
1008
1009         return obj
1010
1011     # Creates a pseudo string field type from the node `node` and
1012     # returns it.
1013     def _create_string_field_type(self, node):
1014         obj = _String()
1015         encoding_node = node.get('encoding')
1016
1017         if encoding_node is not None:
1018             obj.encoding = _encoding_str_to_encoding(encoding_node)
1019
1020         return obj
1021
1022     # Creates a pseudo structure field type from the node `node` and
1023     # returns it.
1024     def _create_struct_field_type(self, node):
1025         ctx_obj_name = 'Structure field type'
1026         obj = _Struct()
1027         min_align_node = node.get('min-align')
1028
1029         if min_align_node is not None:
1030             _validate_alignment(min_align_node, ctx_obj_name)
1031             obj.min_align = min_align_node
1032
1033         fields_node = node.get('fields')
1034
1035         if fields_node is not None:
1036             for field_name, field_node in fields_node.items():
1037                 _validate_identifier(field_name, ctx_obj_name, 'field name')
1038
1039                 try:
1040                     obj.fields[field_name] = self._create_type(field_node)
1041                 except _ConfigParseError as exc:
1042                     _append_error_ctx(exc, ctx_obj_name,
1043                                       f'Cannot create field `{field_name}`')
1044
1045         return obj
1046
1047     # Creates a pseudo array field type from the node `node` and returns
1048     # it.
1049     def _create_array_field_type(self, node):
1050         obj = _Array()
1051         obj.length = node['length']
1052
1053         try:
1054             obj.element_type = self._create_type(node['element-type'])
1055         except _ConfigParseError as exc:
1056             _append_error_ctx(exc, 'Array field type',
1057                               'Cannot create element field type')
1058
1059         return obj
1060
1061     # Creates a pseudo field type from the node `node` and returns it.
1062     #
1063     # This method checks the `class` property of `node` to determine
1064     # which function of `self._class_name_to_create_field_type_func` to
1065     # call to create the corresponding pseudo field type.
1066     def _create_type(self, type_node):
1067         return self._class_name_to_create_field_type_func[type_node['class']](type_node)
1068
1069     # Creates a pseudo clock type from the node `node` and returns it.
1070     def _create_clock(self, node):
1071         clock = _Clock()
1072         uuid_node = node.get('uuid')
1073
1074         if uuid_node is not None:
1075             try:
1076                 clock.uuid = uuid.UUID(uuid_node)
1077             except ValueError as exc:
1078                 raise _ConfigParseError('Clock type',
1079                                         f'Malformed UUID `{uuid_node}`: {exc}')
1080
1081         descr_node = node.get('description')
1082
1083         if descr_node is not None:
1084             clock.description = descr_node
1085
1086         freq_node = node.get('freq')
1087
1088         if freq_node is not None:
1089             clock.freq = freq_node
1090
1091         error_cycles_node = node.get('error-cycles')
1092
1093         if error_cycles_node is not None:
1094             clock.error_cycles = error_cycles_node
1095
1096         offset_node = node.get('offset')
1097
1098         if offset_node is not None:
1099             offset_cycles_node = offset_node.get('cycles')
1100
1101             if offset_cycles_node is not None:
1102                 clock.offset_cycles = offset_cycles_node
1103
1104             offset_seconds_node = offset_node.get('seconds')
1105
1106             if offset_seconds_node is not None:
1107                 clock.offset_seconds = offset_seconds_node
1108
1109         absolute_node = node.get('absolute')
1110
1111         if absolute_node is not None:
1112             clock.absolute = absolute_node
1113
1114         return_ctype_node = node.get('$return-ctype')
1115
1116         if return_ctype_node is None:
1117             # barectf 2.1: `return-ctype` property was renamed to
1118             # `$return-ctype`
1119             return_ctype_node = node.get('return-ctype')
1120
1121         if return_ctype_node is not None:
1122             clock.return_ctype = return_ctype_node
1123
1124         return clock
1125
1126     # Registers all the clock types of the metadata node
1127     # `metadata_node`, creating pseudo clock types during the process,
1128     # within this parser.
1129     #
1130     # The pseudo clock types in `self._clocks` are then accessible when
1131     # creating a pseudo integer field type (see
1132     # _create_integer_field_type() and _set_int_clock_prop_mapping()).
1133     def _register_clocks(self, metadata_node):
1134         self._clocks = collections.OrderedDict()
1135         clocks_node = metadata_node.get('clocks')
1136
1137         if clocks_node is None:
1138             return
1139
1140         for clock_name, clock_node in clocks_node.items():
1141             _validate_identifier(clock_name, 'Metadata', 'clock type name')
1142             assert clock_name not in self._clocks
1143
1144             try:
1145                 clock = self._create_clock(clock_node)
1146             except _ConfigParseError as exc:
1147                 _append_error_ctx(exc, 'Metadata',
1148                                   f'Cannot create clock type `{clock}`')
1149
1150             clock.name = clock_name
1151             self._clocks[clock_name] = clock
1152
1153     # Creates an environment object (`collections.OrderedDict`) from the
1154     # metadata node `metadata_node` and returns it.
1155     def _create_env(self, metadata_node):
1156         env_node = metadata_node.get('env')
1157
1158         if env_node is None:
1159             return collections.OrderedDict()
1160
1161         for env_name, env_value in env_node.items():
1162             _validate_identifier(env_name, 'Metadata',
1163                                  'environment variable name')
1164
1165         return copy.deepcopy(env_node)
1166
1167     # Creates a pseudo trace type from the metadata node `metadata_node`
1168     # and returns it.
1169     def _create_trace(self, metadata_node):
1170         ctx_obj_name = 'Trace type'
1171         trace = _Trace()
1172         trace_node = metadata_node['trace']
1173         trace.byte_order = self._bo
1174         uuid_node = trace_node.get('uuid')
1175
1176         if uuid_node is not None:
1177             # The `uuid` property of the trace type node can be `auto`
1178             # to make barectf generate a UUID.
1179             if uuid_node == 'auto':
1180                 trace.uuid = uuid.uuid1()
1181             else:
1182                 try:
1183                     trace.uuid = uuid.UUID(uuid_node)
1184                 except ValueError as exc:
1185                     raise _ConfigParseError(ctx_obj_name,
1186                                             f'Malformed UUID `{uuid_node}`: {exc}')
1187
1188         pht_node = trace_node.get('packet-header-type')
1189
1190         if pht_node is not None:
1191             try:
1192                 trace.packet_header_type = self._create_type(pht_node)
1193             except _ConfigParseError as exc:
1194                 _append_error_ctx(exc, ctx_obj_name,
1195                                   'Cannot create packet header field type')
1196
1197         return trace
1198
1199     # Creates a pseudo event type from the event node `event_node` and
1200     # returns it.
1201     def _create_event(self, event_node):
1202         ctx_obj_name = 'Event type'
1203         event = _Event()
1204         log_level_node = event_node.get('log-level')
1205
1206         if log_level_node is not None:
1207             assert type(log_level_node) is int
1208             event.log_level = metadata.LogLevel(None, log_level_node)
1209
1210         ct_node = event_node.get('context-type')
1211
1212         if ct_node is not None:
1213             try:
1214                 event.context_type = self._create_type(ct_node)
1215             except _ConfigParseError as exc:
1216                 _append_error_ctx(exc, ctx_obj_name,
1217                                   'Cannot create context field type')
1218
1219         pt_node = event_node.get('payload-type')
1220
1221         if pt_node is not None:
1222             try:
1223                 event.payload_type = self._create_type(pt_node)
1224             except _ConfigParseError as exc:
1225                 _append_error_ctx(exc, ctx_obj_name,
1226                                   'Cannot create payload field type')
1227
1228         return event
1229
1230     # Creates a pseudo stream type named `stream_name` from the stream
1231     # node `stream_node` and returns it.
1232     def _create_stream(self, stream_name, stream_node):
1233         ctx_obj_name = 'Stream type'
1234         stream = _Stream()
1235         pct_node = stream_node.get('packet-context-type')
1236
1237         if pct_node is not None:
1238             try:
1239                 stream.packet_context_type = self._create_type(pct_node)
1240             except _ConfigParseError as exc:
1241                 _append_error_ctx(exc, ctx_obj_name,
1242                                   'Cannot create packet context field type')
1243
1244         eht_node = stream_node.get('event-header-type')
1245
1246         if eht_node is not None:
1247             try:
1248                 stream.event_header_type = self._create_type(eht_node)
1249             except _ConfigParseError as exc:
1250                 _append_error_ctx(exc, ctx_obj_name,
1251                                   'Cannot create event header field type')
1252
1253         ect_node = stream_node.get('event-context-type')
1254
1255         if ect_node is not None:
1256             try:
1257                 stream.event_context_type = self._create_type(ect_node)
1258             except _ConfigParseError as exc:
1259                 _append_error_ctx(exc, ctx_obj_name,
1260                                   'Cannot create event context field type')
1261
1262         events_node = stream_node['events']
1263         cur_id = 0
1264
1265         for ev_name, ev_node in events_node.items():
1266             try:
1267                 ev = self._create_event(ev_node)
1268             except _ConfigParseError as exc:
1269                 _append_error_ctx(exc, ctx_obj_name,
1270                                   f'Cannot create event type `{ev_name}`')
1271
1272             ev.id = cur_id
1273             ev.name = ev_name
1274             stream.events[ev_name] = ev
1275             cur_id += 1
1276
1277         default_node = stream_node.get('$default')
1278
1279         if default_node is not None:
1280             if self._meta.default_stream_name is not None and self._meta.default_stream_name != stream_name:
1281                 msg = f'Cannot specify more than one default stream type (default stream type already set to `{self._meta.default_stream_name}`)'
1282                 raise _ConfigParseError('Stream type', msg)
1283
1284             self._meta.default_stream_name = stream_name
1285
1286         return stream
1287
1288     # Creates a `collections.OrderedDict` object where keys are stream
1289     # type names and values are pseudo stream types from the metadata
1290     # node `metadata_node` and returns it.
1291     def _create_streams(self, metadata_node):
1292         streams = collections.OrderedDict()
1293         streams_node = metadata_node['streams']
1294         cur_id = 0
1295
1296         for stream_name, stream_node in streams_node.items():
1297             try:
1298                 stream = self._create_stream(stream_name, stream_node)
1299             except _ConfigParseError as exc:
1300                 _append_error_ctx(exc, 'Metadata',
1301                                   f'Cannot create stream type `{stream_name}`')
1302
1303             stream.id = cur_id
1304             stream.name = stream_name
1305             streams[stream_name] = stream
1306             cur_id += 1
1307
1308         return streams
1309
1310     # Creates a pseudo metadata object from the configuration node
1311     # `root` and returns it.
1312     def _create_metadata(self, root):
1313         self._meta = _Metadata()
1314         metadata_node = root['metadata']
1315
1316         if '$default-stream' in metadata_node and metadata_node['$default-stream'] is not None:
1317             default_stream_node = metadata_node['$default-stream']
1318             self._meta.default_stream_name = default_stream_node
1319
1320         self._set_byte_order(metadata_node)
1321         self._register_clocks(metadata_node)
1322         self._meta.clocks = self._clocks
1323         self._meta.env = self._create_env(metadata_node)
1324         self._meta.trace = self._create_trace(metadata_node)
1325         self._meta.streams = self._create_streams(metadata_node)
1326
1327         # validate the pseudo metadata object
1328         _MetadataSpecialFieldsValidator().validate(self._meta)
1329         _BarectfMetadataValidator().validate(self._meta)
1330
1331         return self._meta
1332
1333     # Gets and validates the tracing prefix as found in the
1334     # configuration node `config_node` and returns it.
1335     def _get_prefix(self, config_node):
1336         prefix = config_node.get('prefix', 'barectf_')
1337         _validate_identifier(prefix, '`prefix` property', 'prefix')
1338         return prefix
1339
1340     # Gets the options as found in the configuration node `config_node`
1341     # and returns a corresponding `config.ConfigOptions` object.
1342     def _get_options(self, config_node):
1343         gen_prefix_def = False
1344         gen_default_stream_def = False
1345         options_node = config_node.get('options')
1346
1347         if options_node is not None:
1348             gen_prefix_def = options_node.get('gen-prefix-def',
1349                                               gen_prefix_def)
1350             gen_default_stream_def = options_node.get('gen-default-stream-def',
1351                                                       gen_default_stream_def)
1352
1353         return config.ConfigOptions(gen_prefix_def, gen_default_stream_def)
1354
1355     # Returns the last included file name from the parser's inclusion
1356     # file name stack.
1357     def _get_last_include_file(self):
1358         if self._include_stack:
1359             return self._include_stack[-1]
1360
1361         return self._root_path
1362
1363     # Loads the inclusion file having the path `yaml_path` and returns
1364     # its content as a `collections.OrderedDict` object.
1365     def _load_include(self, yaml_path):
1366         for inc_dir in self._include_dirs:
1367             # Current inclusion dir + file name path.
1368             #
1369             # Note: os.path.join() only takes the last argument if it's
1370             # absolute.
1371             inc_path = os.path.join(inc_dir, yaml_path)
1372
1373             # real path (symbolic links resolved)
1374             real_path = os.path.realpath(inc_path)
1375
1376             # normalized path (weird stuff removed!)
1377             norm_path = os.path.normpath(real_path)
1378
1379             if not os.path.isfile(norm_path):
1380                 # file doesn't exist: skip
1381                 continue
1382
1383             if norm_path in self._include_stack:
1384                 base_path = self._get_last_include_file()
1385                 raise _ConfigParseError(f'File `{base_path}`',
1386                                         f'Cannot recursively include file `{norm_path}`')
1387
1388             self._include_stack.append(norm_path)
1389
1390             # load raw content
1391             return self._yaml_ordered_load(norm_path)
1392
1393         if not self._ignore_include_not_found:
1394             base_path = self._get_last_include_file()
1395             raise _ConfigParseError(f'File `{base_path}`',
1396                                     f'Cannot include file `{yaml_path}`: file not found in inclusion directories')
1397
1398     # Returns a list of all the inclusion file paths as found in the
1399     # inclusion node `include_node`.
1400     def _get_include_paths(self, include_node):
1401         if include_node is None:
1402             # none
1403             return []
1404
1405         if type(include_node) is str:
1406             # wrap as array
1407             return [include_node]
1408
1409         # already an array
1410         assert type(include_node) is list
1411         return include_node
1412
1413     # Updates the node `base_node` with an overlay node `overlay_node`.
1414     #
1415     # Both the inclusion and field type inheritance features use this
1416     # update mechanism.
1417     def _update_node(self, base_node, overlay_node):
1418         for olay_key, olay_value in overlay_node.items():
1419             if olay_key in base_node:
1420                 base_value = base_node[olay_key]
1421
1422                 if type(olay_value) is collections.OrderedDict and type(base_value) is collections.OrderedDict:
1423                     # merge both objects
1424                     self._update_node(base_value, olay_value)
1425                 elif type(olay_value) is list and type(base_value) is list:
1426                     # append extension array items to base items
1427                     base_value += olay_value
1428                 else:
1429                     # fall back to replacing base property
1430                     base_node[olay_key] = olay_value
1431             else:
1432                 # set base property from overlay property
1433                 base_node[olay_key] = olay_value
1434
1435     # Processes inclusions using `last_overlay_node` as the last overlay
1436     # node to use to "patch" the node.
1437     #
1438     # If `last_overlay_node` contains an `$include` property, then this
1439     # method patches the current base node (initially empty) in order
1440     # using the content of the inclusion files (recursively).
1441     #
1442     # At the end, this method removes the `$include` of
1443     # `last_overlay_node` and then patches the current base node with
1444     # its other properties before returning the result (always a deep
1445     # copy).
1446     def _process_node_include(self, last_overlay_node,
1447                               process_base_include_cb,
1448                               process_children_include_cb=None):
1449         # process children inclusions first
1450         if process_children_include_cb is not None:
1451             process_children_include_cb(last_overlay_node)
1452
1453         incl_prop_name = '$include'
1454
1455         if incl_prop_name in last_overlay_node:
1456             include_node = last_overlay_node[incl_prop_name]
1457         else:
1458             # no inclusions!
1459             return last_overlay_node
1460
1461         include_paths = self._get_include_paths(include_node)
1462         cur_base_path = self._get_last_include_file()
1463         base_node = None
1464
1465         # keep the inclusion paths and remove the `$include` property
1466         include_paths = copy.deepcopy(include_paths)
1467         del last_overlay_node[incl_prop_name]
1468
1469         for include_path in include_paths:
1470             # load raw YAML from included file
1471             overlay_node = self._load_include(include_path)
1472
1473             if overlay_node is None:
1474                 # Cannot find inclusion file, but we're ignoring those
1475                 # errors, otherwise _load_include() itself raises a
1476                 # config error.
1477                 continue
1478
1479             # recursively process inclusions
1480             try:
1481                 overlay_node = process_base_include_cb(overlay_node)
1482             except _ConfigParseError as exc:
1483                 _append_error_ctx(exc, f'File `{cur_base_path}`')
1484
1485             # pop inclusion stack now that we're done including
1486             del self._include_stack[-1]
1487
1488             # At this point, `base_node` is fully resolved (does not
1489             # contain any `$include` property).
1490             if base_node is None:
1491                 base_node = overlay_node
1492             else:
1493                 self._update_node(base_node, overlay_node)
1494
1495         # Finally, update the latest base node with our last overlay
1496         # node.
1497         if base_node is None:
1498             # Nothing was included, which is possible when we're
1499             # ignoring inclusion errors.
1500             return last_overlay_node
1501
1502         self._update_node(base_node, last_overlay_node)
1503         return base_node
1504
1505     # Process the inclusions of the event type node `event_node`,
1506     # returning the effective node.
1507     def _process_event_include(self, event_node):
1508         # Make sure the event type node is valid for the inclusion
1509         # processing stage.
1510         self._schema_validator.validate(event_node,
1511                                         '2/config/event-pre-include')
1512
1513         # process inclusions
1514         return self._process_node_include(event_node,
1515                                           self._process_event_include)
1516
1517     # Process the inclusions of the stream type node `stream_node`,
1518     # returning the effective node.
1519     def _process_stream_include(self, stream_node):
1520         def process_children_include(stream_node):
1521             if 'events' in stream_node:
1522                 events_node = stream_node['events']
1523
1524                 for key in list(events_node):
1525                     events_node[key] = self._process_event_include(events_node[key])
1526
1527         # Make sure the stream type node is valid for the inclusion
1528         # processing stage.
1529         self._schema_validator.validate(stream_node,
1530                                         '2/config/stream-pre-include')
1531
1532         # process inclusions
1533         return self._process_node_include(stream_node,
1534                                           self._process_stream_include,
1535                                           process_children_include)
1536
1537     # Process the inclusions of the trace type node `trace_node`,
1538     # returning the effective node.
1539     def _process_trace_include(self, trace_node):
1540         # Make sure the trace type node is valid for the inclusion
1541         # processing stage.
1542         self._schema_validator.validate(trace_node,
1543                                         '2/config/trace-pre-include')
1544
1545         # process inclusions
1546         return self._process_node_include(trace_node,
1547                                           self._process_trace_include)
1548
1549     # Process the inclusions of the clock type node `clock_node`,
1550     # returning the effective node.
1551     def _process_clock_include(self, clock_node):
1552         # Make sure the clock type node is valid for the inclusion
1553         # processing stage.
1554         self._schema_validator.validate(clock_node,
1555                                         '2/config/clock-pre-include')
1556
1557         # process inclusions
1558         return self._process_node_include(clock_node,
1559                                           self._process_clock_include)
1560
1561     # Process the inclusions of the metadata node `metadata_node`,
1562     # returning the effective node.
1563     def _process_metadata_include(self, metadata_node):
1564         def process_children_include(metadata_node):
1565             if 'trace' in metadata_node:
1566                 metadata_node['trace'] = self._process_trace_include(metadata_node['trace'])
1567
1568             if 'clocks' in metadata_node:
1569                 clocks_node = metadata_node['clocks']
1570
1571                 for key in list(clocks_node):
1572                     clocks_node[key] = self._process_clock_include(clocks_node[key])
1573
1574             if 'streams' in metadata_node:
1575                 streams_node = metadata_node['streams']
1576
1577                 for key in list(streams_node):
1578                     streams_node[key] = self._process_stream_include(streams_node[key])
1579
1580         # Make sure the metadata node is valid for the inclusion
1581         # processing stage.
1582         self._schema_validator.validate(metadata_node,
1583                                         '2/config/metadata-pre-include')
1584
1585         # process inclusions
1586         return self._process_node_include(metadata_node,
1587                                           self._process_metadata_include,
1588                                           process_children_include)
1589
1590     # Process the inclusions of the configuration node `config_node`,
1591     # returning the effective node.
1592     def _process_config_includes(self, config_node):
1593         # Process inclusions in this order:
1594         #
1595         # 1. Clock type node, event type nodes, and trace type nodes
1596         #    (the order between those is not important).
1597         #
1598         # 2. Stream type nodes.
1599         #
1600         # 3. Metadata node.
1601         #
1602         # This is because:
1603         #
1604         # * A metadata node can include clock type nodes, a trace type
1605         #   node, stream type nodes, and event type nodes (indirectly).
1606         #
1607         # * A stream type node can include event type nodes.
1608         #
1609         # We keep a stack of absolute paths to included files
1610         # (`self._include_stack`) to detect recursion.
1611         #
1612         # First, make sure the configuration object itself is valid for
1613         # the inclusion processing stage.
1614         self._schema_validator.validate(config_node,
1615                                         '2/config/config-pre-include')
1616
1617         # Process metadata node inclusions.
1618         #
1619         # self._process_metadata_include() returns a new (or the same)
1620         # metadata node without any `$include` property in it,
1621         # recursively.
1622         config_node['metadata'] = self._process_metadata_include(config_node['metadata'])
1623
1624         return config_node
1625
1626     # Expands the field type aliases found in the metadata node
1627     # `metadata_node` using the aliases of the `type_aliases_node` node.
1628     #
1629     # This method modifies `metadata_node`.
1630     #
1631     # When this method returns:
1632     #
1633     # * Any field type alias is replaced with its full field type
1634     #   equivalent.
1635     #
1636     # * The `type-aliases` property of `metadata_node` is removed.
1637     def _expand_field_type_aliases(self, metadata_node, type_aliases_node):
1638         def resolve_field_type_aliases(parent_node, key, from_descr,
1639                                        alias_set=None):
1640             if key not in parent_node:
1641                 return
1642
1643             # This set holds all the aliases we need to expand,
1644             # recursively. This is used to detect cycles.
1645             if alias_set is None:
1646                 alias_set = set()
1647
1648             node = parent_node[key]
1649
1650             if node is None:
1651                 return
1652
1653             if type(node) is str:
1654                 alias = node
1655
1656                 if alias not in resolved_aliases:
1657                     # Only check for a field type alias cycle when we
1658                     # didn't resolve the alias yet, as a given node can
1659                     # refer to the same field type alias more than once.
1660                     if alias in alias_set:
1661                         msg = f'Cycle detected during the `{alias}` field type alias resolution'
1662                         raise _ConfigParseError(from_descr, msg)
1663
1664                     # try to load field type alias node named `alias`
1665                     if alias not in type_aliases_node:
1666                         raise _ConfigParseError(from_descr,
1667                                                 f'Field type alias `{alias}` does not exist')
1668
1669                     # resolve it
1670                     alias_set.add(alias)
1671                     resolve_field_type_aliases(type_aliases_node, alias,
1672                                                from_descr, alias_set)
1673                     resolved_aliases.add(alias)
1674
1675                 parent_node[key] = copy.deepcopy(type_aliases_node[node])
1676                 return
1677
1678             # traverse, resolving field type aliases as needed
1679             for pkey in ['$inherit', 'inherit', 'value-type', 'element-type']:
1680                 resolve_field_type_aliases(node, pkey, from_descr, alias_set)
1681
1682             # structure field type fields
1683             pkey = 'fields'
1684
1685             if pkey in node:
1686                 assert type(node[pkey]) is collections.OrderedDict
1687
1688                 for field_name in node[pkey]:
1689                     resolve_field_type_aliases(node[pkey], field_name,
1690                                                from_descr, alias_set)
1691
1692         def resolve_field_type_aliases_from(parent_node, key):
1693             resolve_field_type_aliases(parent_node, key,
1694                                        f'`{key}` property')
1695
1696         # set of resolved field type aliases
1697         resolved_aliases = set()
1698
1699         # Expand field type aliases within trace, stream, and event
1700         # types now.
1701         try:
1702             resolve_field_type_aliases_from(metadata_node['trace'],
1703                                             'packet-header-type')
1704         except _ConfigParseError as exc:
1705             _append_error_ctx(exc, 'Trace type')
1706
1707         for stream_name, stream in metadata_node['streams'].items():
1708             try:
1709                 resolve_field_type_aliases_from(stream, 'packet-context-type')
1710                 resolve_field_type_aliases_from(stream, 'event-header-type')
1711                 resolve_field_type_aliases_from(stream, 'event-context-type')
1712
1713                 for event_name, event in stream['events'].items():
1714                     try:
1715                         resolve_field_type_aliases_from(event, 'context-type')
1716                         resolve_field_type_aliases_from(event, 'payload-type')
1717                     except _ConfigParseError as exc:
1718                         _append_error_ctx(exc, f'Event type `{event_name}`')
1719             except _ConfigParseError as exc:
1720                 _append_error_ctx(exc, f'Stream type `{stream_name}`')
1721
1722         # remove the (now unneeded) `type-aliases` node
1723         del metadata_node['type-aliases']
1724
1725     # Applies field type inheritance to all field types found in
1726     # `metadata_node`.
1727     #
1728     # This method modifies `metadata_node`.
1729     #
1730     # When this method returns, no field type node has an `$inherit` or
1731     # `inherit` property.
1732     def _expand_field_type_inheritance(self, metadata_node):
1733         def apply_inheritance(parent_node, key):
1734             if key not in parent_node:
1735                 return
1736
1737             node = parent_node[key]
1738
1739             if node is None:
1740                 return
1741
1742             # process children first
1743             for pkey in ['$inherit', 'inherit', 'value-type', 'element-type']:
1744                 apply_inheritance(node, pkey)
1745
1746             # structure field type fields
1747             pkey = 'fields'
1748
1749             if pkey in node:
1750                 assert type(node[pkey]) is collections.OrderedDict
1751
1752                 for field_name, field_type in node[pkey].items():
1753                     apply_inheritance(node[pkey], field_name)
1754
1755             # apply inheritance of this node
1756             if 'inherit' in node:
1757                 # barectf 2.1: `inherit` property was renamed to `$inherit`
1758                 assert '$inherit' not in node
1759                 node['$inherit'] = node['inherit']
1760                 del node['inherit']
1761
1762             inherit_key = '$inherit'
1763
1764             if inherit_key in node:
1765                 assert type(node[inherit_key]) is collections.OrderedDict
1766
1767                 # apply inheritance below
1768                 apply_inheritance(node, inherit_key)
1769
1770                 # `node` is an overlay on the `$inherit` node
1771                 base_node = node[inherit_key]
1772                 del node[inherit_key]
1773                 self._update_node(base_node, node)
1774
1775                 # set updated base node as this node
1776                 parent_node[key] = base_node
1777
1778         apply_inheritance(metadata_node['trace'], 'packet-header-type')
1779
1780         for stream in metadata_node['streams'].values():
1781             apply_inheritance(stream, 'packet-context-type')
1782             apply_inheritance(stream, 'event-header-type')
1783             apply_inheritance(stream, 'event-context-type')
1784
1785             for event in stream['events'].values():
1786                 apply_inheritance(event, 'context-type')
1787                 apply_inheritance(event, 'payload-type')
1788
1789     # Calls _expand_field_type_aliases() and
1790     # _expand_field_type_inheritance() if the metadata node
1791     # `metadata_node` has a `type-aliases` property.
1792     def _expand_field_types(self, metadata_node):
1793         type_aliases_node = metadata_node.get('type-aliases')
1794
1795         if type_aliases_node is None:
1796             # If there's no `type-aliases` node, then there's no field
1797             # type aliases and therefore no possible inheritance.
1798             return
1799
1800         # first, expand field type aliases
1801         self._expand_field_type_aliases(metadata_node, type_aliases_node)
1802
1803         # next, apply inheritance to create effective field types
1804         self._expand_field_type_inheritance(metadata_node)
1805
1806     # Replaces the textual log levels in event type nodes of the
1807     # metadata node `metadata_node` with their numeric equivalent (as
1808     # found in the `$log-levels` or `log-levels` node of
1809     # `metadata_node`).
1810     #
1811     # This method modifies `metadata_node`.
1812     #
1813     # When this method returns, the `$log-levels` or `log-level`
1814     # property of `metadata_node` is removed.
1815     def _expand_log_levels(self, metadata_node):
1816         if 'log-levels' in metadata_node:
1817             # barectf 2.1: `log-levels` property was renamed to
1818             # `$log-levels`
1819             assert '$log-levels' not in metadata_node
1820             metadata_node['$log-levels'] = metadata_node['log-levels']
1821             del metadata_node['log-levels']
1822
1823         log_levels_key = '$log-levels'
1824         log_levels_node = metadata_node.get(log_levels_key)
1825
1826         if log_levels_node is None:
1827             # no log level aliases
1828             return
1829
1830         # not needed anymore
1831         del metadata_node[log_levels_key]
1832
1833         for stream_name, stream in metadata_node['streams'].items():
1834             try:
1835                 for event_name, event in stream['events'].items():
1836                     prop_name = 'log-level'
1837                     ll_node = event.get(prop_name)
1838
1839                     if ll_node is None:
1840                         continue
1841
1842                     if type(ll_node) is str:
1843                         if ll_node not in log_levels_node:
1844                             exc = _ConfigParseError('`log-level` property',
1845                                                     f'Log level alias `{ll_node}` does not exist')
1846                             exc.append_ctx(f'Event type `{event_name}`')
1847                             raise exc
1848
1849                         event[prop_name] = log_levels_node[ll_node]
1850             except _ConfigParseError as exc:
1851                 _append_error_ctx(exc, f'Stream type `{stream_name}`')
1852
1853     # Dumps the node `node` as YAML, passing `kwds` to yaml.dump().
1854     def _yaml_ordered_dump(self, node, **kwds):
1855         class ODumper(yaml.Dumper):
1856             pass
1857
1858         def dict_representer(dumper, node):
1859             return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
1860                                             node.items())
1861
1862         ODumper.add_representer(collections.OrderedDict, dict_representer)
1863
1864         # Python -> YAML
1865         return yaml.dump(node, Dumper=ODumper, **kwds)
1866
1867     # Loads the content of the YAML file having the path `yaml_path` as
1868     # a Python object.
1869     #
1870     # All YAML maps are loaded as `collections.OrderedDict` objects.
1871     def _yaml_ordered_load(self, yaml_path):
1872         class OLoader(yaml.Loader):
1873             pass
1874
1875         def construct_mapping(loader, node):
1876             loader.flatten_mapping(node)
1877
1878             return collections.OrderedDict(loader.construct_pairs(node))
1879
1880         OLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
1881                                 construct_mapping)
1882
1883         # YAML -> Python
1884         try:
1885             with open(yaml_path, 'r') as f:
1886                 node = yaml.load(f, OLoader)
1887         except (OSError, IOError) as exc:
1888             raise _ConfigParseError(f'File `{yaml_path}`',
1889                                     f'Cannot open file: {exc}')
1890
1891         assert type(node) is collections.OrderedDict
1892         return node
1893
1894     def _parse(self):
1895         self._version = None
1896         self._include_stack = []
1897
1898         # load the configuration object as is from the root YAML file
1899         try:
1900             config_node = self._yaml_ordered_load(self._root_path)
1901         except _ConfigParseError as exc:
1902             _append_error_ctx(exc, 'Configuration',
1903                               f'Cannot parse YAML file `{self._root_path}`')
1904
1905         # Make sure the configuration object is minimally valid, that
1906         # is, it contains a valid `version` property.
1907         #
1908         # This step does not validate the whole configuration object
1909         # yet because we don't have an effective configuration object;
1910         # we still need to:
1911         #
1912         # * Process inclusions.
1913         # * Expand field types (inheritance and aliases).
1914         self._schema_validator.validate(config_node, 'config/config-min')
1915
1916         # Process configuration object inclusions.
1917         #
1918         # self._process_config_includes() returns a new (or the same)
1919         # configuration object without any `$include` property in it,
1920         # recursively.
1921         config_node = self._process_config_includes(config_node)
1922
1923         # Make sure that the current configuration object is valid
1924         # considering field types are not expanded yet.
1925         self._schema_validator.validate(config_node,
1926                                         '2/config/config-pre-field-type-expansion')
1927
1928         # Expand field types.
1929         #
1930         # This process:
1931         #
1932         # 1. Replaces field type aliases with "effective" field
1933         #    types, recursively.
1934         #
1935         #    After this step, the `type-aliases` property of the
1936         #    `metadata` node is gone.
1937         #
1938         # 2. Applies inheritance, following the `$inherit`/`inherit`
1939         #    properties.
1940         #
1941         #    After this step, field type objects do not contain
1942         #    `$inherit` or `inherit` properties.
1943         #
1944         # This is done blindly, in that the process _doesn't_ validate
1945         # field type objects at this point.
1946         self._expand_field_types(config_node['metadata'])
1947
1948         # Make sure that the current configuration object is valid
1949         # considering log levels are not expanded yet.
1950         self._schema_validator.validate(config_node,
1951                                         '2/config/config-pre-log-level-expansion')
1952
1953         # Expand log levels, that is, replace log level strings with
1954         # their equivalent numeric values.
1955         self._expand_log_levels(config_node['metadata'])
1956
1957         # validate the whole, effective configuration object
1958         self._schema_validator.validate(config_node, '2/config/config')
1959
1960         # dump config if required
1961         if self._dump_config:
1962             print(self._yaml_ordered_dump(config_node, indent=2,
1963                                           default_flow_style=False))
1964
1965         # get prefix, options, and metadata pseudo-object
1966         prefix = self._get_prefix(config_node)
1967         opts = self._get_options(config_node)
1968         pseudo_meta = self._create_metadata(config_node)
1969
1970         # create public configuration
1971         self._config = config.Config(pseudo_meta.to_public(), prefix, opts)
1972
1973     @property
1974     def config(self):
1975         return self._config
1976
1977
1978 def _from_file(path, include_dirs, ignore_include_not_found, dump_config):
1979     try:
1980         return _YamlConfigParser(path, include_dirs, ignore_include_not_found,
1981                                  dump_config).config
1982     except _ConfigParseError as exc:
1983         _append_error_ctx(exc, 'Configuration',
1984                           f'Cannot create configuration from YAML file `{path}`')