1 # The MIT License (MIT)
3 # Copyright (c) 2023 Philippe Proulx <eeppeliteloop@gmail.com>
5 # Permission is hereby granted, free of charge, to any person obtaining
6 # a copy of this software and associated documentation files (the
7 # "Software"), to deal in the Software without restriction, including
8 # without limitation the rights to use, copy, modify, merge, publish,
9 # distribute, sublicense, and/or sell copies of the Software, and to
10 # permit persons to whom the Software is furnished to do so, subject to
11 # the following conditions:
13 # The above copyright notice and this permission notice shall be
14 # included in all copies or substantial portions of the Software.
16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17 # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18 # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
19 # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
20 # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
21 # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
22 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 # This module is the portable Normand processor. It offers both the
25 # parse() function and the command-line tool (run the module itself)
26 # without external dependencies except a `typing` module for Python 3.4.
28 # Feel free to copy this module file to your own project to use Normand.
30 # Upstream repository: <https://github.com/efficios/normand>.
32 __author__
= "Philippe Proulx"
51 from typing
import Any
, Dict
, List
, Union
, Pattern
, Callable
, NoReturn
, Optional
54 # Text location (line and column numbers).
57 def _create(cls
, line_no
: int, col_no
: int):
58 self
= cls
.__new
__(cls
)
59 self
._init
(line_no
, col_no
)
62 def __init__(*args
, **kwargs
): # type: ignore
63 raise NotImplementedError
65 def _init(self
, line_no
: int, col_no
: int):
66 self
._line
_no
= line_no
82 def __init__(self
, text_loc
: TextLoc
):
83 self
._text
_loc
= text_loc
85 # Source text location.
90 # Returns the size, in bytes, of this item.
93 def size(self
) -> int:
98 class _RepableItem(_Item
):
103 class _Byte(_RepableItem
):
104 def __init__(self
, val
: int, text_loc
: TextLoc
):
105 super().__init
__(text_loc
)
118 return "_Byte({}, {})".format(hex(self
._val
), self
._text
_loc
)
122 class _Str(_RepableItem
):
123 def __init__(self
, data
: bytes
, text_loc
: TextLoc
):
124 super().__init
__(text_loc
)
134 return len(self
._data
)
137 return "_Str({}, {})".format(repr(self
._data
), self
._text
_loc
)
142 class ByteOrder(enum
.Enum
):
152 def __init__(self
, bo
: ByteOrder
, text_loc
: TextLoc
):
153 super().__init
__(text_loc
)
167 def __init__(self
, name
: str, text_loc
: TextLoc
):
168 super().__init
__(text_loc
)
181 return "_Label({}, {})".format(repr(self
._name
), self
._text
_loc
)
185 class _Offset(_Item
):
186 def __init__(self
, val
: int, text_loc
: TextLoc
):
187 super().__init
__(text_loc
)
200 return "_Offset({}, {})".format(repr(self
._val
), self
._text
_loc
)
203 # Mixin of containing an AST expression and its string.
205 def __init__(self
, expr_str
: str, expr
: ast
.Expression
):
206 self
._expr
_str
= expr_str
212 return self
._expr
_str
214 # Expression node to evaluate.
221 class _Var(_Item
, _ExprMixin
):
223 self
, name
: str, expr_str
: str, expr
: ast
.Expression
, text_loc
: TextLoc
225 super().__init
__(text_loc
)
226 _ExprMixin
.__init
__(self
, expr_str
, expr
)
239 return "_Var({}, {}, {}, {})".format(
240 repr(self
._name
), repr(self
._expr
_str
), repr(self
._expr
), self
._text
_loc
244 # Value, possibly needing more than one byte.
245 class _Val(_RepableItem
, _ExprMixin
):
247 self
, expr_str
: str, expr
: ast
.Expression
, len: int, text_loc
: TextLoc
249 super().__init
__(text_loc
)
250 _ExprMixin
.__init
__(self
, expr_str
, expr
)
260 return self
._len
// 8
263 return "_Val({}, {}, {}, {})".format(
264 repr(self
._expr
_str
), repr(self
._expr
), repr(self
._len
), self
._text
_loc
268 # Expression item type.
269 _ExprItemT
= Union
[_Val
, _Var
]
273 class _Group(_RepableItem
):
274 def __init__(self
, items
: List
[_Item
], text_loc
: TextLoc
):
275 super().__init
__(text_loc
)
277 self
._size
= sum([item
.size
for item
in self
._items
])
289 return "_Group({}, {})".format(repr(self
._items
), self
._text
_loc
)
294 def __init__(self
, item
: _RepableItem
, mul
: int, text_loc
: TextLoc
):
295 super().__init
__(text_loc
)
304 # Repetition multiplier.
311 return self
._item
.size
* self
._mul
314 return "_Rep({}, {}, {})".format(
315 repr(self
._item
), repr(self
._mul
), self
._text
_loc
319 # A parsing error containing a message and a text location.
320 class ParseError(RuntimeError):
322 def _create(cls
, msg
: str, text_loc
: TextLoc
):
323 self
= cls
.__new
__(cls
)
324 self
._init
(msg
, text_loc
)
327 def __init__(self
, *args
, **kwargs
): # type: ignore
328 raise NotImplementedError
330 def _init(self
, msg
: str, text_loc
: TextLoc
):
331 super().__init
__(msg
)
332 self
._text
_loc
= text_loc
334 # Source text location.
337 return self
._text
_loc
340 # Raises a parsing error, forwarding the parameters to the constructor.
341 def _raise_error(msg
: str, text_loc
: TextLoc
) -> NoReturn
:
342 raise ParseError
._create
(msg
, text_loc
) # pyright: ignore[reportPrivateUsage]
345 # Variable (and label) dictionary type.
346 VarsT
= Dict
[str, int]
349 # Python name pattern.
350 _py_name_pat
= re
.compile(r
"[a-zA-Z_][a-zA-Z0-9_]*")
355 # The constructor accepts a Normand input. After building, use the `res`
356 # property to get the resulting main group.
358 # Builds a parser to parse the Normand input `normand`, parsing
360 def __init__(self
, normand
: str, variables
: VarsT
, labels
: VarsT
):
361 self
._normand
= normand
365 self
._label
_names
= set(labels
.keys())
366 self
._var
_names
= set(variables
.keys())
369 # Result (main group).
374 # Current text location.
377 return TextLoc
._create
( # pyright: ignore[reportPrivateUsage]
378 self
._line
_no
, self
._col
_no
381 # Returns `True` if this parser is done parsing.
383 return self
._at
== len(self
._normand
)
385 # Returns `True` if this parser isn't done parsing.
386 def _isnt_done(self
):
387 return not self
._is
_done
()
389 # Raises a parse error, creating it using the message `msg` and the
390 # current text location.
391 def _raise_error(self
, msg
: str) -> NoReturn
:
392 _raise_error(msg
, self
._text
_loc
)
394 # Tries to make the pattern `pat` match the current substring,
395 # returning the match object and updating `self._at`,
396 # `self._line_no`, and `self._col_no` on success.
397 def _try_parse_pat(self
, pat
: Pattern
[str]):
398 m
= pat
.match(self
._normand
, self
._at
)
403 # Skip matched string
404 self
._at
+= len(m
.group(0))
407 self
._line
_no
+= m
.group(0).count("\n")
409 # Update column number
410 for i
in reversed(range(self
._at
)):
411 if self
._normand
[i
] == "\n" or i
== 0:
413 self
._col
_no
= self
._at
+ 1
415 self
._col
_no
= self
._at
- i
419 # Return match object
422 # Expects the pattern `pat` to match the current substring,
423 # returning the match object and updating `self._at`,
424 # `self._line_no`, and `self._col_no` on success, or raising a parse
425 # error with the message `error_msg` on error.
426 def _expect_pat(self
, pat
: Pattern
[str], error_msg
: str):
428 m
= self
._try
_parse
_pat
(pat
)
432 self
._raise
_error
(error_msg
)
434 # Return match object
437 # Pattern for _skip_ws_and_comments()
438 _ws_or_syms_or_comments_pat
= re
.compile(
439 r
"(?:[\s!@/\\?&:;.,+[\]_=|-]|#[^#]*?(?:\n|#))*"
442 # Skips as many whitespaces, insignificant symbol characters, and
443 # comments as possible.
444 def _skip_ws_and_comments(self
):
445 self
._try
_parse
_pat
(self
._ws
_or
_syms
_or
_comments
_pat
)
447 # Pattern for _try_parse_hex_byte()
448 _nibble_pat
= re
.compile(r
"[A-Fa-f0-9]")
450 # Tries to parse a hexadecimal byte, returning a byte item on
452 def _try_parse_hex_byte(self
):
453 begin_text_loc
= self
._text
_loc
455 # Match initial nibble
456 m_high
= self
._try
_parse
_pat
(self
._nibble
_pat
)
462 # Expect another nibble
463 self
._skip
_ws
_and
_comments
()
464 m_low
= self
._expect
_pat
(
465 self
._nibble
_pat
, "Expecting another hexadecimal nibble"
469 return _Byte(int(m_high
.group(0) + m_low
.group(0), 16), begin_text_loc
)
471 # Patterns for _try_parse_bin_byte()
472 _bin_byte_bit_pat
= re
.compile(r
"[01]")
473 _bin_byte_prefix_pat
= re
.compile(r
"%")
475 # Tries to parse a binary byte, returning a byte item on success.
476 def _try_parse_bin_byte(self
):
477 begin_text_loc
= self
._text
_loc
480 if self
._try
_parse
_pat
(self
._bin
_byte
_prefix
_pat
) is None:
485 bits
= [] # type: List[str]
488 self
._skip
_ws
_and
_comments
()
489 m
= self
._expect
_pat
(self
._bin
_byte
_bit
_pat
, "Expecting a bit (`0` or `1`)")
490 bits
.append(m
.group(0))
493 return _Byte(int("".join(bits
), 2), begin_text_loc
)
495 # Patterns for _try_parse_dec_byte()
496 _dec_byte_prefix_pat
= re
.compile(r
"\$\s*")
497 _dec_byte_val_pat
= re
.compile(r
"(?P<neg>-?)(?P<val>\d+)")
499 # Tries to parse a decimal byte, returning a byte item on success.
500 def _try_parse_dec_byte(self
):
501 begin_text_loc
= self
._text
_loc
504 if self
._try
_parse
_pat
(self
._dec
_byte
_prefix
_pat
) is None:
509 m
= self
._expect
_pat
(self
._dec
_byte
_val
_pat
, "Expecting a decimal constant")
512 val
= int(m
.group("val")) * (-1 if m
.group("neg") == "-" else 1)
515 if val
< -128 or val
> 255:
516 _raise_error("Invalid decimal byte value {}".format(val
), begin_text_loc
)
522 return _Byte(val
, begin_text_loc
)
524 # Tries to parse a byte, returning a byte item on success.
525 def _try_parse_byte(self
):
527 item
= self
._try
_parse
_hex
_byte
()
533 item
= self
._try
_parse
_bin
_byte
()
539 item
= self
._try
_parse
_dec
_byte
()
544 # Patterns for _try_parse_str()
545 _str_prefix_pat
= re
.compile(r
'(?:u(?P<len>16|32)(?P<bo>be|le))?\s*"')
546 _str_suffix_pat
= re
.compile(r
'"')
547 _str_str_pat
= re
.compile(r
'(?:(?:\\.)|[^"])*')
549 # Strings corresponding to escape sequence characters
550 _str_escape_seq_strs
= {
564 # Tries to parse a string, returning a string item on success.
565 def _try_parse_str(self
):
566 begin_text_loc
= self
._text
_loc
569 m
= self
._try
_parse
_pat
(self
._str
_prefix
_pat
)
578 if m
.group("len") is not None:
579 encoding
= "utf_{}_{}".format(m
.group("len"), m
.group("bo"))
582 m
= self
._expect
_pat
(self
._str
_str
_pat
, "Expecting a literal string")
584 # Expect end of string
585 self
._expect
_pat
(self
._str
_suffix
_pat
, 'Expecting `"` (end of literal string)')
587 # Replace escape sequences
590 for ec
in '0abefnrtv"\\':
591 val
= val
.replace(r
"\{}".format(ec
), self
._str
_escape
_seq
_strs
[ec
])
594 data
= val
.encode(encoding
)
597 return _Str(data
, begin_text_loc
)
599 # Patterns for _try_parse_group()
600 _group_prefix_pat
= re
.compile(r
"\(")
601 _group_suffix_pat
= re
.compile(r
"\)")
603 # Tries to parse a group, returning a group item on success.
604 def _try_parse_group(self
):
605 begin_text_loc
= self
._text
_loc
608 if self
._try
_parse
_pat
(self
._group
_prefix
_pat
) is None:
613 items
= self
._parse
_items
()
615 # Expect end of group
616 self
._skip
_ws
_and
_comments
()
618 self
._group
_suffix
_pat
, "Expecting an item or `)` (end of group)"
622 return _Group(items
, begin_text_loc
)
624 # Returns a stripped expression string and an AST expression node
625 # from the expression string `expr_str` at text location `text_loc`.
626 def _ast_expr_from_str(self
, expr_str
: str, text_loc
: TextLoc
):
627 # Create an expression node from the expression string
628 expr_str
= expr_str
.strip().replace("\n", " ")
631 expr
= ast
.parse(expr_str
, mode
="eval")
634 "Invalid expression `{}`: invalid syntax".format(expr_str
),
638 return expr_str
, expr
640 # Patterns for _try_parse_val_and_len()
641 _val_expr_pat
= re
.compile(r
"([^}:]+):")
642 _val_len_pat
= re
.compile(r
"\s*(8|16|24|32|40|48|56|64)")
644 # Tries to parse a value and length, returning a value item on
646 def _try_parse_val_and_len(self
):
647 begin_text_loc
= self
._text
_loc
650 m_expr
= self
._try
_parse
_pat
(self
._val
_expr
_pat
)
657 m_len
= self
._expect
_pat
(
658 self
._val
_len
_pat
, "Expecting a length (multiple of eight bits)"
661 # Create an expression node from the expression string
662 expr_str
, expr
= self
._ast
_expr
_from
_str
(m_expr
.group(1), begin_text_loc
)
672 # Patterns for _try_parse_val_and_len()
673 _var_pat
= re
.compile(
674 r
"(?P<name>{})\s*=\s*(?P<expr>[^}}]+)".format(_py_name_pat
.pattern
)
677 # Tries to parse a variable, returning a variable item on success.
678 def _try_parse_var(self
):
679 begin_text_loc
= self
._text
_loc
682 m
= self
._try
_parse
_pat
(self
._var
_pat
)
689 name
= m
.group("name")
691 if name
== _icitte_name
:
693 "`{}` is a reserved variable name".format(_icitte_name
), begin_text_loc
696 if name
in self
._label
_names
:
697 _raise_error("Existing label named `{}`".format(name
), begin_text_loc
)
699 # Add to known variable names
700 self
._var
_names
.add(name
)
702 # Create an expression node from the expression string
703 expr_str
, expr
= self
._ast
_expr
_from
_str
(m
.group("expr"), begin_text_loc
)
713 # Pattern for _try_parse_bo_name()
714 _bo_pat
= re
.compile(r
"[bl]e")
716 # Tries to parse a byte order name, returning a byte order item on
718 def _try_parse_bo_name(self
):
719 begin_text_loc
= self
._text
_loc
722 m
= self
._try
_parse
_pat
(self
._bo
_pat
)
728 # Return corresponding item
729 if m
.group(0) == "be":
730 return _Bo(ByteOrder
.BE
, begin_text_loc
)
732 assert m
.group(0) == "le"
733 return _Bo(ByteOrder
.LE
, begin_text_loc
)
735 # Patterns for _try_parse_val_or_bo()
736 _val_var_bo_prefix_pat
= re
.compile(r
"\{\s*")
737 _val_var_bo_suffix_pat
= re
.compile(r
"\s*}")
739 # Tries to parse a value, a variable, or a byte order, returning an
741 def _try_parse_val_or_var_or_bo(self
):
743 if self
._try
_parse
_pat
(self
._val
_var
_bo
_prefix
_pat
) is None:
748 item
= self
._try
_parse
_var
()
752 item
= self
._try
_parse
_val
_and
_len
()
756 item
= self
._try
_parse
_bo
_name
()
759 # At this point it's invalid
760 self
._raise
_error
("Expecting a value, a variable, or a byte order")
763 self
._expect
_pat
(self
._val
_var
_bo
_suffix
_pat
, "Expecting `}`")
766 # Pattern for _try_parse_offset_val() and _try_parse_rep()
767 _pos_const_int_pat
= re
.compile(r
"0[Xx][A-Fa-f0-9]+|\d+")
769 # Tries to parse an offset value (after the initial `<`), returning
770 # an offset item on success.
771 def _try_parse_offset_val(self
):
772 begin_text_loc
= self
._text
_loc
775 m
= self
._try
_parse
_pat
(self
._pos
_const
_int
_pat
)
782 return _Offset(int(m
.group(0), 0), begin_text_loc
)
784 # Tries to parse a label name (after the initial `<`), returning a
785 # label item on success.
786 def _try_parse_label_name(self
):
787 begin_text_loc
= self
._text
_loc
790 m
= self
._try
_parse
_pat
(_py_name_pat
)
799 if name
== _icitte_name
:
801 "`{}` is a reserved label name".format(_icitte_name
), begin_text_loc
804 if name
in self
._label
_names
:
805 _raise_error("Duplicate label name `{}`".format(name
), begin_text_loc
)
807 if name
in self
._var
_names
:
808 _raise_error("Existing variable named `{}`".format(name
), begin_text_loc
)
810 # Add to known label names
811 self
._label
_names
.add(name
)
814 return _Label(name
, begin_text_loc
)
816 # Patterns for _try_parse_label_or_offset()
817 _label_offset_prefix_pat
= re
.compile(r
"<\s*")
818 _label_offset_suffix_pat
= re
.compile(r
"\s*>")
820 # Tries to parse a label or an offset, returning an item on success.
821 def _try_parse_label_or_offset(self
):
823 if self
._try
_parse
_pat
(self
._label
_offset
_prefix
_pat
) is None:
828 item
= self
._try
_parse
_offset
_val
()
832 item
= self
._try
_parse
_label
_name
()
835 # At this point it's invalid
836 self
._raise
_error
("Expecting a label name or an offset value")
839 self
._expect
_pat
(self
._label
_offset
_suffix
_pat
, "Expecting `>`")
842 # Tries to parse a base item (anything except a repetition),
843 # returning it on success.
844 def _try_parse_base_item(self
):
846 item
= self
._try
_parse
_byte
()
852 item
= self
._try
_parse
_str
()
857 # Value, variable, or byte order item?
858 item
= self
._try
_parse
_val
_or
_var
_or
_bo
()
863 # Label or offset item?
864 item
= self
._try
_parse
_label
_or
_offset
()
870 item
= self
._try
_parse
_group
()
875 # Pattern for _try_parse_rep()
876 _rep_prefix_pat
= re
.compile(r
"\*\s*")
878 # Tries to parse a repetition, returning the multiplier on success,
880 def _try_parse_rep(self
):
882 if self
._try
_parse
_pat
(self
._rep
_prefix
_pat
) is None:
886 # Expect and return a decimal multiplier
887 self
._skip
_ws
_and
_comments
()
888 m
= self
._expect
_pat
(
889 self
._pos
_const
_int
_pat
, "Expecting a positive integral multiplier"
891 return int(m
.group(0), 0)
893 # Tries to parse an item, possibly followed by a repetition,
894 # returning `True` on success.
896 # Appends any parsed item to `items`.
897 def _try_append_item(self
, items
: List
[_Item
]):
898 self
._skip
_ws
_and
_comments
()
901 item
= self
._try
_parse
_base
_item
()
907 # Parse repetition if the base item is repeatable
908 if isinstance(item
, _RepableItem
):
909 self
._skip
_ws
_and
_comments
()
910 rep_text_loc
= self
._text
_loc
911 rep
= self
._try
_parse
_rep
()
914 # No item, but that's okay
917 # Convert to repetition item
918 item
= _Rep(item
, rep
, rep_text_loc
)
923 # Parses and returns items, skipping whitespaces, insignificant
924 # symbols, and comments when allowed, and stopping at the first
926 def _parse_items(self
) -> List
[_Item
]:
927 items
= [] # type: List[_Item]
929 while self
._isnt
_done
():
931 if not self
._try
_append
_item
(items
):
932 # Unknown at this point
937 # Parses the whole Normand input, setting `self._res` to the main
938 # group item on success.
940 if len(self
._normand
.strip()) == 0:
941 # Special case to make sure there's something to consume
942 self
._res
= _Group([], self
._text
_loc
)
945 # Parse first level items
946 items
= self
._parse
_items
()
948 # Make sure there's nothing left
949 self
._skip
_ws
_and
_comments
()
951 if self
._isnt
_done
():
953 "Unexpected character `{}`".format(self
._normand
[self
._at
])
956 # Set main group item
957 self
._res
= _Group(items
, self
._text
_loc
)
960 # The return type of parse().
969 bo
: Optional
[ByteOrder
],
971 self
= cls
.__new
__(cls
)
972 self
._init
(data
, variables
, labels
, offset
, bo
)
975 def __init__(self
, *args
, **kwargs
): # type: ignore
976 raise NotImplementedError
984 bo
: Optional
[ByteOrder
],
987 self
._vars
= variables
988 self
._labels
= labels
989 self
._offset
= offset
997 # Dictionary of updated variable names to their last computed value.
1002 # Dictionary of updated main group label names to their computed
1013 # Updated byte order.
1015 def byte_order(self
):
1019 # Raises a parse error for the item `item`, creating it using the
1021 def _raise_error_for_item(msg
: str, item
: _Item
) -> NoReturn
:
1022 _raise_error(msg
, item
.text_loc
)
1025 # The `ICITTE` reserved name.
1026 _icitte_name
= "ICITTE"
1029 # Value expression validator.
1030 class _ExprValidator(ast
.NodeVisitor
):
1031 def __init__(self
, item
: _ExprItemT
, syms
: VarsT
):
1034 self
._parent
_is
_call
= False
1036 def generic_visit(self
, node
: ast
.AST
):
1037 if type(node
) is ast
.Call
:
1038 self
._parent
_is
_call
= True
1039 elif type(node
) is ast
.Name
and not self
._parent
_is
_call
:
1040 # Make sure the name refers to a known label name
1041 if node
.id != _icitte_name
and node
.id not in self
._syms
:
1043 "Unknown variable/label name `{}` in expression `{}`".format(
1044 node
.id, self
._item
.expr_str
1046 self
._item
.text_loc
,
1049 # TODO: Restrict the set of allowed node types
1051 super().generic_visit(node
)
1052 self
._parent
_is
_call
= False
1055 # Keeper of labels for a given group instance.
1057 # A group instance is one iteration of a given group.
1058 class _GroupInstanceLabels
:
1060 self
._instance
_labels
= {} # type: Dict[_Group, Dict[int, VarsT]]
1062 # Assigns the labels `labels` to a new instance of `group`.
1063 def add(self
, group
: _Group
, labels
: VarsT
):
1064 if group
not in self
._instance
_labels
:
1065 self
._instance
_labels
[group
] = {}
1067 spec_instance_labels
= self
._instance
_labels
[group
]
1068 spec_instance_labels
[len(spec_instance_labels
)] = labels
.copy()
1070 # Returns the labels (not a copy) of the instance `instance_index`
1071 # of the group `group`.
1072 def labels(self
, group
: _Group
, instance_index
: int):
1073 return self
._instance
_labels
[group
][instance_index
]
1076 # Generator of data and labels from a group item.
1078 # Generation happens in memory at construction time. After building, use
1079 # the `data`, `variables`, `labels`, `offset`, and `bo` properties to
1080 # get the resulting context.
1088 bo
: Optional
[ByteOrder
],
1090 self
._group
_instance
_labels
= _GroupInstanceLabels()
1091 self
._resolve
_labels
(group
, offset
, labels
.copy())
1092 self
._vars
= variables
.copy()
1093 self
._offset
= offset
1095 self
._main
_group
= group
1103 # Updated variables.
1105 def variables(self
):
1108 # Updated main group labels.
1111 return self
._group
_instance
_labels
.labels(self
._main
_group
, 0)
1118 # Updated byte order.
1123 # Fills `self._group_instance_labels` with the labels for each group
1124 # instance in `item`, starting at current offset `offset` with the
1125 # current labels `labels`.
1127 # Returns the new current offset.
1128 def _resolve_labels(self
, item
: _Item
, offset
: int, labels
: VarsT
) -> int:
1129 if type(item
) is _Group
:
1130 # First pass: compute immediate labels of this instance
1131 group_labels
= labels
.copy()
1132 group_offset
= offset
1134 for subitem
in item
.items
:
1135 if type(subitem
) is _Offset
:
1136 group_offset
= subitem
.val
1137 elif type(subitem
) is _Label
:
1138 assert subitem
.name
not in group_labels
1139 group_labels
[subitem
.name
] = group_offset
1141 group_offset
+= subitem
.size
1143 # Add to group instance labels
1144 self
._group
_instance
_labels
.add(item
, group_labels
)
1146 # Second pass: handle each item
1147 for subitem
in item
.items
:
1148 offset
= self
._resolve
_labels
(subitem
, offset
, group_labels
)
1149 elif type(item
) is _Rep
:
1150 for _
in range(item
.mul
):
1151 offset
= self
._resolve
_labels
(item
.item
, offset
, labels
)
1152 elif type(item
) is _Offset
:
1159 def _handle_byte_item(self
, item
: _Byte
):
1160 self
._data
.append(item
.val
)
1161 self
._offset
+= item
.size
1163 def _handle_str_item(self
, item
: _Str
):
1164 self
._data
+= item
.data
1165 self
._offset
+= item
.size
1167 def _handle_bo_item(self
, item
: _Bo
):
1170 def _eval_expr(self
, item
: _ExprItemT
):
1171 # Get the labels of the current group instance as the initial
1172 # symbols (copied because we're adding stuff).
1173 assert self
._cur
_group
is not None
1174 syms
= self
._group
_instance
_labels
.labels(
1175 self
._cur
_group
, self
._group
_instance
_indexes
[self
._cur
_group
]
1178 # Set the `ICITTE` name to the current offset (before encoding)
1179 syms
[_icitte_name
] = self
._offset
1181 # Add the current variables
1182 syms
.update(self
._vars
)
1184 # Validate the node and its children
1185 _ExprValidator(item
, syms
).visit(item
.expr
)
1187 # Compile and evaluate expression node
1189 val
= eval(compile(item
.expr
, "", "eval"), None, syms
)
1190 except Exception as exc
:
1191 _raise_error_for_item(
1192 "Failed to evaluate expression `{}`: {}".format(item
.expr_str
, exc
),
1197 if type(val
) is not int:
1198 _raise_error_for_item(
1199 "Invalid expression `{}`: unexpected result type `{}`".format(
1200 item
.expr_str
, type(val
).__name
__
1207 def _handle_var_item(self
, item
: _Var
):
1209 self
._vars
[item
.name
] = self
._eval
_expr
(item
)
1211 def _handle_val_item(self
, item
: _Val
):
1213 val
= self
._eval
_expr
(item
)
1216 if val
< -(2 ** (item
.len - 1)) or val
> 2**item
.len - 1:
1217 _raise_error_for_item(
1218 "Value {:,} is outside the {}-bit range when evaluating expression `{}` at byte offset {:,}".format(
1219 val
, item
.len, item
.expr_str
, self
._offset
1224 # Encode result on 64 bits (to extend the sign bit whatever the
1225 # value of `item.len`).
1226 if self
._bo
is None and item
.len > 8:
1227 _raise_error_for_item(
1228 "Current byte order isn't defined at first value (`{}`) to encode on more than 8 bits".format(
1236 ">" if self
._bo
in (None, ByteOrder
.BE
) else "<",
1237 "Q" if val
>= 0 else "q",
1242 # Keep only the requested length
1243 len_bytes
= item
.len // 8
1245 if self
._bo
in (None, ByteOrder
.BE
):
1246 # Big endian: keep last bytes
1247 data
= data
[-len_bytes
:]
1249 # Little endian: keep first bytes
1250 assert self
._bo
== ByteOrder
.LE
1251 data
= data
[:len_bytes
]
1253 # Append to current bytes and update offset
1255 self
._offset
+= len(data
)
1257 def _handle_group_item(self
, item
: _Group
):
1258 # Update the instance index of `item`
1259 if item
not in self
._group
_instance
_indexes
:
1260 self
._group
_instance
_indexes
[item
] = 0
1262 self
._group
_instance
_indexes
[item
] += 1
1264 # Changed current group
1265 old_cur_group
= self
._cur
_group
1266 self
._cur
_group
= item
1269 for subitem
in item
.items
:
1270 self
._handle
_item
(subitem
)
1272 # Restore current group
1273 self
._cur
_group
= old_cur_group
1275 def _handle_rep_item(self
, item
: _Rep
):
1276 for _
in range(item
.mul
):
1277 self
._handle
_item
(item
.item
)
1279 def _handle_offset_item(self
, item
: _Offset
):
1280 self
._offset
= item
.val
1282 def _handle_item(self
, item
: _Item
):
1283 if type(item
) in self
._item
_handlers
:
1284 self
._item
_handlers
[type(item
)](item
)
1288 self
._data
= bytearray()
1289 self
._group
_instance
_indexes
= {} # type: Dict[_Group, int]
1290 self
._cur
_group
= None
1293 self
._item
_handlers
= {
1294 _Byte
: self
._handle
_byte
_item
,
1295 _Str
: self
._handle
_str
_item
,
1296 _Bo
: self
._handle
_bo
_item
,
1297 _Val
: self
._handle
_val
_item
,
1298 _Var
: self
._handle
_var
_item
,
1299 _Group
: self
._handle
_group
_item
,
1300 _Rep
: self
._handle
_rep
_item
,
1301 _Offset
: self
._handle
_offset
_item
,
1302 } # type: Dict[type, Callable[[Any], None]]
1304 # Handle the group item
1305 self
._handle
_item
(self
._main
_group
)
1308 # Returns a `ParseResult` instance containing the bytes encoded by the
1309 # input string `normand`.
1311 # `init_variables` is a dictionary of initial variable names (valid
1312 # Python names) to integral values. A variable name must not be the
1313 # reserved name `ICITTE`.
1315 # `init_labels` is a dictionary of initial label names (valid Python
1316 # names) to integral values. A label name must not be the reserved name
1319 # `init_offset` is the initial offset.
1321 # `init_byte_order` is the initial byte order.
1323 # Raises `ParseError` on any parsing error.
1326 init_variables
: Optional
[VarsT
] = None,
1327 init_labels
: Optional
[VarsT
] = None,
1328 init_offset
: int = 0,
1329 init_byte_order
: Optional
[ByteOrder
] = None,
1331 if init_variables
is None:
1334 if init_labels
is None:
1338 _Parser(normand
, init_variables
, init_labels
).res
,
1344 return ParseResult
._create
( # pyright: ignore[reportPrivateUsage]
1345 gen
.data
, gen
.variables
, gen
.labels
, gen
.offset
, gen
.bo
1349 # Parses the command-line arguments.
1350 def _parse_cli_args():
1354 ap
= argparse
.ArgumentParser()
1361 help="initial offset (positive)",
1367 choices
=["be", "le"],
1369 help="initial byte order (`be` or `le`)",
1375 help="add an initial variable (may be repeated)",
1382 help="add an initial label (may be repeated)",
1385 "--version", action
="version", version
="Normand {}".format(__version__
)
1392 help="input path (none means standard input)",
1396 return ap
.parse_args()
1399 # Raises a command-line error with the message `msg`.
1400 def _raise_cli_error(msg
: str) -> NoReturn
:
1401 raise RuntimeError("Command-line error: {}".format(msg
))
1404 # Returns a dictionary of string to integers from the list of strings
1405 # `args` containing `NAME=VAL` entries.
1406 def _dict_from_arg(args
: Optional
[List
[str]]):
1407 d
= {} # type: Dict[str, int]
1413 m
= re
.match(r
"({})=(\d+)$".format(_py_name_pat
.pattern
), arg
)
1416 _raise_cli_error("Invalid assignment {}".format(arg
))
1418 d
[m
.group(1)] = int(m
.group(2))
1423 # CLI entry point without exception handling.
1428 args
= _parse_cli_args()
1431 if args
.path
is None:
1432 normand
= sys
.stdin
.read()
1434 with
open(args
.path
) as f
:
1437 # Variables and labels
1438 variables
= _dict_from_arg(args
.var
)
1439 labels
= _dict_from_arg(args
.label
)
1443 _raise_cli_error("Invalid negative offset {}")
1445 # Validate and set byte order
1446 bo
= None # type: Optional[ByteOrder]
1448 if args
.byte_order
is not None:
1449 if args
.byte_order
== "be":
1452 assert args
.byte_order
== "le"
1457 res
= parse(normand
, variables
, labels
, args
.offset
, bo
)
1458 except ParseError
as exc
:
1461 if args
.path
is not None:
1462 prefix
= "{}:".format(os
.path
.abspath(args
.path
))
1465 "{}{}:{} - {}".format(
1466 prefix
, exc
.text_loc
.line_no
, exc
.text_loc
.col_no
, str(exc
)
1471 sys
.stdout
.buffer.write(res
.data
)
1474 # Prints the exception message `msg` and exits with status 1.
1475 def _fail(msg
: str) -> NoReturn
:
1476 if not msg
.endswith("."):
1479 print(msg
, file=sys
.stderr
)
1487 except Exception as exc
:
1491 if __name__
== "__main__":