# Upstream repository: <https://github.com/efficios/normand>.
__author__ = "Philippe Proulx"
-__version__ = "0.6.1"
+__version__ = "0.12.0"
__all__ = [
+ "__author__",
+ "__version__",
"ByteOrder",
+ "LabelsT",
"parse",
"ParseError",
"ParseResult",
- "TextLoc",
- "SymbolsT",
- "__author__",
- "__version__",
+ "TextLocation",
+ "VariablesT",
]
import re
import abc
import ast
import sys
+import copy
import enum
import math
import struct
-from typing import (
- Any,
- Set,
- Dict,
- List,
- Tuple,
- Union,
- Pattern,
- Callable,
- NoReturn,
- Optional,
-)
+import typing
+from typing import Any, Set, Dict, List, Union, Pattern, Callable, NoReturn, Optional
# Text location (line and column numbers).
-class TextLoc:
+class TextLocation:
@classmethod
def _create(cls, line_no: int, col_no: int):
self = cls.__new__(cls)
return self._col_no
def __repr__(self):
- return "TextLoc({}, {})".format(self._line_no, self._col_no)
+ return "TextLocation({}, {})".format(self._line_no, self._col_no)
# Any item.
class _Item:
- def __init__(self, text_loc: TextLoc):
+ def __init__(self, text_loc: TextLocation):
self._text_loc = text_loc
# Source text location.
# Single byte.
class _Byte(_ScalarItem, _RepableItem):
- def __init__(self, val: int, text_loc: TextLoc):
+ def __init__(self, val: int, text_loc: TextLocation):
super().__init__(text_loc)
self._val = val
return 1
def __repr__(self):
- return "_Byte({}, {})".format(hex(self._val), self._text_loc)
+ return "_Byte({}, {})".format(hex(self._val), repr(self._text_loc))
# String.
class _Str(_ScalarItem, _RepableItem):
- def __init__(self, data: bytes, text_loc: TextLoc):
+ def __init__(self, data: bytes, text_loc: TextLocation):
super().__init__(text_loc)
self._data = data
return len(self._data)
def __repr__(self):
- return "_Str({}, {})".format(repr(self._data), self._text_loc)
+ return "_Str({}, {})".format(repr(self._data), repr(self._text_loc))
# Byte order.
# Byte order setting.
class _SetBo(_Item):
- def __init__(self, bo: ByteOrder, text_loc: TextLoc):
+ def __init__(self, bo: ByteOrder, text_loc: TextLocation):
super().__init__(text_loc)
self._bo = bo
return self._bo
def __repr__(self):
- return "_SetBo({}, {})".format(repr(self._bo), self._text_loc)
+ return "_SetBo({}, {})".format(repr(self._bo), repr(self._text_loc))
# Label.
class _Label(_Item):
- def __init__(self, name: str, text_loc: TextLoc):
+ def __init__(self, name: str, text_loc: TextLocation):
super().__init__(text_loc)
self._name = name
return self._name
def __repr__(self):
- return "_Label({}, {})".format(repr(self._name), self._text_loc)
+ return "_Label({}, {})".format(repr(self._name), repr(self._text_loc))
# Offset setting.
class _SetOffset(_Item):
- def __init__(self, val: int, text_loc: TextLoc):
+ def __init__(self, val: int, text_loc: TextLocation):
+ super().__init__(text_loc)
+ self._val = val
+
+ # Offset value (bytes).
+ @property
+ def val(self):
+ return self._val
+
+ def __repr__(self):
+ return "_SetOffset({}, {})".format(repr(self._val), repr(self._text_loc))
+
+
+# Offset alignment.
+class _AlignOffset(_Item):
+ def __init__(self, val: int, pad_val: int, text_loc: TextLocation):
super().__init__(text_loc)
self._val = val
+ self._pad_val = pad_val
- # Offset value.
+ # Alignment value (bits).
@property
def val(self):
return self._val
+ # Padding byte value.
+ @property
+ def pad_val(self):
+ return self._pad_val
+
def __repr__(self):
- return "_SetOffset({}, {})".format(repr(self._val), self._text_loc)
+ return "_AlignOffset({}, {}, {})".format(
+ repr(self._val), repr(self._pad_val), repr(self._text_loc)
+ )
# Mixin of containing an AST expression and its string.
return self._expr
+# Fill until some offset.
+class _FillUntil(_Item, _ExprMixin):
+ def __init__(
+ self, expr_str: str, expr: ast.Expression, pad_val: int, text_loc: TextLocation
+ ):
+ super().__init__(text_loc)
+ _ExprMixin.__init__(self, expr_str, expr)
+ self._pad_val = pad_val
+
+ # Padding byte value.
+ @property
+ def pad_val(self):
+ return self._pad_val
+
+ def __repr__(self):
+ return "_FillUntil({}, {}, {}, {})".format(
+ repr(self._expr_str),
+ repr(self._expr),
+ repr(self._pad_val),
+ repr(self._text_loc),
+ )
+
+
# Variable assignment.
class _VarAssign(_Item, _ExprMixin):
def __init__(
- self, name: str, expr_str: str, expr: ast.Expression, text_loc: TextLoc
+ self, name: str, expr_str: str, expr: ast.Expression, text_loc: TextLocation
):
super().__init__(text_loc)
_ExprMixin.__init__(self, expr_str, expr)
def __repr__(self):
return "_VarAssign({}, {}, {}, {})".format(
- repr(self._name), repr(self._expr_str), repr(self._expr), self._text_loc
+ repr(self._name),
+ repr(self._expr_str),
+ repr(self._expr),
+ repr(self._text_loc),
)
# Fixed-length number, possibly needing more than one byte.
class _FlNum(_ScalarItem, _RepableItem, _ExprMixin):
def __init__(
- self, expr_str: str, expr: ast.Expression, len: int, text_loc: TextLoc
+ self, expr_str: str, expr: ast.Expression, len: int, text_loc: TextLocation
):
super().__init__(text_loc)
_ExprMixin.__init__(self, expr_str, expr)
def __repr__(self):
return "_FlNum({}, {}, {}, {})".format(
- repr(self._expr_str), repr(self._expr), repr(self._len), self._text_loc
+ repr(self._expr_str),
+ repr(self._expr),
+ repr(self._len),
+ repr(self._text_loc),
)
# LEB128 integer.
class _Leb128Int(_Item, _RepableItem, _ExprMixin):
- def __init__(self, expr_str: str, expr: ast.Expression, text_loc: TextLoc):
+ def __init__(self, expr_str: str, expr: ast.Expression, text_loc: TextLocation):
super().__init__(text_loc)
_ExprMixin.__init__(self, expr_str, expr)
self.__class__.__name__,
repr(self._expr_str),
repr(self._expr),
- self._text_loc,
+ repr(self._text_loc),
)
# Group of items.
class _Group(_Item, _RepableItem):
- def __init__(self, items: List[_Item], text_loc: TextLoc):
+ def __init__(self, items: List[_Item], text_loc: TextLocation):
super().__init__(text_loc)
self._items = items
return self._items
def __repr__(self):
- return "_Group({}, {})".format(repr(self._items), self._text_loc)
+ return "_Group({}, {})".format(repr(self._items), repr(self._text_loc))
# Repetition item.
class _Rep(_Item, _ExprMixin):
def __init__(
- self, item: _Item, expr_str: str, expr: ast.Expression, text_loc: TextLoc
+ self, item: _Item, expr_str: str, expr: ast.Expression, text_loc: TextLocation
):
super().__init__(text_loc)
_ExprMixin.__init__(self, expr_str, expr)
def __repr__(self):
return "_Rep({}, {}, {}, {})".format(
- repr(self._item), repr(self._expr_str), repr(self._expr), self._text_loc
+ repr(self._item),
+ repr(self._expr_str),
+ repr(self._expr),
+ repr(self._text_loc),
+ )
+
+
+# Conditional item.
+class _Cond(_Item, _ExprMixin):
+ def __init__(
+ self, item: _Item, expr_str: str, expr: ast.Expression, text_loc: TextLocation
+ ):
+ super().__init__(text_loc)
+ _ExprMixin.__init__(self, expr_str, expr)
+ self._item = item
+
+ # Conditional item.
+ @property
+ def item(self):
+ return self._item
+
+ def __repr__(self):
+ return "_Cond({}, {}, {}, {})".format(
+ repr(self._item),
+ repr(self._expr_str),
+ repr(self._expr),
+ repr(self._text_loc),
+ )
+
+
+# Macro definition item.
+class _MacroDef(_Item):
+ def __init__(
+ self, name: str, param_names: List[str], group: _Group, text_loc: TextLocation
+ ):
+ super().__init__(text_loc)
+ self._name = name
+ self._param_names = param_names
+ self._group = group
+
+ # Name.
+ @property
+ def name(self):
+ return self._name
+
+ # Parameters.
+ @property
+ def param_names(self):
+ return self._param_names
+
+ # Contained items.
+ @property
+ def group(self):
+ return self._group
+
+ def __repr__(self):
+ return "_MacroDef({}, {}, {}, {})".format(
+ repr(self._name),
+ repr(self._param_names),
+ repr(self._group),
+ repr(self._text_loc),
+ )
+
+
+# Macro expansion parameter.
+class _MacroExpParam:
+ def __init__(self, expr_str: str, expr: ast.Expression, text_loc: TextLocation):
+ self._expr_str = expr_str
+ self._expr = expr
+ self._text_loc = text_loc
+
+ # Expression string.
+ @property
+ def expr_str(self):
+ return self._expr_str
+
+ # Expression.
+ @property
+ def expr(self):
+ return self._expr
+
+ # Source text location.
+ @property
+ def text_loc(self):
+ return self._text_loc
+
+ def __repr__(self):
+ return "_MacroExpParam({}, {}, {})".format(
+ repr(self._expr_str), repr(self._expr), repr(self._text_loc)
)
-# Expression item type.
-_ExprItemT = Union[_FlNum, _Leb128Int, _VarAssign, _Rep]
+# Macro expansion item.
+class _MacroExp(_Item, _RepableItem):
+ def __init__(
+ self,
+ name: str,
+ params: List[_MacroExpParam],
+ text_loc: TextLocation,
+ ):
+ super().__init__(text_loc)
+ self._name = name
+ self._params = params
+
+ # Name.
+ @property
+ def name(self):
+ return self._name
+
+ # Parameters.
+ @property
+ def params(self):
+ return self._params
+
+ def __repr__(self):
+ return "_MacroExp({}, {}, {})".format(
+ repr(self._name),
+ repr(self._params),
+ repr(self._text_loc),
+ )
# A parsing error containing a message and a text location.
class ParseError(RuntimeError):
@classmethod
- def _create(cls, msg: str, text_loc: TextLoc):
+ def _create(cls, msg: str, text_loc: TextLocation):
self = cls.__new__(cls)
self._init(msg, text_loc)
return self
def __init__(self, *args, **kwargs): # type: ignore
raise NotImplementedError
- def _init(self, msg: str, text_loc: TextLoc):
+ def _init(self, msg: str, text_loc: TextLocation):
super().__init__(msg)
self._text_loc = text_loc
# Raises a parsing error, forwarding the parameters to the constructor.
-def _raise_error(msg: str, text_loc: TextLoc) -> NoReturn:
+def _raise_error(msg: str, text_loc: TextLocation) -> NoReturn:
raise ParseError._create(msg, text_loc) # pyright: ignore[reportPrivateUsage]
-# Variable/label dictionary type.
-SymbolsT = Dict[str, int]
+# Variables dictionary type (for type hints).
+VariablesT = Dict[str, Union[int, float]]
+
+
+# Labels dictionary type (for type hints).
+LabelsT = Dict[str, int]
# Python name pattern.
_py_name_pat = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*")
+# Macro definition dictionary.
+_MacroDefsT = Dict[str, _MacroDef]
+
+
# Normand parser.
#
# The constructor accepts a Normand input. After building, use the `res`
class _Parser:
# Builds a parser to parse the Normand input `normand`, parsing
# immediately.
- def __init__(self, normand: str, variables: SymbolsT, labels: SymbolsT):
+ def __init__(self, normand: str, variables: VariablesT, labels: LabelsT):
self._normand = normand
self._at = 0
self._line_no = 1
self._col_no = 1
self._label_names = set(labels.keys())
self._var_names = set(variables.keys())
+ self._macro_defs = {} # type: _MacroDefsT
self._parse()
# Result (main group).
def res(self):
return self._res
+ # Macro definitions.
+ @property
+ def macro_defs(self):
+ return self._macro_defs
+
# Current text location.
@property
def _text_loc(self):
- return TextLoc._create( # pyright: ignore[reportPrivateUsage]
+ return TextLocation._create( # pyright: ignore[reportPrivateUsage]
self._line_no, self._col_no
)
# Pattern for _skip_ws_and_comments()
_ws_or_syms_or_comments_pat = re.compile(
- r"(?:[\s!@/\\?&:;.,+[\]_=|-]|#[^#]*?(?:\n|#))*"
+ r"(?:[\s/\\?&:;.,[\]_=|-]|#[^#]*?(?:\n|#))*"
)
# Skips as many whitespaces, insignificant symbol characters, and
def _skip_ws_and_comments(self):
self._try_parse_pat(self._ws_or_syms_or_comments_pat)
+ # Pattern for _skip_ws()
+ _ws_pat = re.compile(r"\s*")
+
+ # Skips as many whitespaces as possible.
+ def _skip_ws(self):
+ self._try_parse_pat(self._ws_pat)
+
# Pattern for _try_parse_hex_byte()
_nibble_pat = re.compile(r"[A-Fa-f0-9]")
return _Byte(int("".join(bits), 2), begin_text_loc)
# Patterns for _try_parse_dec_byte()
- _dec_byte_prefix_pat = re.compile(r"\$\s*")
+ _dec_byte_prefix_pat = re.compile(r"\$")
_dec_byte_val_pat = re.compile(r"(?P<neg>-?)(?P<val>\d+)")
# Tries to parse a decimal byte, returning a byte item on success.
return
# Expect the value
+ self._skip_ws()
m = self._expect_pat(self._dec_byte_val_pat, "Expecting a decimal constant")
# Compute value
# Return item
return _Str(data, begin_text_loc)
+ # Common right parenthesis pattern
+ _right_paren_pat = re.compile(r"\)")
+
# Patterns for _try_parse_group()
- _group_prefix_pat = re.compile(r"\(")
- _group_suffix_pat = re.compile(r"\)")
+ _group_prefix_pat = re.compile(r"\(|!g(?:roup)?\b")
# Tries to parse a group, returning a group item on success.
def _try_parse_group(self):
begin_text_loc = self._text_loc
# Match prefix
- if self._try_parse_pat(self._group_prefix_pat) is None:
+ m_open = self._try_parse_pat(self._group_prefix_pat)
+
+ if m_open is None:
# No match
return
# Expect end of group
self._skip_ws_and_comments()
- self._expect_pat(
- self._group_suffix_pat, "Expecting an item or `)` (end of group)"
- )
+
+ if m_open.group(0) == "(":
+ pat = self._right_paren_pat
+ exp = ")"
+ else:
+ pat = self._block_end_pat
+ exp = "!end"
+
+ self._expect_pat(pat, "Expecting an item or `{}` (end of group)".format(exp))
# Return item
return _Group(items, begin_text_loc)
# Returns a stripped expression string and an AST expression node
# from the expression string `expr_str` at text location `text_loc`.
- def _ast_expr_from_str(self, expr_str: str, text_loc: TextLoc):
+ def _ast_expr_from_str(self, expr_str: str, text_loc: TextLocation):
# Create an expression node from the expression string
expr_str = expr_str.strip().replace("\n", " ")
begin_text_loc,
)
- # Patterns for _try_parse_num_and_attr()
- _var_assign_pat = re.compile(
- r"(?P<name>{})\s*=\s*(?P<expr>[^}}]+)".format(_py_name_pat.pattern)
- )
+ # Patterns for _try_parse_var_assign()
+ _var_assign_name_equal_pat = re.compile(r"({})\s*=".format(_py_name_pat.pattern))
+ _var_assign_expr_pat = re.compile(r"[^}]+")
# Tries to parse a variable assignment, returning a variable
# assignment item on success.
begin_text_loc = self._text_loc
# Match
- m = self._try_parse_pat(self._var_assign_pat)
+ m = self._try_parse_pat(self._var_assign_name_equal_pat)
if m is None:
# No match
return
# Validate name
- name = m.group("name")
+ name = m.group(1)
if name == _icitte_name:
_raise_error(
if name in self._label_names:
_raise_error("Existing label named `{}`".format(name), begin_text_loc)
- # Add to known variable names
- self._var_names.add(name)
+ # Expect an expression
+ self._skip_ws()
+ m = self._expect_pat(self._var_assign_expr_pat, "Expecting an expression")
# Create an expression node from the expression string
- expr_str, expr = self._ast_expr_from_str(m.group("expr"), begin_text_loc)
+ expr_str, expr = self._ast_expr_from_str(m.group(0), begin_text_loc)
+
+ # Add to known variable names
+ self._var_names.add(name)
# Return item
return _VarAssign(
return _SetBo(ByteOrder.LE, begin_text_loc)
# Patterns for _try_parse_val_or_bo()
- _val_var_assign_set_bo_prefix_pat = re.compile(r"\{\s*")
- _val_var_assign_set_bo_suffix_pat = re.compile(r"\s*}")
+ _val_var_assign_set_bo_prefix_pat = re.compile(r"\{")
+ _val_var_assign_set_bo_suffix_pat = re.compile(r"\}")
# Tries to parse a value, a variable assignment, or a byte order
# setting, returning an item on success.
# No match
return
+ self._skip_ws()
+
# Variable assignment item?
item = self._try_parse_var_assign()
)
# Expect suffix
+ self._skip_ws()
self._expect_pat(self._val_var_assign_set_bo_suffix_pat, "Expecting `}`")
return item
- # Pattern for _try_parse_set_offset_val() and _try_parse_rep()
+ # Common constant integer patterns
_pos_const_int_pat = re.compile(r"0[Xx][A-Fa-f0-9]+|\d+")
+ _const_int_pat = re.compile(r"(?P<neg>-)?(?:{})".format(_pos_const_int_pat.pattern))
# Tries to parse an offset setting value (after the initial `<`),
# returning an offset item on success.
return _Label(name, begin_text_loc)
# Patterns for _try_parse_label_or_set_offset()
- _label_set_offset_prefix_pat = re.compile(r"<\s*")
- _label_set_offset_suffix_pat = re.compile(r"\s*>")
+ _label_set_offset_prefix_pat = re.compile(r"<")
+ _label_set_offset_suffix_pat = re.compile(r">")
# Tries to parse a label or an offset setting, returning an item on
# success.
return
# Offset setting item?
+ self._skip_ws()
item = self._try_parse_set_offset_val()
if item is None:
self._raise_error("Expecting a label name or an offset setting value")
# Expect suffix
+ self._skip_ws()
self._expect_pat(self._label_set_offset_suffix_pat, "Expecting `>`")
return item
- # Tries to parse a base item (anything except a repetition),
- # returning it on success.
- def _try_parse_base_item(self):
- # Byte item?
- item = self._try_parse_byte()
+ # Pattern for _parse_pad_val()
+ _pad_val_prefix_pat = re.compile(r"~")
+
+ # Tries to parse a padding value, returning the padding value, or 0
+ # if none.
+ def _parse_pad_val(self):
+ # Padding value?
+ self._skip_ws()
+ pad_val = 0
+
+ if self._try_parse_pat(self._pad_val_prefix_pat) is not None:
+ self._skip_ws()
+ pad_val_text_loc = self._text_loc
+ m = self._expect_pat(
+ self._pos_const_int_pat,
+ "Expecting a positive constant integer (byte value)",
+ )
- if item is not None:
- return item
+ # Validate
+ pad_val = int(m.group(0), 0)
- # String item?
- item = self._try_parse_str()
+ if pad_val > 255:
+ _raise_error(
+ "Invalid padding byte value {}".format(pad_val),
+ pad_val_text_loc,
+ )
- if item is not None:
- return item
+ return pad_val
- # Value, variable assignment, or byte order setting item?
- item = self._try_parse_val_or_var_assign_or_set_bo()
+ # Patterns for _try_parse_align_offset()
+ _align_offset_prefix_pat = re.compile(r"@")
+ _align_offset_val_pat = re.compile(r"\d+")
- if item is not None:
- return item
+ # Tries to parse an offset alignment, returning an offset alignment
+ # item on success.
+ def _try_parse_align_offset(self):
+ begin_text_loc = self._text_loc
- # Label or offset setting item?
- item = self._try_parse_label_or_set_offset()
+ # Match prefix
+ if self._try_parse_pat(self._align_offset_prefix_pat) is None:
+ # No match
+ return
- if item is not None:
- return item
+ # Expect an alignment
+ self._skip_ws()
+ align_text_loc = self._text_loc
+ m = self._expect_pat(
+ self._align_offset_val_pat,
+ "Expecting an alignment (positive multiple of eight bits)",
+ )
- # Group item?
- item = self._try_parse_group()
+ # Validate alignment
+ val = int(m.group(0))
- if item is not None:
- return item
+ if val <= 0 or (val % 8) != 0:
+ _raise_error(
+ "Invalid alignment value {} (not a positive multiple of eight)".format(
+ val
+ ),
+ align_text_loc,
+ )
+
+ # Padding value
+ pad_val = self._parse_pad_val()
+
+ # Return item
+ return _AlignOffset(val, pad_val, begin_text_loc)
- # Pattern for _try_parse_rep()
- _rep_prefix_pat = re.compile(r"\*\s*")
- _rep_expr_prefix_pat = re.compile(r"\{")
- _rep_expr_pat = re.compile(r"[^}p]+")
- _rep_expr_suffix_pat = re.compile(r"\}")
+ # Patterns for _try_parse_fill_until()
+ _fill_until_prefix_pat = re.compile(r"\+")
+ _fill_until_pad_val_prefix_pat = re.compile(r"~")
+
+ # Tries to parse a filling, returning a filling item on success.
+ def _try_parse_fill_until(self):
+ begin_text_loc = self._text_loc
- # Tries to parse a repetition, returning the expression string and
- # AST expression node on success.
- def _try_parse_rep(self):
# Match prefix
- if self._try_parse_pat(self._rep_prefix_pat) is None:
+ if self._try_parse_pat(self._fill_until_prefix_pat) is None:
# No match
return
- # Expect and return a decimal multiplier
- self._skip_ws_and_comments()
+ # Expect expression
+ self._skip_ws()
+ expr_str, expr = self._expect_const_int_name_expr(True)
- # Integer?
- m = self._try_parse_pat(self._pos_const_int_pat)
+ # Padding value
+ pad_val = self._parse_pad_val()
+
+ # Return item
+ return _FillUntil(expr_str, expr, pad_val, begin_text_loc)
+
+ # Patterns for _expect_rep_mul_expr()
+ _inner_expr_prefix_pat = re.compile(r"\{")
+ _inner_expr_pat = re.compile(r"[^}]+")
+ _inner_expr_suffix_pat = re.compile(r"\}")
+
+ # Parses a constant integer if `accept_const_int` is `True`
+ # (possibly negative if `allow_neg` is `True`), a name, or an
+ # expression within `{` and `}`.
+ def _expect_const_int_name_expr(
+ self, accept_const_int: bool, allow_neg: bool = False
+ ):
+ expr_text_loc = self._text_loc
+
+ # Constant integer?
+ m = None
+
+ if accept_const_int:
+ m = self._try_parse_pat(self._const_int_pat)
if m is None:
- # Expression?
- if self._try_parse_pat(self._rep_expr_prefix_pat) is None:
- # At this point it's invalid
- self._raise_error("Expecting a positive integral multiplier or `{`")
+ # Name?
+ m = self._try_parse_pat(_py_name_pat)
- # Expect an expression
- expr_str_loc = self._text_loc
- m = self._expect_pat(self._rep_expr_pat, "Expecting an expression")
- expr_str = self._ast_expr_from_str(m.group(0), expr_str_loc)
+ if m is None:
+ # Expression?
+ if self._try_parse_pat(self._inner_expr_prefix_pat) is None:
+ pos_msg = "" if allow_neg else "positive "
- # Expect `}`
- self._expect_pat(self._rep_expr_suffix_pat, "Expecting `}`")
- expr_str = m.group(0)
+ if accept_const_int:
+ mid_msg = "a {}constant integer, a name, or `{{`".format(
+ pos_msg
+ )
+ else:
+ mid_msg = "a name or `{`"
+
+ # At this point it's invalid
+ self._raise_error("Expecting {}".format(mid_msg))
+
+ # Expect an expression
+ self._skip_ws()
+ expr_text_loc = self._text_loc
+ m = self._expect_pat(self._inner_expr_pat, "Expecting an expression")
+ expr_str = m.group(0)
+
+ # Expect `}`
+ self._skip_ws()
+ self._expect_pat(self._inner_expr_suffix_pat, "Expecting `}`")
+ else:
+ expr_str = m.group(0)
else:
- expr_str_loc = self._text_loc
+ if m.group("neg") == "-" and not allow_neg:
+ _raise_error("Expecting a positive constant integer", expr_text_loc)
+
expr_str = m.group(0)
- return self._ast_expr_from_str(expr_str, expr_str_loc)
+ return self._ast_expr_from_str(expr_str, expr_text_loc)
- # Tries to parse an item, possibly followed by a repetition,
- # returning `True` on success.
- #
- # Appends any parsed item to `items`.
- def _try_append_item(self, items: List[_Item]):
- self._skip_ws_and_comments()
+ # Parses the multiplier expression of a repetition (block or
+ # post-item) and returns the expression string and AST node.
+ def _expect_rep_mul_expr(self):
+ return self._expect_const_int_name_expr(True)
- # Parse a base item
- item = self._try_parse_base_item()
+ # Common block end pattern
+ _block_end_pat = re.compile(r"!end\b")
- if item is None:
- # No item
- return False
+ # Pattern for _try_parse_rep_block()
+ _rep_block_prefix_pat = re.compile(r"!r(?:epeat)?\b")
- # Parse repetition if the base item is repeatable
- if isinstance(item, _RepableItem):
- self._skip_ws_and_comments()
- rep_text_loc = self._text_loc
- rep_ret = self._try_parse_rep()
+ # Tries to parse a repetition block, returning a repetition item on
+ # success.
+ def _try_parse_rep_block(self):
+ begin_text_loc = self._text_loc
- if rep_ret is not None:
- item = _Rep(item, rep_ret[0], rep_ret[1], rep_text_loc)
+ # Match prefix
+ if self._try_parse_pat(self._rep_block_prefix_pat) is None:
+ # No match
+ return
- items.append(item)
- return True
+ # Expect expression
+ self._skip_ws_and_comments()
+ expr_str, expr = self._expect_rep_mul_expr()
- # Parses and returns items, skipping whitespaces, insignificant
- # symbols, and comments when allowed, and stopping at the first
- # unknown character.
- def _parse_items(self) -> List[_Item]:
- items = [] # type: List[_Item]
+ # Parse items
+ self._skip_ws_and_comments()
+ items_text_loc = self._text_loc
+ items = self._parse_items()
- while self._isnt_done():
- # Try to append item
- if not self._try_append_item(items):
- # Unknown at this point
- break
+ # Expect end of block
+ self._skip_ws_and_comments()
+ self._expect_pat(
+ self._block_end_pat, "Expecting an item or `!end` (end of repetition block)"
+ )
- return items
+ # Return item
+ return _Rep(_Group(items, items_text_loc), expr_str, expr, begin_text_loc)
- # Parses the whole Normand input, setting `self._res` to the main
- # group item on success.
- def _parse(self):
- if len(self._normand.strip()) == 0:
- # Special case to make sure there's something to consume
- self._res = _Group([], self._text_loc)
+ # Pattern for _try_parse_cond_block()
+ _cond_block_prefix_pat = re.compile(r"!if\b")
+
+ # Tries to parse a conditional block, returning a conditional item
+ # on success.
+ def _try_parse_cond_block(self):
+ begin_text_loc = self._text_loc
+
+ # Match prefix
+ if self._try_parse_pat(self._cond_block_prefix_pat) is None:
+ # No match
return
- # Parse first level items
+ # Expect expression
+ self._skip_ws_and_comments()
+ expr_str, expr = self._expect_const_int_name_expr(False)
+
+ # Parse items
+ self._skip_ws_and_comments()
+ items_text_loc = self._text_loc
items = self._parse_items()
- # Make sure there's nothing left
+ # Expect end of block
self._skip_ws_and_comments()
+ self._expect_pat(
+ self._block_end_pat,
+ "Expecting an item or `!end` (end of conditional block)",
+ )
- if self._isnt_done():
- self._raise_error(
- "Unexpected character `{}`".format(self._normand[self._at])
- )
+ # Return item
+ return _Cond(_Group(items, items_text_loc), expr_str, expr, begin_text_loc)
- # Set main group item
- self._res = _Group(items, self._text_loc)
+ # Common left parenthesis pattern
+ _left_paren_pat = re.compile(r"\(")
+ # Patterns for _try_parse_macro_def() and _try_parse_macro_exp()
+ _macro_params_comma_pat = re.compile(",")
-# The return type of parse().
-class ParseResult:
- @classmethod
- def _create(
- cls,
- data: bytearray,
- variables: SymbolsT,
- labels: SymbolsT,
- offset: int,
- bo: Optional[ByteOrder],
- ):
+ # Patterns for _try_parse_macro_def()
+ _macro_def_prefix_pat = re.compile(r"!m(?:acro)?\b")
+
+ # Tries to parse a macro definition, adding it to `self._macro_defs`
+ # and returning `True` on success.
+ def _try_parse_macro_def(self):
+ begin_text_loc = self._text_loc
+
+ # Match prefix
+ if self._try_parse_pat(self._macro_def_prefix_pat) is None:
+ # No match
+ return False
+
+ # Expect a name
+ self._skip_ws()
+ name_text_loc = self._text_loc
+ m = self._expect_pat(_py_name_pat, "Expecting a valid macro name")
+
+ # Validate name
+ name = m.group(0)
+
+ if name in self._macro_defs:
+ _raise_error("Duplicate macro named `{}`".format(name), name_text_loc)
+
+ # Expect `(`
+ self._skip_ws()
+ self._expect_pat(self._left_paren_pat, "Expecting `(`")
+
+ # Try to parse comma-separated parameter names
+ param_names = [] # type: List[str]
+ expect_comma = False
+
+ while True:
+ self._skip_ws()
+
+ # End?
+ if self._try_parse_pat(self._right_paren_pat) is not None:
+ # End
+ break
+
+ # Comma?
+ if expect_comma:
+ self._expect_pat(self._macro_params_comma_pat, "Expecting `,`")
+
+ # Expect parameter name
+ self._skip_ws()
+ param_text_loc = self._text_loc
+ m = self._expect_pat(_py_name_pat, "Expecting valid parameter name")
+
+ if m.group(0) in param_names:
+ _raise_error(
+ "Duplicate macro parameter named `{}`".format(m.group(0)),
+ param_text_loc,
+ )
+
+ param_names.append(m.group(0))
+ expect_comma = True
+
+ # Expect items
+ self._skip_ws_and_comments()
+ items_text_loc = self._text_loc
+ old_var_names = self._var_names.copy()
+ old_label_names = self._label_names.copy()
+ self._var_names = set() # type: Set[str]
+ self._label_names = set() # type: Set[str]
+ items = self._parse_items()
+ self._var_names = old_var_names
+ self._label_names = old_label_names
+
+ # Expect suffix
+ self._expect_pat(
+ self._block_end_pat, "Expecting an item or `!end` (end of macro block)"
+ )
+
+ # Register macro
+ self._macro_defs[name] = _MacroDef(
+ name, param_names, _Group(items, items_text_loc), begin_text_loc
+ )
+
+ return True
+
+ # Patterns for _try_parse_macro_exp()
+ _macro_exp_prefix_pat = re.compile(r"m\b")
+ _macro_exp_colon_pat = re.compile(r":")
+
+ # Tries to parse a macro expansion, returning a macro expansion item
+ # on success.
+ def _try_parse_macro_exp(self):
+ begin_text_loc = self._text_loc
+
+ # Match prefix
+ if self._try_parse_pat(self._macro_exp_prefix_pat) is None:
+ # No match
+ return
+
+ # Expect `:`
+ self._skip_ws()
+ self._expect_pat(self._macro_exp_colon_pat, "Expecting `:`")
+
+ # Expect a macro name
+ self._skip_ws()
+ name_text_loc = self._text_loc
+ m = self._expect_pat(_py_name_pat, "Expecting a valid macro name")
+
+ # Validate name
+ name = m.group(0)
+ macro_def = self._macro_defs.get(name)
+
+ if macro_def is None:
+ _raise_error("Unknown macro name `{}`".format(name), name_text_loc)
+
+ # Expect `(`
+ self._skip_ws()
+ self._expect_pat(self._left_paren_pat, "Expecting `(`")
+
+ # Try to parse comma-separated parameter values
+ params_text_loc = self._text_loc
+ params = [] # type: List[_MacroExpParam]
+ expect_comma = False
+
+ while True:
+ self._skip_ws()
+
+ # End?
+ if self._try_parse_pat(self._right_paren_pat) is not None:
+ # End
+ break
+
+ # Expect a Value
+ if expect_comma:
+ self._expect_pat(self._macro_params_comma_pat, "Expecting `,`")
+
+ self._skip_ws()
+ param_text_loc = self._text_loc
+ params.append(
+ _MacroExpParam(
+ *self._expect_const_int_name_expr(True, True), param_text_loc
+ )
+ )
+ expect_comma = True
+
+ # Validate parameter values
+ if len(params) != len(macro_def.param_names):
+ sing_plur = "" if len(params) == 1 else "s"
+ _raise_error(
+ "Macro expansion passes {} parameter{} while the definition expects {}".format(
+ len(params), sing_plur, len(macro_def.param_names)
+ ),
+ params_text_loc,
+ )
+
+ # Return item
+ return _MacroExp(name, params, begin_text_loc)
+
+ # Tries to parse a base item (anything except a repetition),
+ # returning it on success.
+ def _try_parse_base_item(self):
+ # Byte item?
+ item = self._try_parse_byte()
+
+ if item is not None:
+ return item
+
+ # String item?
+ item = self._try_parse_str()
+
+ if item is not None:
+ return item
+
+ # Value, variable assignment, or byte order setting item?
+ item = self._try_parse_val_or_var_assign_or_set_bo()
+
+ if item is not None:
+ return item
+
+ # Label or offset setting item?
+ item = self._try_parse_label_or_set_offset()
+
+ if item is not None:
+ return item
+
+ # Offset alignment item?
+ item = self._try_parse_align_offset()
+
+ if item is not None:
+ return item
+
+ # Filling item?
+ item = self._try_parse_fill_until()
+
+ if item is not None:
+ return item
+
+ # Group item?
+ item = self._try_parse_group()
+
+ if item is not None:
+ return item
+
+ # Repetition block item?
+ item = self._try_parse_rep_block()
+
+ if item is not None:
+ return item
+
+ # Conditional block item?
+ item = self._try_parse_cond_block()
+
+ if item is not None:
+ return item
+
+ # Macro expansion?
+ item = self._try_parse_macro_exp()
+
+ if item is not None:
+ return item
+
+ # Pattern for _try_parse_rep_post()
+ _rep_post_prefix_pat = re.compile(r"\*")
+
+ # Tries to parse a post-item repetition, returning the expression
+ # string and AST expression node on success.
+ def _try_parse_rep_post(self):
+ # Match prefix
+ if self._try_parse_pat(self._rep_post_prefix_pat) is None:
+ # No match
+ return
+
+ # Return expression string and AST expression
+ self._skip_ws_and_comments()
+ return self._expect_rep_mul_expr()
+
+ # Tries to parse an item, possibly followed by a repetition,
+ # returning `True` on success.
+ #
+ # Appends any parsed item to `items`.
+ def _try_append_item(self, items: List[_Item]):
+ self._skip_ws_and_comments()
+
+ # Base item
+ item = self._try_parse_base_item()
+
+ if item is None:
+ return
+
+ # Parse repetition if the base item is repeatable
+ if isinstance(item, _RepableItem):
+ self._skip_ws_and_comments()
+ rep_text_loc = self._text_loc
+ rep_ret = self._try_parse_rep_post()
+
+ if rep_ret is not None:
+ item = _Rep(item, *rep_ret, rep_text_loc)
+
+ items.append(item)
+ return True
+
+ # Parses and returns items, skipping whitespaces, insignificant
+ # symbols, and comments when allowed, and stopping at the first
+ # unknown character.
+ #
+ # Accepts and registers macro definitions if `accept_macro_defs`
+ # is `True`.
+ def _parse_items(self, accept_macro_defs: bool = False) -> List[_Item]:
+ items = [] # type: List[_Item]
+
+ while self._isnt_done():
+ # Try to append item
+ if not self._try_append_item(items):
+ if accept_macro_defs and self._try_parse_macro_def():
+ continue
+
+ # Unknown at this point
+ break
+
+ return items
+
+ # Parses the whole Normand input, setting `self._res` to the main
+ # group item on success.
+ def _parse(self):
+ if len(self._normand.strip()) == 0:
+ # Special case to make sure there's something to consume
+ self._res = _Group([], self._text_loc)
+ return
+
+ # Parse first level items
+ items = self._parse_items(True)
+
+ # Make sure there's nothing left
+ self._skip_ws_and_comments()
+
+ if self._isnt_done():
+ self._raise_error(
+ "Unexpected character `{}`".format(self._normand[self._at])
+ )
+
+ # Set main group item
+ self._res = _Group(items, self._text_loc)
+
+
+# The return type of parse().
+class ParseResult:
+ @classmethod
+ def _create(
+ cls,
+ data: bytearray,
+ variables: VariablesT,
+ labels: LabelsT,
+ offset: int,
+ bo: Optional[ByteOrder],
+ ):
self = cls.__new__(cls)
self._init(data, variables, labels, offset, bo)
return self
def _init(
self,
data: bytearray,
- variables: SymbolsT,
- labels: SymbolsT,
+ variables: VariablesT,
+ labels: LabelsT,
offset: int,
bo: Optional[ByteOrder],
):
# Expression validator: validates that all the names within the
# expression are allowed.
class _ExprValidator(_NodeVisitor):
- def __init__(self, item: _ExprItemT, allowed_names: Set[str], icitte_allowed: bool):
+ def __init__(self, expr_str: str, text_loc: TextLocation, allowed_names: Set[str]):
super().__init__()
- self._item = item
+ self._expr_str = expr_str
+ self._text_loc = text_loc
self._allowed_names = allowed_names
- self._icitte_allowed = icitte_allowed
def _visit_name(self, name: str):
# Make sure the name refers to a known and reachable
# variable/label name.
- if name == _icitte_name and not self._icitte_allowed:
- _raise_error(
- "Illegal reserved name `{}` in expression `{}`".format(
- _icitte_name, self._item.expr_str
- ),
- self._item.text_loc,
- )
- elif name != _icitte_name and name not in self._allowed_names:
+ if name != _icitte_name and name not in self._allowed_names:
msg = "Illegal (unknown or unreachable) variable/label name `{}` in expression `{}`".format(
- name, self._item.expr_str
+ name, self._expr_str
)
allowed_names = self._allowed_names.copy()
-
- if self._icitte_allowed:
- allowed_names.add(_icitte_name)
+ allowed_names.add(_icitte_name)
if len(allowed_names) > 0:
allowed_names_str = ", ".join(
_raise_error(
msg,
- self._item.text_loc,
+ self._text_loc,
)
-# Expression visitor getting all the contained names.
-class _ExprNamesVisitor(_NodeVisitor):
- def __init__(self):
- self._parent_is_call = False
- self._names = set() # type: Set[str]
-
- @property
- def names(self):
- return self._names
-
- def _visit_name(self, name: str):
- self._names.add(name)
-
-
# Generator state.
class _GenState:
def __init__(
self,
- variables: SymbolsT,
- labels: SymbolsT,
+ variables: VariablesT,
+ labels: LabelsT,
offset: int,
bo: Optional[ByteOrder],
):
self.offset = offset
self.bo = bo
+ def __repr__(self):
+ return "_GenState({}, {}, {}, {})".format(
+ repr(self.variables), repr(self.labels), repr(self.offset), repr(self.bo)
+ )
+
+
+# Fixed-length number item instance.
+class _FlNumItemInst:
+ def __init__(self, item: _FlNum, offset_in_data: int, state: _GenState):
+ self._item = item
+ self._offset_in_data = offset_in_data
+ self._state = state
+
+ @property
+ def item(self):
+ return self._item
+
+ @property
+ def offset_in_data(self):
+ return self._offset_in_data
+
+ @property
+ def state(self):
+ return self._state
+
# Generator of data and final state from a group item.
#
#
# The steps of generation are:
#
-# 1. Validate that each repetition and LEB128 integer expression uses
-# only reachable names and not `ICITTE`.
+# 1. Handle each item in prefix order.
+#
+# The handlers append bytes to `self._data` and update some current
+# state object (`_GenState` instance).
+#
+# When handling a fixed-length number item, try to evaluate its
+# expression using the current state. If this fails, then it might be
+# because the expression refers to a "future" label: save the current
+# offset in `self._data` (generated data) and a snapshot of the
+# current state within `self._fl_num_item_insts` (`_FlNumItemInst`
+# object). _gen_fl_num_item_insts() will deal with this later.
#
-# 2. Compute and keep the effective repetition count and LEB128 integer
-# value for each repetition and LEB128 integer instance.
+# When handling the items of a group, keep a map of immediate label
+# names to their offset. Then, after having processed all the items,
+# update the relevant saved state snapshots in
+# `self._fl_num_item_insts` with those immediate label values.
+# _gen_fl_num_item_insts() will deal with this later.
#
-# 3. Generate bytes, updating the initial state as it goes which becomes
-# the final state after the operation.
+# 2. Handle all the fixed-length number item instances of which the
+# expression evaluation failed before.
#
-# During the generation, when handling a `_Rep` or `_Leb128Int` item,
-# we already have the effective repetition count or value of the
-# instance.
+# At this point, `self._fl_num_item_insts` contains everything that's
+# needed to evaluate the expressions, including the values of
+# "future" labels from the point of view of some fixed-length number
+# item instance.
#
-# When handling a `_Group` item, first update the current labels with
-# all the immediate (not nested) labels, and then handle each
-# contained item. This gives contained item access to "future" outer
-# labels. Then remove the immediate labels from the state so that
-# outer items don't have access to inner labels.
+# If an evaluation fails at this point, then it's a user error.
class _Gen:
def __init__(
self,
group: _Group,
- variables: SymbolsT,
- labels: SymbolsT,
+ macro_defs: _MacroDefsT,
+ variables: VariablesT,
+ labels: LabelsT,
offset: int,
bo: Optional[ByteOrder],
):
- self._validate_vl_exprs(group, set(variables.keys()), set(labels.keys()))
- self._vl_instance_vals = self._compute_vl_instance_vals(
- group, _GenState(variables, labels, offset, bo)
- )
+ self._macro_defs = macro_defs
+ self._fl_num_item_insts = [] # type: List[_FlNumItemInst]
self._gen(group, _GenState(variables, labels, offset, bo))
# Generated bytes.
def bo(self):
return self._final_state.bo
- # Returns the set of used, non-called names within the AST
- # expression `expr`.
- @staticmethod
- def _names_of_expr(expr: ast.Expression):
- visitor = _ExprNamesVisitor()
- visitor.visit(expr)
- return visitor.names
-
- # Validates that all the repetition and LEB128 integer expressions
- # within `group` don't refer, directly or indirectly, to subsequent
- # labels.
- #
- # The strategy here is to keep a set of allowed label names, per
- # group, initialized to `allowed_label_names`, and a set of allowed
- # variable names initialized to `allowed_variable_names`.
- #
- # Then, depending on the type of `item`:
- #
- # `_Label`:
- # Add its name to the local allowed label names: a label
- # occurring before a repetition, and not within a nested group,
- # is always reachable.
- #
- # `_VarAssign`:
- # If all the names within its expression are allowed, then add
- # its name to the allowed variable names.
- #
- # Otherwise, remove its name from the allowed variable names (if
- # it's in there): a variable which refers to an unreachable name
- # is unreachable itself.
- #
- # `_Rep` and `_Leb128`:
- # Make sure all the names within its expression are allowed.
- #
- # `_Group`:
- # Call this function for each contained item with a _copy_ of
- # the current allowed label names and the same current allowed
- # variable names.
- @staticmethod
- def _validate_vl_exprs(
- item: _Item, allowed_variable_names: Set[str], allowed_label_names: Set[str]
- ):
- if type(item) is _Label:
- allowed_label_names.add(item.name)
- elif type(item) is _VarAssign:
- # Check if this variable name is allowed
- allowed = True
-
- for name in _Gen._names_of_expr(item.expr):
- if name not in (
- allowed_label_names | allowed_variable_names | {_icitte_name}
- ):
- # Not allowed
- allowed = False
- break
-
- if allowed:
- allowed_variable_names.add(item.name)
- elif item.name in allowed_variable_names:
- allowed_variable_names.remove(item.name)
- elif isinstance(item, _Leb128Int):
- # Validate the expression (`ICITTE` allowed)
- _ExprValidator(
- item, allowed_label_names | allowed_variable_names, True
- ).visit(item.expr)
- elif type(item) is _Rep:
- # Validate the expression first (`ICITTE` not allowed)
- _ExprValidator(
- item, allowed_label_names | allowed_variable_names, False
- ).visit(item.expr)
-
- # Validate inner item
- _Gen._validate_vl_exprs(
- item.item, allowed_variable_names, allowed_label_names
- )
- elif type(item) is _Group:
- # Copy `allowed_label_names` so that this frame cannot
- # access the nested label names.
- group_allowed_label_names = allowed_label_names.copy()
-
- for subitem in item.items:
- _Gen._validate_vl_exprs(
- subitem, allowed_variable_names, group_allowed_label_names
- )
-
- # Evaluates the expression of `item` considering the current
+ # Evaluates the expression `expr` of which the original string is
+ # `expr_str` at the location `text_loc` considering the current
# generation state `state`.
#
- # If `allow_icitte` is `True`, then the `ICITTE` name is available
- # for the expression to evaluate.
- #
# If `allow_float` is `True`, then the type of the result may be
# `float` too.
@staticmethod
- def _eval_item_expr(
- item: _ExprItemT,
+ def _eval_expr(
+ expr_str: str,
+ expr: ast.Expression,
+ text_loc: TextLocation,
state: _GenState,
- allow_icitte: bool,
allow_float: bool = False,
):
- syms = state.labels.copy()
+ syms = {} # type: VariablesT
+ syms.update(state.labels)
- # Set the `ICITTE` name to the current offset, if any
- if allow_icitte:
- syms[_icitte_name] = state.offset
+ # Set the `ICITTE` name to the current offset
+ syms[_icitte_name] = state.offset
# Add the current variables
syms.update(state.variables)
# Validate the node and its children
- _ExprValidator(item, set(syms.keys()), True).visit(item.expr)
+ _ExprValidator(expr_str, text_loc, set(syms.keys())).visit(expr)
# Compile and evaluate expression node
try:
- val = eval(compile(item.expr, "", "eval"), None, syms)
+ val = eval(compile(expr, "", "eval"), None, syms)
except Exception as exc:
- _raise_error_for_item(
- "Failed to evaluate expression `{}`: {}".format(item.expr_str, exc),
- item,
+ _raise_error(
+ "Failed to evaluate expression `{}`: {}".format(expr_str, exc),
+ text_loc,
)
+ # Convert `bool` result type to `int` to normalize
+ if type(val) is bool:
+ val = int(val)
+
# Validate result type
expected_types = {int} # type: Set[type]
type_msg = "`int`"
type_msg += " or `float`"
if type(val) not in expected_types:
- _raise_error_for_item(
+ _raise_error(
"Invalid expression `{}`: expecting result type {}, not `{}`".format(
- item.expr_str, type_msg, type(val).__name__
+ expr_str, type_msg, type(val).__name__
),
- item,
+ text_loc,
)
return val
+ # Evaluates the expression of `item` considering the current
+ # generation state `state`.
+ #
+ # If `allow_float` is `True`, then the type of the result may be
+ # `float` too.
+ @staticmethod
+ def _eval_item_expr(
+ item: Union[_FlNum, _Leb128Int, _FillUntil, _VarAssign, _Rep, _Cond],
+ state: _GenState,
+ allow_float: bool = False,
+ ):
+ return _Gen._eval_expr(
+ item.expr_str, item.expr, item.text_loc, state, allow_float
+ )
+
+ # Handles the byte item `item`.
+ def _handle_byte_item(self, item: _Byte, state: _GenState):
+ self._data.append(item.val)
+ state.offset += item.size
+
+ # Handles the string item `item`.
+ def _handle_str_item(self, item: _Str, state: _GenState):
+ self._data += item.data
+ state.offset += item.size
+
+ # Handles the byte order setting item `item`.
+ def _handle_set_bo_item(self, item: _SetBo, state: _GenState):
+ # Update current byte order
+ state.bo = item.bo
+
+ # Handles the variable assignment item `item`.
+ def _handle_var_assign_item(self, item: _VarAssign, state: _GenState):
+ # Update variable
+ state.variables[item.name] = self._eval_item_expr(item, state, True)
+
+ # Handles the fixed-length number item `item`.
+ def _handle_fl_num_item(self, item: _FlNum, state: _GenState):
+ # Validate current byte order
+ if state.bo is None and item.len > 8:
+ _raise_error_for_item(
+ "Current byte order isn't defined at first fixed-length number (`{}`) to encode on more than 8 bits".format(
+ item.expr_str
+ ),
+ item,
+ )
+
+ # Try an immediate evaluation. If it fails, then keep everything
+ # needed to (try to) generate the bytes of this item later.
+ try:
+ data = self._gen_fl_num_item_inst_data(item, state)
+ except Exception:
+ self._fl_num_item_insts.append(
+ _FlNumItemInst(item, len(self._data), copy.deepcopy(state))
+ )
+
+ # Reserve space in `self._data` for this instance
+ data = bytes([0] * (item.len // 8))
+
+ # Append bytes
+ self._data += data
+
+ # Update offset
+ state.offset += len(data)
+
# Returns the size, in bytes, required to encode the value `val`
# with LEB128 (signed version if `is_signed` is `True`).
@staticmethod
# Seven bits per byte
return math.ceil(bits / 7)
- # Computes the effective value for each repetition and LEB128
- # integer instance, filling `instance_vals` (if not `None`) and
- # returning `instance_vals`.
- #
- # At this point it must be known that, for a given variable-length
- # item, its expression only contains reachable names.
- #
- # When handling a `_Rep` item, this function appends its effective
- # multiplier to `instance_vals` _before_ handling its repeated item.
- #
- # When handling a `_VarAssign` item, this function only evaluates it
- # if all its names are reachable.
- @staticmethod
- def _compute_vl_instance_vals(
- item: _Item, state: _GenState, instance_vals: Optional[List[int]] = None
- ):
- if instance_vals is None:
- instance_vals = []
-
- if isinstance(item, _ScalarItem):
- state.offset += item.size
- elif type(item) is _Label:
- state.labels[item.name] = state.offset
- elif type(item) is _VarAssign:
- # Check if all the names are reachable
- do_eval = True
-
- for name in _Gen._names_of_expr(item.expr):
- if (
- name != _icitte_name
- and name not in state.variables
- and name not in state.labels
- ):
- # A name is unknown: cannot evaluate
- do_eval = False
- break
-
- if do_eval:
- # Evaluate the expression and keep the result
- state.variables[item.name] = _Gen._eval_item_expr(
- item, state, True, True
- )
- elif type(item) is _SetOffset:
- state.offset = item.val
- elif isinstance(item, _Leb128Int):
- # Evaluate the expression
- val = _Gen._eval_item_expr(item, state, True)
-
- # Validate result
- if type(item) is _ULeb128Int and val < 0:
- _raise_error_for_item(
- "Invalid expression `{}`: unexpected negative result {:,} for a ULEB128 encoding".format(
- item.expr_str, val
- ),
- item,
- )
+ # Handles the LEB128 integer item `item`.
+ def _handle_leb128_int_item(self, item: _Leb128Int, state: _GenState):
+ # Compute value
+ val = self._eval_item_expr(item, state, False)
- # Add the evaluation result to the to variable-length item
- # instance values.
- instance_vals.append(val)
-
- # Update offset
- state.offset += _Gen._leb128_size_for_val(val, type(item) is _SLeb128Int)
- elif type(item) is _Rep:
- # Evaluate the expression and keep the result
- val = _Gen._eval_item_expr(item, state, False)
-
- # Validate result
- if val < 0:
- _raise_error_for_item(
- "Invalid expression `{}`: unexpected negative result {:,}".format(
- item.expr_str, val
- ),
- item,
- )
+ # Size in bytes
+ size = self._leb128_size_for_val(val, type(item) is _SLeb128Int)
- # Add to repetition instance values
- instance_vals.append(val)
+ # For each byte
+ for _ in range(size):
+ # Seven LSBs, MSB of the byte set (continue)
+ self._data.append((val & 0x7F) | 0x80)
+ val >>= 7
- # Process the repeated item `val` times
- for _ in range(val):
- _Gen._compute_vl_instance_vals(item.item, state, instance_vals)
- elif type(item) is _Group:
- prev_labels = state.labels.copy()
+ # Clear MSB of last byte (stop)
+ self._data[-1] &= ~0x80
- # Process each item
- for subitem in item.items:
- _Gen._compute_vl_instance_vals(subitem, state, instance_vals)
+ # Update offset
+ state.offset += size
- state.labels = prev_labels
+ # Handles the group item `item`, removing the immediate labels from
+ # `state` at the end if `remove_immediate_labels` is `True`.
+ def _handle_group_item(
+ self, item: _Group, state: _GenState, remove_immediate_labels: bool = True
+ ):
+ first_fl_num_item_inst_index = len(self._fl_num_item_insts)
+ immediate_labels = {} # type: LabelsT
- return instance_vals
+ # Handle each item
+ for subitem in item.items:
+ if type(subitem) is _Label:
+ # Add to local immediate labels
+ immediate_labels[subitem.name] = state.offset
- def _zero_item_size(self, item: _Item, next_vl_instance: int):
- return 0, next_vl_instance
+ self._handle_item(subitem, state)
- def _scalar_item_size(self, item: _ScalarItem, next_vl_instance: int):
- return item.size, next_vl_instance
+ # Remove immediate labels from current state if needed
+ if remove_immediate_labels:
+ for name in immediate_labels:
+ del state.labels[name]
- def _leb128_int_item_size(self, item: _Leb128Int, next_vl_instance: int):
- # Get the value from `self._vl_instance_vals` _before_
- # incrementing `next_vl_instance` to honor the order of
- # _compute_vl_instance_vals().
- return (
- self._leb128_size_for_val(
- self._vl_instance_vals[next_vl_instance], type(item) is _SLeb128Int
- ),
- next_vl_instance + 1,
- )
+ # Add all immediate labels to all state snapshots since
+ # `first_fl_num_item_inst_index`.
+ for inst in self._fl_num_item_insts[first_fl_num_item_inst_index:]:
+ inst.state.labels.update(immediate_labels)
- def _group_item_size(self, item: _Group, next_vl_instance: int):
- size = 0
+ # Handles the repetition item `item`.
+ def _handle_rep_item(self, item: _Rep, state: _GenState):
+ # Compute the repetition count
+ mul = _Gen._eval_item_expr(item, state)
- for subitem in item.items:
- subitem_size, next_vl_instance = self._item_size(subitem, next_vl_instance)
- size += subitem_size
+ # Validate result
+ if mul < 0:
+ _raise_error_for_item(
+ "Invalid expression `{}`: unexpected negative result {:,}".format(
+ item.expr_str, mul
+ ),
+ item,
+ )
- return size, next_vl_instance
+ # Generate item data `mul` times
+ for _ in range(mul):
+ self._handle_item(item.item, state)
+
+ # Handles the conditional item `item`.
+ def _handle_cond_item(self, item: _Rep, state: _GenState):
+ # Compute the conditional value
+ val = _Gen._eval_item_expr(item, state)
+
+ # Generate item data if needed
+ if val:
+ self._handle_item(item.item, state)
+
+ # Evaluates the parameters of the macro expansion item `item`
+ # considering the initial state `init_state` and returns a new state
+ # to handle the items of the macro.
+ def _eval_macro_exp_params(self, item: _MacroExp, init_state: _GenState):
+ # New state
+ exp_state = _GenState({}, {}, init_state.offset, init_state.bo)
+
+ # Evaluate the parameter expressions
+ macro_def = self._macro_defs[item.name]
+
+ for param_name, param in zip(macro_def.param_names, item.params):
+ exp_state.variables[param_name] = _Gen._eval_expr(
+ param.expr_str, param.expr, param.text_loc, init_state, True
+ )
- def _rep_item_size(self, item: _Rep, next_vl_instance: int):
- # Get the value from `self._vl_instance_vals` _before_
- # incrementing `next_vl_instance` to honor the order of
- # _compute_vl_instance_vals().
- mul = self._vl_instance_vals[next_vl_instance]
- next_vl_instance += 1
- size = 0
+ return exp_state
- for _ in range(mul):
- iter_size, next_vl_instance = self._item_size(item.item, next_vl_instance)
- size += iter_size
+ # Handles the macro expansion item `item`.
+ def _handle_macro_exp_item(self, item: _MacroExp, state: _GenState):
+ # New state
+ exp_state = self._eval_macro_exp_params(item, state)
- return size, next_vl_instance
+ # Process the contained group
+ init_data_size = len(self._data)
+ self._handle_item(self._macro_defs[item.name].group, exp_state)
- # Returns the size of `item` and the new next repetition instance.
- def _item_size(self, item: _Item, next_vl_instance: int):
- return self._item_size_funcs[type(item)](item, next_vl_instance)
+ # Update state offset and return
+ state.offset += len(self._data) - init_data_size
- # Handles the byte item `item`.
- def _handle_byte_item(self, item: _Byte, state: _GenState, next_vl_instance: int):
- self._data.append(item.val)
- state.offset += item.size
- return next_vl_instance
+ # Handles the offset setting item `item`.
+ def _handle_set_offset_item(self, item: _SetOffset, state: _GenState):
+ state.offset = item.val
- # Handles the string item `item`.
- def _handle_str_item(self, item: _Str, state: _GenState, next_vl_instance: int):
- self._data += item.data
- state.offset += item.size
- return next_vl_instance
+ # Handles the offset alignment item `item` (adds padding).
+ def _handle_align_offset_item(self, item: _AlignOffset, state: _GenState):
+ init_offset = state.offset
+ align_bytes = item.val // 8
+ state.offset = (state.offset + align_bytes - 1) // align_bytes * align_bytes
+ self._data += bytes([item.pad_val] * (state.offset - init_offset))
- # Handles the byte order setting item `item`.
- def _handle_set_bo_item(
- self, item: _SetBo, state: _GenState, next_vl_instance: int
- ):
- # Update current byte order
- state.bo = item.bo
- return next_vl_instance
+ # Handles the filling item `item` (adds padding).
+ def _handle_fill_until_item(self, item: _FillUntil, state: _GenState):
+ # Compute the new offset
+ new_offset = _Gen._eval_item_expr(item, state)
- # Handles the variable assignment item `item`.
- def _handle_var_assign_item(
- self, item: _VarAssign, state: _GenState, next_vl_instance: int
- ):
- # Update variable
- state.variables[item.name] = self._eval_item_expr(item, state, True, True)
- return next_vl_instance
+ # Validate the new offset
+ if new_offset < state.offset:
+ _raise_error_for_item(
+ "Invalid expression `{}`: new offset {:,} is less than current offset {:,}".format(
+ item.expr_str, new_offset, state.offset
+ ),
+ item,
+ )
+
+ # Fill
+ self._data += bytes([item.pad_val] * (new_offset - state.offset))
+
+ # Update offset
+ state.offset = new_offset
+
+ # Handles the label item `item`.
+ def _handle_label_item(self, item: _Label, state: _GenState):
+ state.labels[item.name] = state.offset
+
+ # Handles the item `item`, returning the updated next repetition
+ # instance.
+ def _handle_item(self, item: _Item, state: _GenState):
+ return self._item_handlers[type(item)](item, state)
- # Handles the fixed-length integer item `item`.
- def _handle_fl_int_item(self, val: int, item: _FlNum, state: _GenState):
+ # Generates the data for a fixed-length integer item instance having
+ # the value `val` and returns it.
+ def _gen_fl_int_item_inst_data(self, val: int, item: _FlNum, state: _GenState):
# Validate range
if val < -(2 ** (item.len - 1)) or val > 2**item.len - 1:
_raise_error_for_item(
- "Value {:,} is outside the {}-bit range when evaluating expression `{}` at byte offset {:,}".format(
- val, item.len, item.expr_str, state.offset
+ "Value {:,} is outside the {}-bit range when evaluating expression `{}`".format(
+ val, item.len, item.expr_str
),
item,
)
assert state.bo == ByteOrder.LE
data = data[:len_bytes]
- # Append to current bytes and update offset
- self._data += data
+ # Return data
+ return data
- # Handles the fixed-length integer item `item`.
- def _handle_fl_float_item(self, val: float, item: _FlNum, state: _GenState):
+ # Generates the data for a fixed-length floating point number item
+ # instance having the value `val` and returns it.
+ def _gen_fl_float_item_inst_data(self, val: float, item: _FlNum, state: _GenState):
# Validate length
if item.len not in (32, 64):
_raise_error_for_item(
item,
)
- # Encode result
- self._data += struct.pack(
+ # Encode and return result
+ return struct.pack(
"{}{}".format(
">" if state.bo in (None, ByteOrder.BE) else "<",
"f" if item.len == 32 else "d",
val,
)
- # Handles the fixed-length number item `item`.
- def _handle_fl_num_item(
- self, item: _FlNum, state: _GenState, next_vl_instance: int
- ):
+ # Generates the data for a fixed-length number item instance and
+ # returns it.
+ def _gen_fl_num_item_inst_data(self, item: _FlNum, state: _GenState):
# Compute value
- val = self._eval_item_expr(item, state, True, True)
-
- # Validate current byte order
- if state.bo is None and item.len > 8:
- _raise_error_for_item(
- "Current byte order isn't defined at first fixed-length number (`{}`) to encode on more than 8 bits".format(
- item.expr_str
- ),
- item,
- )
+ val = self._eval_item_expr(item, state, True)
# Handle depending on type
if type(val) is int:
- self._handle_fl_int_item(val, item, state)
+ return self._gen_fl_int_item_inst_data(val, item, state)
else:
assert type(val) is float
- self._handle_fl_float_item(val, item, state)
-
- # Update offset
- state.offset += item.size
-
- return next_vl_instance
-
- # Handles the LEB128 integer item `item`.
- def _handle_leb128_int_item(
- self, item: _Leb128Int, state: _GenState, next_vl_instance: int
- ):
- # Get the precomputed value
- val = self._vl_instance_vals[next_vl_instance]
-
- # Size in bytes
- size = self._leb128_size_for_val(val, type(item) is _SLeb128Int)
+ return self._gen_fl_float_item_inst_data(val, item, state)
- # For each byte
- for _ in range(size):
- # Seven LSBs, MSB of the byte set (continue)
- self._data.append((val & 0x7F) | 0x80)
- val >>= 7
+ # Generates the data for all the fixed-length number item instances
+ # and writes it at the correct offset within `self._data`.
+ def _gen_fl_num_item_insts(self):
+ for inst in self._fl_num_item_insts:
+ # Generate bytes
+ data = self._gen_fl_num_item_inst_data(inst.item, inst.state)
- # Clear MSB of last byte (stop)
- self._data[-1] &= ~0x80
-
- # Consumed this instance
- return next_vl_instance + 1
-
- # Handles the group item `item`, only removing the immediate labels
- # from `state.labels` if `remove_immediate_labels` is `True`.
- def _handle_group_item(
- self,
- item: _Group,
- state: _GenState,
- next_vl_instance: int,
- remove_immediate_labels: bool = True,
- ):
- # Compute the values of the immediate (not nested) labels. Those
- # labels are reachable by any expression within the group.
- offset = state.offset
- immediate_label_names = set() # type: Set[str]
- tmp_next_vl_instance = next_vl_instance
-
- for subitem in item.items:
- if type(subitem) is _SetOffset:
- # Update offset
- offset = subitem.val
- elif type(subitem) is _Label:
- # New immediate label
- state.labels[subitem.name] = offset
- immediate_label_names.add(subitem.name)
-
- subitem_size, tmp_next_vl_instance = self._item_size(
- subitem, tmp_next_vl_instance
- )
- offset += subitem_size
-
- # Handle each item now with the actual state
- for subitem in item.items:
- next_vl_instance = self._handle_item(subitem, state, next_vl_instance)
-
- # Remove immediate labels if required so that outer items won't
- # reach inner labels.
- if remove_immediate_labels:
- for name in immediate_label_names:
- del state.labels[name]
-
- return next_vl_instance
-
- # Handles the repetition item `item`.
- def _handle_rep_item(self, item: _Rep, state: _GenState, next_vl_instance: int):
- # Get the precomputed repetition count
- mul = self._vl_instance_vals[next_vl_instance]
-
- # Consumed this instance
- next_vl_instance += 1
-
- for _ in range(mul):
- next_vl_instance = self._handle_item(item.item, state, next_vl_instance)
-
- return next_vl_instance
-
- # Handles the offset setting item `item`.
- def _handle_set_offset_item(
- self, item: _SetOffset, state: _GenState, next_vl_instance: int
- ):
- state.offset = item.val
- return next_vl_instance
-
- # Handles the label item `item`.
- def _handle_label_item(self, item: _Label, state: _GenState, next_vl_instance: int):
- return next_vl_instance
-
- # Handles the item `item`, returning the updated next repetition
- # instance.
- def _handle_item(self, item: _Item, state: _GenState, next_vl_instance: int):
- return self._item_handlers[type(item)](item, state, next_vl_instance)
+ # Insert bytes into `self._data`
+ self._data[inst.offset_in_data : inst.offset_in_data + len(data)] = data
# Generates the data (`self._data`) and final state
# (`self._final_state`) from `group` and the initial state `state`.
# Item handlers
self._item_handlers = {
+ _AlignOffset: self._handle_align_offset_item,
_Byte: self._handle_byte_item,
+ _Cond: self._handle_cond_item,
+ _FillUntil: self._handle_fill_until_item,
_FlNum: self._handle_fl_num_item,
_Group: self._handle_group_item,
_Label: self._handle_label_item,
+ _MacroExp: self._handle_macro_exp_item,
_Rep: self._handle_rep_item,
_SetBo: self._handle_set_bo_item,
_SetOffset: self._handle_set_offset_item,
_Str: self._handle_str_item,
_ULeb128Int: self._handle_leb128_int_item,
_VarAssign: self._handle_var_assign_item,
- } # type: Dict[type, Callable[[Any, _GenState, int], int]]
-
- # Item size getters
- self._item_size_funcs = {
- _Byte: self._scalar_item_size,
- _FlNum: self._scalar_item_size,
- _Group: self._group_item_size,
- _Label: self._zero_item_size,
- _Rep: self._rep_item_size,
- _SetBo: self._zero_item_size,
- _SetOffset: self._zero_item_size,
- _SLeb128Int: self._leb128_int_item_size,
- _Str: self._scalar_item_size,
- _ULeb128Int: self._leb128_int_item_size,
- _VarAssign: self._zero_item_size,
- } # type: Dict[type, Callable[[Any, int], Tuple[int, int]]]
+ } # type: Dict[type, Callable[[Any, _GenState], None]]
# Handle the group item, _not_ removing the immediate labels
# because the `labels` property offers them.
- self._handle_group_item(group, state, 0, False)
+ self._handle_group_item(group, state, False)
# This is actually the final state
self._final_state = state
+ # Generate all the fixed-length number bytes now that we know
+ # their full state
+ self._gen_fl_num_item_insts()
+
# Returns a `ParseResult` instance containing the bytes encoded by the
# input string `normand`.
# Raises `ParseError` on any parsing error.
def parse(
normand: str,
- init_variables: Optional[SymbolsT] = None,
- init_labels: Optional[SymbolsT] = None,
+ init_variables: Optional[VariablesT] = None,
+ init_labels: Optional[LabelsT] = None,
init_offset: int = 0,
init_byte_order: Optional[ByteOrder] = None,
):
if init_labels is None:
init_labels = {}
+ parser = _Parser(normand, init_variables, init_labels)
gen = _Gen(
- _Parser(normand, init_variables, init_labels).res,
+ parser.res,
+ parser.macro_defs,
init_variables,
init_labels,
init_offset,
# Returns a dictionary of string to integers from the list of strings
# `args` containing `NAME=VAL` entries.
def _dict_from_arg(args: Optional[List[str]]):
- d = {} # type: Dict[str, int]
+ d = {} # type: LabelsT
if args is None:
return d
normand = f.read()
# Variables and labels
- variables = _dict_from_arg(args.var)
+ variables = typing.cast(VariablesT, _dict_from_arg(args.var))
labels = _dict_from_arg(args.label)
# Validate offset
projects / normand.git / blobdiff