From cd33dfe61ea9d6c08460d800bb995cdb774c3c48 Mon Sep 17 00:00:00 2001 From: Philippe Proulx Date: Wed, 11 Oct 2023 10:48:28 -0400 Subject: [PATCH] Add transformation block support MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit A transformation block represents the transformed bytes of one or more items, for example: !transform base64 "hello how are you" {ICITTE:8} * 20 !end 61 47 56 73 62 47 38 67 61 47 39 33 49 47 46 79 ┆ aGVsbG8gaG93IGFy 5a 53 42 35 62 33 55 52 45 68 4d 55 46 52 59 58 ┆ ZSB5b3UREhMUFRYX 47 42 6b 61 47 78 77 64 48 68 38 67 49 53 49 6a ┆ GBkaGxwdHh8gISIj 4a 41 3d 3d ┆ JA== It's effectively calling a function with the bytes of the block items. As of this version, Normand only features a specific set of transformation functions. In the future the user should be able to define its own transformations (named functions passed to normand.parse() I guess). The gzip and bzip2 formats include a timestamp, so the `tests/*.nt` approach doesn't work here: adding `tests/test_trans_gz_bz2.py` which uses normand.parse() to compress some data D, and then decompresses the result using the correct module and compares to D. Change-Id: I31eb296fa9ba726ee9695cc1cf049abd7cabaf52 Signed-off-by: Philippe Proulx Reviewed-on: https://review.lttng.org/c/normand/+/11030 Tested-by: jenkins --- README.adoc | 215 +++++++++++++++++++++++++++- normand/normand.py | 230 +++++++++++++++++++++++++----- pyproject.toml | 2 +- tests/fail-trans-inval-type.nt | 3 + tests/fail-trans-missing-type.nt | 3 + tests/fail-trans-unknown-label.nt | 8 ++ tests/pass-comment-all.nt | 4 + tests/pass-readme-learn-trans.nt | 38 +++++ tests/pass-trans-a85.nt | 17 +++ tests/pass-trans-a85p.nt | 17 +++ tests/pass-trans-b16.nt | 21 +++ tests/pass-trans-b32.nt | 19 +++ tests/pass-trans-b64.nt | 17 +++ tests/pass-trans-b64u.nt | 17 +++ tests/pass-trans-b85.nt | 17 +++ tests/pass-trans-b85p.nt | 17 +++ tests/pass-trans-qp.nt | 19 +++ tests/pass-trans-qpt.nt | 21 +++ tests/test_api.py | 6 +- tests/test_trans_gz_bz2.py | 43 ++++++ 20 files changed, 685 insertions(+), 49 deletions(-) create mode 100644 tests/fail-trans-inval-type.nt create mode 100644 tests/fail-trans-missing-type.nt create mode 100644 tests/fail-trans-unknown-label.nt create mode 100644 tests/pass-readme-learn-trans.nt create mode 100644 tests/pass-trans-a85.nt create mode 100644 tests/pass-trans-a85p.nt create mode 100644 tests/pass-trans-b16.nt create mode 100644 tests/pass-trans-b32.nt create mode 100644 tests/pass-trans-b64.nt create mode 100644 tests/pass-trans-b64u.nt create mode 100644 tests/pass-trans-b85.nt create mode 100644 tests/pass-trans-b85p.nt create mode 100644 tests/pass-trans-qp.nt create mode 100644 tests/pass-trans-qpt.nt create mode 100644 tests/test_trans_gz_bz2.py diff --git a/README.adoc b/README.adoc index 52e81c1..7469654 100644 --- a/README.adoc +++ b/README.adoc @@ -29,7 +29,7 @@ _**Normand**_ is a text-to-binary processor with its own language. This package offers both a portable {py3} module and a command-line tool. -WARNING: This version of Normand is 0.20, meaning both the Normand +WARNING: This version of Normand is 0.21, meaning both the Normand language and the module/CLI interface aren't stable. ifdef::env-github[] @@ -279,6 +279,29 @@ ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ┆ •••••••• ff ff ff ff ff ff ff ff c8 ┆ ••••••••• ---- +Transformation:: ++ +Input: ++ +---- +"end of file @ " {end:8} + +!transform gzip + "this part will be gzipped" +!end + + +---- ++ +Output: ++ +---- +65 6e 64 20 6f 66 20 66 69 6c 65 20 40 20 3c 1f ┆ end of file @ <• +8b 08 00 7b 7b 26 65 02 ff 2b c9 c8 2c 56 28 48 ┆ •••{{&e••+••,V(H +2c 2a 51 28 cf cc c9 51 48 4a 55 48 af ca 2c 28 ┆ ,*Q(•••QHJUH••,( +48 4d 01 00 d4 cc 5b 8a 19 00 00 00 ┆ HM••••[••••• +---- + Multilevel grouping:: + Input: @@ -549,6 +572,8 @@ This is similar to an assembly label. * A <>. +* A <>. + * A <>. * A <>. @@ -888,8 +913,9 @@ A fixed-length number is: For a fixed-length number at some source location{nbsp}__**L**__, this expression may contain the name of any accessible <> (not within a nested group), including the name of a label defined -after{nbsp}__**L**__, as well as the name of any -<> known at{nbsp}__**L**__. +after{nbsp}__**L**__ (except within an <>), +as well as the name of any <> known +at{nbsp}__**L**__. + The value of the special name `ICITTE` (`int` type) in this expression is the <> (before encoding the number). @@ -1546,7 +1572,7 @@ A group is: . The `(`, `!group`, or `!g` opening. -. Zero or more items. +. Zero or more items except, recursively, a macro definition block. . Depending on the group opening: + @@ -1652,12 +1678,14 @@ items). For the name `__NAME__`, this is equivalent to the `pass:[{]__NAME__pass:[}]` form above. -. Zero or more items to be handled when the condition is true. +. Zero or more items to be handled when the condition is true + except, recursively, a macro definition block. . **Optional**: .. The `!else` opening. -.. Zero or more items to be handled when the condition is false. +.. Zero or more items to be handled when the condition is false + except, recursively, a macro definition block . The `!end` closing. @@ -1753,7 +1781,7 @@ repeat). For the name `__NAME__`, this is equivalent to the `pass:[{]__NAME__pass:[}]` form above. -. Zero or more items. +. Zero or more items except, recursively, a macro definition block. . The `!end` closing. @@ -1835,6 +1863,178 @@ ff ee ff ee ff ee ff ee ff ee ff ee ff 11 22 33 ┆ •••••••• ---- ==== +=== Transformation block + +A _transformation block_ represents the bytes of one or more items +transformed into other bytes by a function. + +As of this version, Normand only offers a predetermined set of +transformation functions. + +An encoded block is: + +. The `!transform` or `!t` opening. + +. A transformation function name amongst: ++ +-- +[horizontal] +`base64`:: +`b64`:: + Standard https://datatracker.ietf.org/doc/html/rfc4648.html#section-4[Base64]. + +`base64u`:: +`b64u`:: + URL-safe Base64, using `-` instead of `pass:[+]` and `_` instead of + `/`. + +`base32`:: +`b32`:: + Standard https://datatracker.ietf.org/doc/html/rfc4648.html#section-6[Base32]. + +`base16`:: +`b16`:: + Standard https://datatracker.ietf.org/doc/html/rfc4648.html#section-8[Base16]. + +`ascii85`:: +`a85`:: + https://en.wikipedia.org/wiki/Ascii85[Ascii85] without padding. + +`ascii85p`:: +`a85p`:: + Ascii85 with padding. + +`base85`:: +`b85`:: + https://en.wikipedia.org/wiki/Ascii85[Base85] (like Git-style binary + diffs) without padding. + +`base85p`:: +`b85p`:: + Base85 with padding. + +`quopri`:: +`qp`:: + MIME + https://datatracker.ietf.org/doc/html/rfc2045#section-6.7[quoted-printable] + without quoted whitespaces. + +`quoprit`:: +`qpt`:: + MIME quoted-printable with quoted whitespaces. + +`gzip`:: +`gz`:: + https://en.wikipedia.org/wiki/Gzip[gzip]. + +`bzip2`:: +`bz2`:: + https://en.wikipedia.org/wiki/Bzip2[bzip2]. +-- + +. Zero or more items except, recursively, a macro definition block. ++ +Any {py3} expression within any of those items may not refer to a future +<>. ++ +The value of the special name `ICITTE` in any {py3} expression within +any of those items is the <> _before_ Normand +applies the transformation function. Therefore, labels defined within +those items also have the current offset value _before_ Normand applies +the transformation function. + +. The `!end` closing. + +The <> after having handled the last item of +a transformation block is the value of the current offset before +handling the first item plus the size of the generated (transformed) +bytes. In other words, <> within the items of the block have no impact outside said +block. + +==== +Input: + +---- +aa bb cc dd + +"size of compressed section: " {end - start : 8} + + + +!transform bzip2 + "this will be compressed!" + 89*100 00*5000 +!end + + + +"yes!" +---- + +Output: + +---- +aa bb cc dd 73 69 7a 65 20 6f 66 20 63 6f 6d 70 ┆ ••••size of comp +72 65 73 73 65 64 20 73 65 63 74 69 6f 6e 3a 20 ┆ ressed section: +52 42 5a 68 39 31 41 59 26 53 59 68 e1 8c fc 00 ┆ RBZh91AY&SYh•••• +00 33 d1 e0 c0 00 60 00 5e 66 dc 80 00 20 00 80 ┆ •3••••`•^f••• •• +00 08 20 00 31 40 d3 43 23 26 20 ca 87 a9 a1 e8 ┆ •• •1@•C#& ••••• +18 29 44 80 9c 80 49 bf cc b3 e8 45 ed e2 76 ad ┆ •)D•••I••••E••v• +0f 12 8b 8a d6 cd 40 04 7e 2e e4 8a 70 a1 20 d1 ┆ ••••••@•~.••p• • +c3 19 f8 79 65 73 21 ┆ •••yes! +---- +==== + +==== +Input: + +---- +88*16 + +!t a85 + "I am determined to be cheerful and happy in whatever situation " + "I may find myself. For I have learned that the greater part of " + "our misery or unhappiness is determined not by our circumstance " + "but by our disposition." +!end + +@128~99h + +!t qp {ICITTE - beg : 8} * 50 !end +---- + +Output: + +---- +88 88 88 88 88 88 88 88 88 88 88 88 88 88 88 88 ┆ •••••••••••••••• +38 4b 5f 47 59 2b 43 6f 26 2a 41 54 44 58 25 44 ┆ 8K_GY+Co&*ATDX%D +49 6d 3f 24 46 44 69 3a 32 41 4b 59 4a 72 41 53 ┆ Im?$FDi:2AKYJrAS +23 6d 6f 46 5f 69 31 2f 44 49 61 6c 27 40 3b 70 ┆ #moF_i1/DIal'@;p +31 32 2b 44 47 5e 39 47 41 28 45 2c 41 54 68 58 ┆ 12+DG^9GA(E,AThX +2a 2b 45 4d 37 3d 46 5e 5d 42 2b 44 66 2d 5b 68 ┆ *+EM7=F^]B+Df-[h +2b 44 6b 50 34 2b 44 2c 3e 2a 41 30 3e 60 37 46 ┆ +DkP4+D,>*A0>`7F +28 4b 30 22 2f 67 2a 57 25 45 5a 64 70 72 42 4f ┆ (K0"/g*W%EZdprBO +51 27 71 2b 44 62 55 74 45 63 2c 48 21 2b 45 56 ┆ Q'q+DbUtEc,H!+EV +3a 2a 46 3c 47 5b 3d 41 4b 59 57 2b 41 52 54 5b ┆ :*F +63 2e 46 3c 47 25 3c 2b 45 29 43 43 2b 43 66 2c ┆ c.F>. ** A <>. ** A <>. +** A <>. ** A <>. . The ``pass:[*]`` character. diff --git a/normand/normand.py b/normand/normand.py index 95c76eb..8e1acf2 100644 --- a/normand/normand.py +++ b/normand/normand.py @@ -30,7 +30,7 @@ # Upstream repository: . __author__ = "Philippe Proulx" -__version__ = "0.20.0" +__version__ = "0.21.0" __all__ = [ "__author__", "__version__", @@ -47,12 +47,17 @@ __all__ = [ import re import abc import ast +import bz2 import sys import copy import enum +import gzip import math +import base64 +import quopri import struct import typing +import functools from typing import Any, Set, Dict, List, Union, Pattern, Callable, NoReturn, Optional @@ -356,7 +361,6 @@ class _Str(_Item, _RepableItem, _ExprMixin): def __repr__(self): return "_Str({}, {}, {}, {})".format( - self.__class__.__name__, repr(self._expr_str), repr(self._expr), repr(self._codec), @@ -380,22 +384,20 @@ class _Group(_Item, _RepableItem): # Repetition item. -class _Rep(_Item, _ExprMixin): +class _Rep(_Group, _ExprMixin): def __init__( - self, item: _Item, expr_str: str, expr: ast.Expression, text_loc: TextLocation + self, + items: List[_Item], + expr_str: str, + expr: ast.Expression, + text_loc: TextLocation, ): - super().__init__(text_loc) + super().__init__(items, text_loc) _ExprMixin.__init__(self, expr_str, expr) - self._item = item - - # Item to repeat. - @property - def item(self): - return self._item def __repr__(self): return "_Rep({}, {}, {}, {})".format( - repr(self._item), + repr(self._items), repr(self._expr_str), repr(self._expr), repr(self._text_loc), @@ -406,8 +408,8 @@ class _Rep(_Item, _ExprMixin): class _Cond(_Item, _ExprMixin): def __init__( self, - true_item: _Item, - false_item: _Item, + true_item: _Group, + false_item: _Group, expr_str: str, expr: ast.Expression, text_loc: TextLocation, @@ -437,15 +439,48 @@ class _Cond(_Item, _ExprMixin): ) +# Transformation. +class _Trans(_Group, _RepableItem): + def __init__( + self, + items: List[_Item], + name: str, + func: Callable[[Union[bytes, bytearray]], bytes], + text_loc: TextLocation, + ): + super().__init__(items, text_loc) + self._name = name + self._func = func + + @property + def name(self): + return self._name + + # Transforms the data `data`. + def trans(self, data: Union[bytes, bytearray]): + return self._func(data) + + def __repr__(self): + return "_Trans({}, {}, {}, {})".format( + repr(self._items), + repr(self._name), + repr(self._func), + repr(self._text_loc), + ) + + # Macro definition item. -class _MacroDef(_Item): +class _MacroDef(_Group): def __init__( - self, name: str, param_names: List[str], group: _Group, text_loc: TextLocation + self, + name: str, + param_names: List[str], + items: List[_Item], + text_loc: TextLocation, ): - super().__init__(text_loc) + super().__init__(items, text_loc) self._name = name self._param_names = param_names - self._group = group # Name. @property @@ -457,16 +492,11 @@ class _MacroDef(_Item): def param_names(self): return self._param_names - # Contained items. - @property - def group(self): - return self._group - def __repr__(self): return "_MacroDef({}, {}, {}, {})".format( repr(self._name), repr(self._param_names), - repr(self._group), + repr(self._items), repr(self._text_loc), ) @@ -1560,7 +1590,6 @@ class _Parser: # Parse items self._skip_ws_and_comments_and_syms() - items_text_loc = self._text_loc items = self._parse_items() # Expect end of block @@ -1570,7 +1599,7 @@ class _Parser: ) # Return item - return _Rep(_Group(items, items_text_loc), expr_str, expr, begin_text_loc) + return _Rep(items, expr_str, expr, begin_text_loc) # Pattern for _try_parse_cond_block() _cond_block_prefix_pat = re.compile(r"!if\b") @@ -1621,6 +1650,87 @@ class _Parser: begin_text_loc, ) + # Pattern for _try_parse_trans_block() + _trans_block_prefix_pat = re.compile(r"!t(?:ransform)?\b") + _trans_block_type_pat = re.compile( + r"(?:(?:base|b)64(?:u)?|(?:base|b)(?:16|32)|(?:ascii|a|base|b)85(?:p)?|(?:quopri|qp)(?:t)?|gzip|gz|bzip2|bz2)\b" + ) + + # Tries to parse a transformation block, returning a transformation + # block item on success. + def _try_parse_trans_block(self): + begin_text_loc = self._text_loc + + # Match prefix + if self._try_parse_pat(self._trans_block_prefix_pat) is None: + # No match + return + + # Expect type + self._skip_ws_and_comments() + m = self._expect_pat( + self._trans_block_type_pat, "Expecting a known transformation type" + ) + + # Parse items + self._skip_ws_and_comments_and_syms() + items = self._parse_items() + + # Expect end of block + self._expect_pat( + self._block_end_pat, + "Expecting an item or `!end` (end of transformation block)", + ) + + # Choose encoding function + enc = m.group(0) + + if enc in ("base64", "b64"): + func = base64.standard_b64encode + name = "standard Base64" + elif enc in ("base64u", "b64u"): + func = base64.urlsafe_b64encode + name = "URL-safe Base64" + elif enc in ("base32", "b32"): + func = base64.b32encode + name = "Base32" + elif enc in ("base16", "b16"): + func = base64.b16encode + name = "Base16" + elif enc in ("ascii85", "a85"): + func = base64.a85encode + name = "Ascii85" + elif enc in ("ascii85p", "a85p"): + func = functools.partial(base64.a85encode, pad=True) + name = "padded Ascii85" + elif enc in ("base85", "b85"): + func = base64.b85encode + name = "Base85" + elif enc in ("base85p", "b85p"): + func = functools.partial(base64.b85encode, pad=True) + name = "padded Base85" + elif enc in ("quopri", "qp"): + func = quopri.encodestring + name = "MIME quoted-printable" + elif enc in ("quoprit", "qpt"): + func = functools.partial(quopri.encodestring, quotetabs=True) + name = "MIME quoted-printable (with quoted tabs)" + elif enc in ("gzip", "gz"): + func = gzip.compress + name = "gzip" + else: + assert enc in ("bzip2", "bz2") + func = bz2.compress + name = "bzip2" + + # Return item + return _Trans( + items, + name, + func, + begin_text_loc, + ) + # Common left parenthesis pattern _left_paren_pat = re.compile(r"\(") @@ -1687,7 +1797,6 @@ class _Parser: # Expect items self._skip_ws_and_comments_and_syms() - items_text_loc = self._text_loc old_var_names = self._var_names.copy() old_label_names = self._label_names.copy() self._var_names = set() # type: Set[str] @@ -1702,9 +1811,7 @@ class _Parser: ) # Register macro - self._macro_defs[name] = _MacroDef( - name, param_names, _Group(items, items_text_loc), begin_text_loc - ) + self._macro_defs[name] = _MacroDef(name, param_names, items, begin_text_loc) return True @@ -1844,12 +1951,18 @@ class _Parser: if item is not None: return item - # Macro expansion? + # Macro expansion item? item = self._try_parse_macro_exp() if item is not None: return item + # Transformation block item? + item = self._try_parse_trans_block() + + if item is not None: + return item + # Pattern for _try_parse_rep_post() _rep_post_prefix_pat = re.compile(r"\*") @@ -1885,7 +1998,7 @@ class _Parser: rep_ret = self._try_parse_rep_post() if rep_ret is not None: - item = _Rep(item, *rep_ret, text_loc=rep_text_loc) + item = _Rep([item], *rep_ret, text_loc=rep_text_loc) items.append(item) return True @@ -2161,6 +2274,7 @@ class _Gen: self._macro_defs = macro_defs self._fl_num_item_insts = [] # type: List[_FlNumItemInst] self._parse_error_msgs = [] # type: List[ParseErrorMessage] + self._in_trans = False self._gen(group, _GenState(variables, labels, offset, bo)) # Generated bytes. @@ -2313,6 +2427,14 @@ class _Gen: try: data = self._gen_fl_num_item_inst_data(item, state) except Exception: + if self._in_trans: + _raise_error_for_item( + "Invalid expression `{}`: failed to evaluate within a transformation block".format( + item.expr_str + ), + item, + ) + self._fl_num_item_insts.append( _FlNumItemInst( item, @@ -2425,20 +2547,51 @@ class _Gen: item, ) - # Generate item data `mul` times + # Generate group data `mul` times for _ in range(mul): - self._handle_item(item.item, state) + self._handle_group_item(item, state) # Handles the conditional item `item`. def _handle_cond_item(self, item: _Cond, state: _GenState): # Compute the conditional value val = _Gen._eval_item_expr(item, state) - # Generate item data if needed + # Generate selected group data if val: - self._handle_item(item.true_item, state) + self._handle_group_item(item.true_item, state) else: - self._handle_item(item.false_item, state) + self._handle_group_item(item.false_item, state) + + # Handles the transformation item `item`. + def _handle_trans_item(self, item: _Trans, state: _GenState): + init_in_trans = self._in_trans + self._in_trans = True + init_data_len = len(self._data) + init_offset = state.offset + + # Generate group data + self._handle_group_item(item, state) + + # Remove and keep group data + to_trans = self._data[init_data_len:] + del self._data[init_data_len:] + + # Encode group data and append to current data + try: + transformed = item.trans(to_trans) + except Exception as exc: + _raise_error_for_item( + "Cannot apply the {} transformation to this data: {}".format( + item.name, exc + ), + item, + ) + + self._data += transformed + + # Update offset and restore + state.offset = init_offset + len(transformed) + self._in_trans = init_in_trans # Evaluates the parameters of the macro expansion item `item` # considering the initial state `init_state` and returns a new state @@ -2478,7 +2631,7 @@ class _Gen: ) ) self._parse_error_msgs.append(parse_error_msg) - self._handle_item(self._macro_defs[item.name].group, exp_state) + self._handle_group_item(self._macro_defs[item.name], exp_state) self._parse_error_msgs.pop() except ParseError as exc: _augment_error(exc, parse_error_msg_text, item.text_loc) @@ -2636,6 +2789,7 @@ class _Gen: _SetOffset: self._handle_set_offset_item, _SLeb128Int: self._handle_leb128_int_item, _Str: self._handle_str_item, + _Trans: self._handle_trans_item, _ULeb128Int: self._handle_leb128_int_item, _VarAssign: self._handle_var_assign_item, } # type: Dict[type, Callable[[Any, _GenState], None]] diff --git a/pyproject.toml b/pyproject.toml index 97c2a1b..1b76630 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ [tool.poetry] name = 'normand' -version = '0.20.0' +version = '0.21.0' description = 'Text-to-binary processor with its own language' license = 'MIT' authors = ['Philippe Proulx '] diff --git a/tests/fail-trans-inval-type.nt b/tests/fail-trans-inval-type.nt new file mode 100644 index 0000000..d0a73ba --- /dev/null +++ b/tests/fail-trans-inval-type.nt @@ -0,0 +1,3 @@ +!transform rar "salut lol" !end +--- +1:12 - Expecting a known transformation type diff --git a/tests/fail-trans-missing-type.nt b/tests/fail-trans-missing-type.nt new file mode 100644 index 0000000..ddb9001 --- /dev/null +++ b/tests/fail-trans-missing-type.nt @@ -0,0 +1,3 @@ +!transform "salut lol" !end +--- +1:12 - Expecting a known transformation type diff --git a/tests/fail-trans-unknown-label.nt b/tests/fail-trans-unknown-label.nt new file mode 100644 index 0000000..d28ce29 --- /dev/null +++ b/tests/fail-trans-unknown-label.nt @@ -0,0 +1,8 @@ +!transform base64 + "meow" + {far:8} + "mix" + +!end +--- +3:4 - Invalid expression `far`: failed to evaluate within a transformation block diff --git a/tests/pass-comment-all.nt b/tests/pass-comment-all.nt index 24ca19c..8b37fdc 100644 --- a/tests/pass-comment-all.nt +++ b/tests/pass-comment-all.nt @@ -63,6 +63,9 @@ a#bonjour tout le monde#a#bonjour tout le monde#bb#bonjour tout le monde# # macro expansion #bonjour tout le monde#m#bonjour tout le monde#:#bonjour tout le monde#gang#bonjour tout le monde#(#bonjour tout le monde#0x44#bonjour tout le monde#,#bonjour tout le monde#0x88#bonjour tout le monde#)#bonjour tout le monde# +# transformation block +#bonjour tout le monde#!transform#bonjour tout le monde#b16#bonjour tout le monde#"salut"#bonjour tout le monde#!end#bonjour tout le monde# + # post-item repetition "salut"#bonjour tout le monde#*#bonjour tout le monde#4 --- @@ -88,4 +91,5 @@ cc 55 55 55 77 77 77 aa 44 bb 88 +37 33 36 31 36 43 37 35 37 34 73 61 6c 75 74 73 61 6c 75 74 73 61 6c 75 74 73 61 6c 75 74 diff --git a/tests/pass-readme-learn-trans.nt b/tests/pass-readme-learn-trans.nt new file mode 100644 index 0000000..88c0c90 --- /dev/null +++ b/tests/pass-readme-learn-trans.nt @@ -0,0 +1,38 @@ +88*16 + +!t a85 + "I am determined to be cheerful and happy in whatever situation " + "I may find myself. For I have learned that the greater part of " + "our misery or unhappiness is determined not by our circumstance " + "but by our disposition." +!end + +@128~99h + +!t qp {ICITTE - beg : 8} * 50 !end +--- +88 88 88 88 88 88 88 88 88 88 88 88 88 88 88 88 +38 4b 5f 47 59 2b 43 6f 26 2a 41 54 44 58 25 44 +49 6d 3f 24 46 44 69 3a 32 41 4b 59 4a 72 41 53 +23 6d 6f 46 5f 69 31 2f 44 49 61 6c 27 40 3b 70 +31 32 2b 44 47 5e 39 47 41 28 45 2c 41 54 68 58 +2a 2b 45 4d 37 3d 46 5e 5d 42 2b 44 66 2d 5b 68 +2b 44 6b 50 34 2b 44 2c 3e 2a 41 30 3e 60 37 46 +28 4b 30 22 2f 67 2a 57 25 45 5a 64 70 72 42 4f +51 27 71 2b 44 62 55 74 45 63 2c 48 21 2b 45 56 +3a 2a 46 3c 47 5b 3d 41 4b 59 57 2b 41 52 54 5b +6c 45 5a 66 3d 30 45 63 60 46 42 41 66 75 23 37 +45 5a 66 34 35 46 28 4b 42 3b 2b 45 29 39 43 46 +60 28 6c 24 45 2c 5d 4e 2f 41 54 4d 6f 38 42 6c +62 44 2d 41 54 56 4c 28 44 2f 21 6d 21 41 30 3e +63 2e 46 3c 47 25 3c 2b 45 29 43 43 2b 43 66 2c +2b 40 73 29 58 30 46 43 42 26 73 41 4b 59 48 29 +46 3c 47 25 3c 2b 45 29 43 43 2b 43 6f 32 2d 45 +2c 54 66 33 46 44 35 5a 32 2f 63 99 99 99 99 99 +3d 30 30 3d 30 31 3d 30 32 3d 30 33 3d 30 34 3d +30 35 3d 30 36 3d 30 37 3d 30 38 3d 30 39 0a 3d +30 42 3d 30 43 0d 3d 30 45 3d 30 46 3d 31 30 3d +31 31 3d 31 32 3d 31 33 3d 31 34 3d 31 35 3d 31 +36 3d 31 37 3d 31 38 3d 31 39 3d 31 41 3d 31 42 +3d 31 43 3d 31 44 3d 31 45 3d 31 46 20 21 22 23 +24 25 26 27 28 29 2a 2b 2c 2d 3d 0a 2e 2f 30 31 diff --git a/tests/pass-trans-a85.nt b/tests/pass-trans-a85.nt new file mode 100644 index 0000000..3bac685 --- /dev/null +++ b/tests/pass-trans-a85.nt @@ -0,0 +1,17 @@ +!transform ascii85 + "bonjour tout le monde" + {ICITTE - beg1 : 8} * 15 +!end + +!t a85 + "bonjour tout le monde" + {ICITTE - beg2 : 8} * 15 +!end +--- +40 57 2d 2e 21 44 66 70 28 43 46 44 6c 3b 44 2b +44 62 55 33 44 2f 58 3c 26 41 48 32 60 34 21 73 +41 63 33 23 37 28 56 43 24 4f 64 49 53 + +40 57 2d 2e 21 44 66 70 28 43 46 44 6c 3b 44 2b +44 62 55 33 44 2f 58 3c 26 41 48 32 60 34 21 73 +41 63 33 23 37 28 56 43 24 4f 64 49 53 diff --git a/tests/pass-trans-a85p.nt b/tests/pass-trans-a85p.nt new file mode 100644 index 0000000..3530a1f --- /dev/null +++ b/tests/pass-trans-a85p.nt @@ -0,0 +1,17 @@ +!transform ascii85p + "bonjour tout le monde" + {ICITTE - beg1 : 8} * 15 +!end + +!t a85p + "bonjour tout le monde" + {ICITTE - beg2 : 8} * 15 +!end +--- +40 57 2d 2e 21 44 66 70 28 43 46 44 6c 3b 44 2b +44 62 55 33 44 2f 58 3c 26 41 48 32 60 34 21 73 +41 63 33 23 37 28 56 43 24 4f 64 49 53 + +40 57 2d 2e 21 44 66 70 28 43 46 44 6c 3b 44 2b +44 62 55 33 44 2f 58 3c 26 41 48 32 60 34 21 73 +41 63 33 23 37 28 56 43 24 4f 64 49 53 diff --git a/tests/pass-trans-b16.nt b/tests/pass-trans-b16.nt new file mode 100644 index 0000000..807e8f6 --- /dev/null +++ b/tests/pass-trans-b16.nt @@ -0,0 +1,21 @@ +!transform base16 + "bonjour tout le monde" + {ICITTE - beg1 : 8} * 15 +!end + +!t b16 + "bonjour tout le monde" + {ICITTE - beg2 : 8} * 15 +!end +--- +36 32 36 46 36 45 36 41 36 46 37 35 37 32 32 30 +37 34 36 46 37 35 37 34 32 30 36 43 36 35 32 30 +36 44 36 46 36 45 36 34 36 35 30 30 30 31 30 32 +30 33 30 34 30 35 30 36 30 37 30 38 30 39 30 41 +30 42 30 43 30 44 30 45 + +36 32 36 46 36 45 36 41 36 46 37 35 37 32 32 30 +37 34 36 46 37 35 37 34 32 30 36 43 36 35 32 30 +36 44 36 46 36 45 36 34 36 35 30 30 30 31 30 32 +30 33 30 34 30 35 30 36 30 37 30 38 30 39 30 41 +30 42 30 43 30 44 30 45 diff --git a/tests/pass-trans-b32.nt b/tests/pass-trans-b32.nt new file mode 100644 index 0000000..b7559bf --- /dev/null +++ b/tests/pass-trans-b32.nt @@ -0,0 +1,19 @@ +!transform base32 + "bonjour tout le monde" + {ICITTE - beg1 : 8} * 15 +!end + +!t b32 + "bonjour tout le monde" + {ICITTE - beg2 : 8} * 15 +!end +--- +4d 4a 58 57 34 32 54 50 4f 56 5a 43 41 35 44 50 +4f 56 32 43 41 33 44 46 45 42 57 57 36 33 54 45 +4d 55 41 41 43 41 51 44 41 51 43 51 4d 42 59 49 +42 45 46 41 57 44 41 4e 42 59 3d 3d 3d 3d 3d 3d + +4d 4a 58 57 34 32 54 50 4f 56 5a 43 41 35 44 50 +4f 56 32 43 41 33 44 46 45 42 57 57 36 33 54 45 +4d 55 41 41 43 41 51 44 41 51 43 51 4d 42 59 49 +42 45 46 41 57 44 41 4e 42 59 3d 3d 3d 3d 3d 3d diff --git a/tests/pass-trans-b64.nt b/tests/pass-trans-b64.nt new file mode 100644 index 0000000..641fd9b --- /dev/null +++ b/tests/pass-trans-b64.nt @@ -0,0 +1,17 @@ +!transform base64 + "bonjour tout le monde" + {ICITTE - beg1 : 8} * 15 +!end + +!t b64 + "bonjour tout le monde" + {ICITTE - beg2 : 8} * 15 +!end +--- +59 6d 39 75 61 6d 39 31 63 69 42 30 62 33 56 30 +49 47 78 6c 49 47 31 76 62 6d 52 6c 41 41 45 43 +41 77 51 46 42 67 63 49 43 51 6f 4c 44 41 30 4f + +59 6d 39 75 61 6d 39 31 63 69 42 30 62 33 56 30 +49 47 78 6c 49 47 31 76 62 6d 52 6c 41 41 45 43 +41 77 51 46 42 67 63 49 43 51 6f 4c 44 41 30 4f diff --git a/tests/pass-trans-b64u.nt b/tests/pass-trans-b64u.nt new file mode 100644 index 0000000..c1398b7 --- /dev/null +++ b/tests/pass-trans-b64u.nt @@ -0,0 +1,17 @@ +!transform base64u + "bonjour tout le monde" + {ICITTE - beg1 : 8} * 15 +!end + +!t b64u + "bonjour tout le monde" + {ICITTE - beg2 : 8} * 15 +!end +--- +59 6d 39 75 61 6d 39 31 63 69 42 30 62 33 56 30 +49 47 78 6c 49 47 31 76 62 6d 52 6c 41 41 45 43 +41 77 51 46 42 67 63 49 43 51 6f 4c 44 41 30 4f + +59 6d 39 75 61 6d 39 31 63 69 42 30 62 33 56 30 +49 47 78 6c 49 47 31 76 62 6d 52 6c 41 41 45 43 +41 77 51 46 42 67 63 49 43 51 6f 4c 44 41 30 4f diff --git a/tests/pass-trans-b85.nt b/tests/pass-trans-b85.nt new file mode 100644 index 0000000..85d00d5 --- /dev/null +++ b/tests/pass-trans-b85.nt @@ -0,0 +1,17 @@ +!transform base85 + "bonjour tout le monde" + {ICITTE - beg1 : 8} * 15 +!end + +!t b85 + "bonjour tout le monde" + {ICITTE - beg2 : 8} * 15 +!end +--- +56 73 43 44 30 5a 2a 5f 37 59 62 5a 3e 51 5a 41 +5a 25 71 49 5a 45 74 52 35 57 64 48 23 4a 30 7c +57 26 49 32 4d 37 72 59 33 6b 28 65 6f + +56 73 43 44 30 5a 2a 5f 37 59 62 5a 3e 51 5a 41 +5a 25 71 49 5a 45 74 52 35 57 64 48 23 4a 30 7c +57 26 49 32 4d 37 72 59 33 6b 28 65 6f diff --git a/tests/pass-trans-b85p.nt b/tests/pass-trans-b85p.nt new file mode 100644 index 0000000..99be8c7 --- /dev/null +++ b/tests/pass-trans-b85p.nt @@ -0,0 +1,17 @@ +!transform base85p + "bonjour tout le monde" + {ICITTE - beg1 : 8} * 15 +!end + +!t b85p + "bonjour tout le monde" + {ICITTE - beg2 : 8} * 15 +!end +--- +56 73 43 44 30 5a 2a 5f 37 59 62 5a 3e 51 5a 41 +5a 25 71 49 5a 45 74 52 35 57 64 48 23 4a 30 7c +57 26 49 32 4d 37 72 59 33 6b 28 65 6f + +56 73 43 44 30 5a 2a 5f 37 59 62 5a 3e 51 5a 41 +5a 25 71 49 5a 45 74 52 35 57 64 48 23 4a 30 7c +57 26 49 32 4d 37 72 59 33 6b 28 65 6f diff --git a/tests/pass-trans-qp.nt b/tests/pass-trans-qp.nt new file mode 100644 index 0000000..9341ff8 --- /dev/null +++ b/tests/pass-trans-qp.nt @@ -0,0 +1,19 @@ +!transform quopri + "bonjour tout le monde" + {ICITTE - beg1 : 8} * 15 +!end + +!t qp + "bonjour tout le monde" + {ICITTE - beg2 : 8} * 15 +!end +--- +62 6f 6e 6a 6f 75 72 20 74 6f 75 74 20 6c 65 20 +6d 6f 6e 64 65 3d 30 30 3d 30 31 3d 30 32 3d 30 +33 3d 30 34 3d 30 35 3d 30 36 3d 30 37 3d 30 38 +3d 30 39 0a 3d 30 42 3d 30 43 0d 3d 30 45 + +62 6f 6e 6a 6f 75 72 20 74 6f 75 74 20 6c 65 20 +6d 6f 6e 64 65 3d 30 30 3d 30 31 3d 30 32 3d 30 +33 3d 30 34 3d 30 35 3d 30 36 3d 30 37 3d 30 38 +3d 30 39 0a 3d 30 42 3d 30 43 0d 3d 30 45 diff --git a/tests/pass-trans-qpt.nt b/tests/pass-trans-qpt.nt new file mode 100644 index 0000000..d57af61 --- /dev/null +++ b/tests/pass-trans-qpt.nt @@ -0,0 +1,21 @@ +!transform quoprit + "bonjour tout le monde" + {ICITTE - beg1 : 8} * 15 +!end + +!t qpt + "bonjour tout le monde" + {ICITTE - beg2 : 8} * 15 +!end +--- +62 6f 6e 6a 6f 75 72 3d 32 30 74 6f 75 74 3d 32 +30 6c 65 3d 32 30 6d 6f 6e 64 65 3d 30 30 3d 30 +31 3d 30 32 3d 30 33 3d 30 34 3d 30 35 3d 30 36 +3d 30 37 3d 30 38 3d 30 39 0a 3d 30 42 3d 30 43 +0d 3d 30 45 + +62 6f 6e 6a 6f 75 72 3d 32 30 74 6f 75 74 3d 32 +30 6c 65 3d 32 30 6d 6f 6e 64 65 3d 30 30 3d 30 +31 3d 30 32 3d 30 33 3d 30 34 3d 30 35 3d 30 36 +3d 30 37 3d 30 38 3d 30 39 0a 3d 30 42 3d 30 43 +0d 3d 30 45 diff --git a/tests/test_api.py b/tests/test_api.py index 780b86f..77afebf 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -25,14 +25,14 @@ import normand def test_init_labels(): - labels = {"yo": 0x88, "meow": 123} # type: normand.SymbolsT + labels = {"yo": 0x88, "meow": 123} # type: normand.LabelsT res = normand.parse("11 22 {yo:8} 33", init_labels=labels.copy()) assert res.data == bytearray([0x11, 0x22, 0x88, 0x33]) assert res.labels == labels.copy() def test_init_vars(): - variables = {"zoom": 0x88, "bateau": -123.45} # type: normand.SymbolsT + variables = {"zoom": 0x88, "bateau": -123.45} # type: normand.VariablesT res = normand.parse("11 22 {zoom:8} 33", init_variables=variables.copy()) assert res.data == bytearray([0x11, 0x22, 0x88, 0x33]) assert res.variables == variables.copy() @@ -65,7 +65,7 @@ def test_init_bo_le(): def test_final_labels(): - labels = {"yo": 0x88, "meow": 123} # type: normand.SymbolsT + labels = {"yo": 0x88, "meow": 123} # type: normand.LabelsT res = normand.parse( "11 22 (77 88) * 2 33", init_labels=labels.copy() ) diff --git a/tests/test_trans_gz_bz2.py b/tests/test_trans_gz_bz2.py new file mode 100644 index 0000000..faaf736 --- /dev/null +++ b/tests/test_trans_gz_bz2.py @@ -0,0 +1,43 @@ +# The MIT License (MIT) +# +# Copyright (c) 2023 Philippe Proulx +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import bz2 +import gzip +import typing + +import normand + + +def _test_comp(type: str, dec_func: typing.Callable[[bytes], bytes]): + data = b"bonjour tout le monde \x00\x23\x42 \x17" + b"\x7b" * 500 + ntext = "!t {} {} !end".format(type, data.hex()) + res = normand.parse(ntext) + assert dec_func(res.data) == data + + +def test_gz(): + _test_comp("gz", gzip.decompress) + + +def test_bz2(): + _test_comp("bz2", bz2.decompress) -- 2.34.1