Check invalid mask registers

In 32-bit, the REX_B bit in the 3-byte VEX prefix is ignored and the
the highest bit in VEX.vvvv is either 1 or ignored.  In 64-bit, we
need to check invalid mask registers.

gas/

	PR binutis/20705
	* testsuite/gas/i386/i386.exp: Run x86-64-opcode-bad.
	* testsuite/gas/i386/x86-64-opcode-bad.d: New file.
	* testsuite/gas/i386/x86-64-opcode-bad.s: Likewise.

opcodes/

	PR binutis/20705
	* i386-dis.c (get_valid_dis386): Ignore the REX_B bit and
	the highest bit in VEX.vvvv for the 3-byte VEX prefix in
	32-bit mode.  Don't check vex.register_specifier in 32-bit
	mode.
	(OP_E_register): Check invalid mask registers.
	(OP_G): Likewise.
	(OP_VEX): Likewise.

diff --git a/gas/ChangeLog b/gas/ChangeLog
index 2e3adbe6c1486fded7bc27f03bf6248b4f40c55e..50282aba7e9aff67a090919eb4ae9785947c637a 100644 (file)
--- a/gas/ChangeLog
+++ b/gas/ChangeLog
@@ -1,3 +1,10 @@
+2016-10-20  H.J. Lu  <hongjiu.lu@intel.com>
+
+       PR binutis/20705
+       * testsuite/gas/i386/i386.exp: Run x86-64-opcode-bad.
+       * testsuite/gas/i386/x86-64-opcode-bad.d: New file.
+       * testsuite/gas/i386/x86-64-opcode-bad.s: Likewise.
+
 2016-10-19  Renlin Li  <renlin.li@arm.com>
 
        * config/tc-arm.c (encode_arm_shift): Generate unpredictable warning

diff --git a/gas/testsuite/gas/i386/i386.exp b/gas/testsuite/gas/i386/i386.exp
index c2cdf606bce8ec5a13a8c1ce0219f88c67c31893..e2b874da7d498b8d4f17f8364ab715af9f030c77 100644 (file)
--- a/gas/testsuite/gas/i386/i386.exp
+++ b/gas/testsuite/gas/i386/i386.exp
@@ -776,6 +776,7 @@ if [expr ([istarget "i*86-*-*"] || [istarget "x86_64-*-*"]) && [gas_64_check]] t
     run_dump_test "x86-64-pr20141"
     run_list_test "x86-64-avx512vl-1" "-al"
     run_list_test "x86-64-avx512vl-2" "-al"
+    run_dump_test "x86-64-opcode-bad"
 
     if { ![istarget "*-*-aix*"]
       && ![istarget "*-*-beos*"]

diff --git a/gas/testsuite/gas/i386/x86-64-opcode-bad.d b/gas/testsuite/gas/i386/x86-64-opcode-bad.d
new file mode 100644 (file)
index 0000000..24fb5eb
--- /dev/null
+++ b/gas/testsuite/gas/i386/x86-64-opcode-bad.d
@@ -0,0 +1,12 @@
+#as: --64
+#objdump: -dw
+#name: 64bit bad opcodes
+
+.*: +file format .*
+
+Disassembly of section .text:
+
+0+ <.text>:
+ +[a-f0-9]+:   c5 ac 46 f5             kxnorw %k5,\(bad\),%k6
+ +[a-f0-9]+:   c5 2c 46 f5             kxnorw %k5,\(bad\),\(bad\)
+#pass

diff --git a/gas/testsuite/gas/i386/x86-64-opcode-bad.s b/gas/testsuite/gas/i386/x86-64-opcode-bad.s
new file mode 100644 (file)
index 0000000..5acbcbb
--- /dev/null
+++ b/gas/testsuite/gas/i386/x86-64-opcode-bad.s
@@ -0,0 +1,10 @@
+       .text
+# All the followings are bad opcodes for x86-64.
+       .byte 0xc5
+       .byte 0xac
+       .byte 0x46
+       .byte 0xf5
+       .byte 0xc5
+       .byte 0x2c
+       .byte 0x46
+       .byte 0xf5

diff --git a/opcodes/ChangeLog b/opcodes/ChangeLog
index 36ed58061866b56d33dbb1aa9706625934c65344..75c99acab81adaea13b9bfc01281a5eea49dc38a 100644 (file)
--- a/opcodes/ChangeLog
+++ b/opcodes/ChangeLog
@@ -1,3 +1,12 @@
+2016-10-20  H.J. Lu  <hongjiu.lu@intel.com>
+
+       PR binutis/20705
+       * i386-dis.c (get_valid_dis386): Ignore the REX_B bit and
+       the highest bit in VEX.vvvv for the 3-byte VEX prefix in
+       32-bit mode.  Don't check vex.register_specifier in 32-bit
+       mode.
+       (OP_VEX): Check for invalid mask registers.
+
 2016-10-18  H.J. Lu  <hongjiu.lu@intel.com>
 
        PR binutis/20699

diff --git a/opcodes/i386-dis.c b/opcodes/i386-dis.c
index 4624a6a6600fc095c1d226be7ccdcfe0acd056e8..1518a1df5eb80df9f4dd3c0750d8ef60d42f62de 100644 (file)
--- a/opcodes/i386-dis.c
+++ b/opcodes/i386-dis.c
@@ -13022,17 +13022,20 @@ get_valid_dis386 (const struct dis386 *dp, disassemble_info *info)
        }
       codep++;
       vex.w = *codep & 0x80;
-      if (vex.w && address_mode == mode_64bit)
-       rex |= REX_W;
-
-      vex.register_specifier = (~(*codep >> 3)) & 0xf;
-      if (address_mode != mode_64bit
-         && vex.register_specifier > 0x7)
+      if (address_mode == mode_64bit)
        {
-         dp = &bad_opcode;
-         return dp;
+         if (vex.w)
+           rex |= REX_W;
+         vex.register_specifier = (~(*codep >> 3)) & 0xf;
+       }
+      else
+       {
+         /* For the 3-byte VEX prefix in 32-bit mode, the REX_B bit
+            is ignored, other REX bits are 0 and the highest bit in
+            VEX.vvvv is also ignored.  */
+         rex = 0;
+         vex.register_specifier = (~(*codep >> 3)) & 0x7;
        }
-
       vex.length = (*codep & 0x4) ? 256 : 128;
       switch ((*codep & 0x3))
        {
@@ -13072,16 +13075,10 @@ get_valid_dis386 (const struct dis386 *dp, disassemble_info *info)
       rex_ignored = rex;
       rex = (*codep & 0x80) ? 0 : REX_R;
 
+      /* For the 2-byte VEX prefix in 32-bit mode, the highest bit in
+        VEX.vvvv is 1.  */
       vex.register_specifier = (~(*codep >> 3)) & 0xf;
-      if (address_mode != mode_64bit
-         && vex.register_specifier > 0x7)
-       {
-         dp = &bad_opcode;
-         return dp;
-       }
-
       vex.w = 0;
-
       vex.length = (*codep & 0x4) ? 256 : 128;
       switch ((*codep & 0x3))
        {
@@ -15266,6 +15263,11 @@ OP_E_register (int bytemode, int sizeflag)
       break;
     case mask_bd_mode:
     case mask_mode:
+      if (reg > 0x7)
+       {
+         oappend ("(bad)");
+         return;
+       }
       names = names_mask;
       break;
     case 0:
@@ -15795,6 +15797,11 @@ OP_G (int bytemode, int sizeflag)
       break;
     case mask_bd_mode:
     case mask_mode:
+      if ((modrm.reg + add) > 0x7)
+       {
+         oappend ("(bad)");
+         return;
+       }
       oappend (names_mask[modrm.reg + add]);
       break;
     default:
@@ -17225,6 +17232,11 @@ OP_VEX (int bytemode, int sizeflag ATTRIBUTE_UNUSED)
          break;
        case mask_bd_mode:
        case mask_mode:
+         if (reg > 0x7)
+           {
+             oappend ("(bad)");
+             return;
+           }
          names = names_mask;
          break;
        default:
@@ -17245,6 +17257,11 @@ OP_VEX (int bytemode, int sizeflag ATTRIBUTE_UNUSED)
          break;
        case mask_bd_mode:
        case mask_mode:
+         if (reg > 0x7)
+           {
+             oappend ("(bad)");
+             return;
+           }
          names = names_mask;
          break;
        default: