[x64] Separate out three-byte opcode decoding

Decoding of three-byte opcodes were within the two-byte decoding
function, separate it out, and fix an incorrect comment about us
no having any three-byte opcodes (that is no longer true).

Also un-nest a large if/else out into parent scope.

Test: out/x64.debug/cctest test-disasm-x64/DisasmX64 --random-seed=1

Bug: v8:10933
Change-Id: I494d67ac75cc4500d5f0045f1087b856e6375f82
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2477426
Commit-Queue: Zhi An Ng <zhin@chromium.org>
Reviewed-by: Bill Budge <bbudge@chromium.org>
Cr-Commit-Position: refs/heads/master@{#70584}

src/diagnostics/x64/disasm-x64.cc

@@ -444,6 +444,7 @@ class DisassemblerX64 {
   int PrintImmediateOp(byte* data);
   const char* TwoByteMnemonic(byte opcode);
   int TwoByteOpcodeInstruction(byte* data);
+  int ThreeByteOpcodeInstruction(byte* data);
   int F6F7Instruction(byte* data);
   int ShiftInstruction(byte* data);
   int JumpShort(byte* data);
@@ -1826,125 +1827,21 @@ int DisassemblerX64::RegisterFPUInstruction(int escape_opcode,
 
 // Handle all two-byte opcodes, which start with 0x0F.
 // These instructions may be affected by an 0x66, 0xF2, or 0xF3 prefix.
-// We do not use any three-byte opcodes, which start with 0x0F38 or 0x0F3A.
 int DisassemblerX64::TwoByteOpcodeInstruction(byte* data) {
   byte opcode = *(data + 1);
   byte* current = data + 2;
   // At return, "current" points to the start of the next instruction.
   const char* mnemonic = TwoByteMnemonic(opcode);
   if (operand_size_ == 0x66) {
+    // These are three-byte opcodes, see ThreeByteOpcodeInstruction.
+    DCHECK_NE(0x38, opcode);
+    DCHECK_NE(0x3A, opcode);
     // 0x66 0x0F prefix.
     int mod, regop, rm;
-    if (opcode == 0x38) {
-      byte third_byte = *current;
-      current = data + 3;
     get_modrm(*current, &mod, &regop, &rm);
-      switch (third_byte) {
-        case 0x15: {
-          AppendToBuffer("blendvpd %s,", NameOfXMMRegister(regop));
-          current += PrintRightXMMOperand(current);
-          AppendToBuffer(",<xmm0>");
-          break;
-        }
-#define SSE34_DIS_CASE(instruction, notUsed1, notUsed2, notUsed3, opcode) \
-  case 0x##opcode: {                                                      \
-    AppendToBuffer(#instruction " %s,", NameOfXMMRegister(regop));        \
-    current += PrintRightXMMOperand(current);                             \
-    break;                                                                \
-  }
-
-        SSSE3_INSTRUCTION_LIST(SSE34_DIS_CASE)
-        SSSE3_UNOP_INSTRUCTION_LIST(SSE34_DIS_CASE)
-        SSE4_INSTRUCTION_LIST(SSE34_DIS_CASE)
-        SSE4_UNOP_INSTRUCTION_LIST(SSE34_DIS_CASE)
-        SSE4_2_INSTRUCTION_LIST(SSE34_DIS_CASE)
-#undef SSE34_DIS_CASE
-        default:
-          UnimplementedInstruction();
-      }
-    } else if (opcode == 0x3A) {
-      byte third_byte = *current;
-      current = data + 3;
-      get_modrm(*current, &mod, &regop, &rm);
-      if (third_byte == 0x17) {
-        AppendToBuffer("extractps ");  // reg/m32, xmm, imm8
-        current += PrintRightOperand(current);
-        AppendToBuffer(",%s,%d", NameOfXMMRegister(regop), (*current) & 3);
-        current += 1;
-      } else if (third_byte == 0x08) {
-        AppendToBuffer("roundps %s,", NameOfXMMRegister(regop));
-        current += PrintRightXMMOperand(current);
-        AppendToBuffer(",0x%x", (*current) & 3);
-        current += 1;
-      } else if (third_byte == 0x09) {
-        AppendToBuffer("roundpd %s,", NameOfXMMRegister(regop));
-        current += PrintRightXMMOperand(current);
-        AppendToBuffer(",0x%x", (*current) & 3);
-        current += 1;
-      } else if (third_byte == 0x0A) {
-        AppendToBuffer("roundss %s,", NameOfXMMRegister(regop));
-        current += PrintRightXMMOperand(current);
-        AppendToBuffer(",0x%x", (*current) & 3);
-        current += 1;
-      } else if (third_byte == 0x0B) {
-        // roundsd xmm, xmm/m64, imm8
-        AppendToBuffer("roundsd %s,", NameOfXMMRegister(regop));
-        current += PrintRightXMMOperand(current);
-        AppendToBuffer(",0x%x", (*current) & 3);
-        current += 1;
-      } else if (third_byte == 0x0E) {
-        AppendToBuffer("pblendw %s,", NameOfXMMRegister(regop));
-        current += PrintRightXMMOperand(current);
-        AppendToBuffer(",0x%x", *current);
-        current += 1;
-      } else if (third_byte == 0x0F) {
-        AppendToBuffer("palignr %s,", NameOfXMMRegister(regop));
-        current += PrintRightXMMOperand(current);
-        AppendToBuffer(",0x%x", (*current));
-        current += 1;
-      } else if (third_byte == 0x14) {
-        AppendToBuffer("pextrb ");  // reg/m32, xmm, imm8
-        current += PrintRightOperand(current);
-        AppendToBuffer(",%s,%d", NameOfXMMRegister(regop), (*current) & 3);
-        current += 1;
-      } else if (third_byte == 0x15) {
-        AppendToBuffer("pextrw ");  // reg/m32, xmm, imm8
-        current += PrintRightOperand(current);
-        AppendToBuffer(",%s,%d", NameOfXMMRegister(regop), (*current) & 7);
-        current += 1;
-      } else if (third_byte == 0x16) {
-        // reg/m32/reg/m64, xmm, imm8
-        AppendToBuffer("pextr%c ", rex_w() ? 'q' : 'd');
-        current += PrintRightOperand(current);
-        AppendToBuffer(",%s,%d", NameOfXMMRegister(regop), (*current) & 3);
-        current += 1;
-      } else if (third_byte == 0x20) {
-        AppendToBuffer("pinsrb ");  // xmm, reg/m32, imm8
-        AppendToBuffer(" %s,", NameOfXMMRegister(regop));
-        current += PrintRightOperand(current);
-        AppendToBuffer(",%d", (*current) & 3);
-        current += 1;
-      } else if (third_byte == 0x21) {
-        // insertps xmm, xmm/m32, imm8
-        AppendToBuffer("insertps %s,", NameOfXMMRegister(regop));
-        current += PrintRightXMMOperand(current);
-        AppendToBuffer(",0x%x", (*current));
-        current += 1;
-      } else if (third_byte == 0x22) {
-        // xmm, reg/m32/reg/m64, imm8
-        AppendToBuffer("pinsr%c ", rex_w() ? 'q' : 'd');
-        AppendToBuffer(" %s,", NameOfXMMRegister(regop));
-        current += PrintRightOperand(current);
-        AppendToBuffer(",%d", (*current) & 3);
-        current += 1;
-      } else {
-        UnimplementedInstruction();
-      }
-    } else if (opcode == 0xC1) {
+    if (opcode == 0xC1) {
       current += PrintOperands("xadd", OPER_REG_OP_ORDER, current);
-    } else {
-      get_modrm(*current, &mod, &regop, &rm);
-      if (opcode == 0x1F) {
+    } else if (opcode == 0x1F) {
       current++;
       if (rm == 4) {  // SIB byte present.
         current++;
@@ -2162,8 +2059,8 @@ int DisassemblerX64::TwoByteOpcodeInstruction(byte* data) {
         UnimplementedInstruction();
       }
       // Not every opcode here has an XMM register as the dst operand.
-        const char* regop_reg = opcode == 0xD7 ? NameOfCPURegister(regop)
-                                               : NameOfXMMRegister(regop);
+      const char* regop_reg =
+          opcode == 0xD7 ? NameOfCPURegister(regop) : NameOfXMMRegister(regop);
       AppendToBuffer("%s %s,", mnemonic, regop_reg);
       current += PrintRightXMMOperand(current);
       if (opcode == 0xC2) {
@@ -2173,7 +2070,6 @@ int DisassemblerX64::TwoByteOpcodeInstruction(byte* data) {
         current += 1;
       }
     }
-    }
   } else if (group_1_prefix_ == 0xF2) {
     // Beginning of instructions with prefix 0xF2.
     int mod, regop, rm;
@@ -2499,6 +2395,122 @@ int DisassemblerX64::TwoByteOpcodeInstruction(byte* data) {
   return static_cast<int>(current - data);
 }
 
+// Handle all three-byte opcodes, which start with 0x0F38 or 0x0F3A.
+// These instructions may be affected by an 0x66, 0xF2, or 0xF3 prefix, but we
+// only have instructions prefixed with 0x66 for now.
+int DisassemblerX64::ThreeByteOpcodeInstruction(byte* data) {
+  DCHECK_EQ(0x0F, *data);
+  // Only support 3-byte opcodes prefixed with 0x66 for now.
+  DCHECK_EQ(0x66, operand_size_);
+  byte second_byte = *(data + 1);
+  byte third_byte = *(data + 2);
+  byte* current = data + 3;
+  int mod, regop, rm;
+  get_modrm(*current, &mod, &regop, &rm);
+  if (second_byte == 0x38) {
+    switch (third_byte) {
+      case 0x15: {
+        AppendToBuffer("blendvpd %s,", NameOfXMMRegister(regop));
+        current += PrintRightXMMOperand(current);
+        AppendToBuffer(",<xmm0>");
+        break;
+      }
+#define SSE34_DIS_CASE(instruction, notUsed1, notUsed2, notUsed3, opcode) \
+  case 0x##opcode: {                                                      \
+    AppendToBuffer(#instruction " %s,", NameOfXMMRegister(regop));        \
+    current += PrintRightXMMOperand(current);                             \
+    break;                                                                \
+  }
+
+        SSSE3_INSTRUCTION_LIST(SSE34_DIS_CASE)
+        SSSE3_UNOP_INSTRUCTION_LIST(SSE34_DIS_CASE)
+        SSE4_INSTRUCTION_LIST(SSE34_DIS_CASE)
+        SSE4_UNOP_INSTRUCTION_LIST(SSE34_DIS_CASE)
+        SSE4_2_INSTRUCTION_LIST(SSE34_DIS_CASE)
+#undef SSE34_DIS_CASE
+      default:
+        UnimplementedInstruction();
+    }
+  } else {
+    DCHECK_EQ(0x3A, second_byte);
+    if (third_byte == 0x17) {
+      AppendToBuffer("extractps ");  // reg/m32, xmm, imm8
+      current += PrintRightOperand(current);
+      AppendToBuffer(",%s,%d", NameOfXMMRegister(regop), (*current) & 3);
+      current += 1;
+    } else if (third_byte == 0x08) {
+      AppendToBuffer("roundps %s,", NameOfXMMRegister(regop));
+      current += PrintRightXMMOperand(current);
+      AppendToBuffer(",0x%x", (*current) & 3);
+      current += 1;
+    } else if (third_byte == 0x09) {
+      AppendToBuffer("roundpd %s,", NameOfXMMRegister(regop));
+      current += PrintRightXMMOperand(current);
+      AppendToBuffer(",0x%x", (*current) & 3);
+      current += 1;
+    } else if (third_byte == 0x0A) {
+      AppendToBuffer("roundss %s,", NameOfXMMRegister(regop));
+      current += PrintRightXMMOperand(current);
+      AppendToBuffer(",0x%x", (*current) & 3);
+      current += 1;
+    } else if (third_byte == 0x0B) {
+      // roundsd xmm, xmm/m64, imm8
+      AppendToBuffer("roundsd %s,", NameOfXMMRegister(regop));
+      current += PrintRightXMMOperand(current);
+      AppendToBuffer(",0x%x", (*current) & 3);
+      current += 1;
+    } else if (third_byte == 0x0E) {
+      AppendToBuffer("pblendw %s,", NameOfXMMRegister(regop));
+      current += PrintRightXMMOperand(current);
+      AppendToBuffer(",0x%x", *current);
+      current += 1;
+    } else if (third_byte == 0x0F) {
+      AppendToBuffer("palignr %s,", NameOfXMMRegister(regop));
+      current += PrintRightXMMOperand(current);
+      AppendToBuffer(",0x%x", (*current));
+      current += 1;
+    } else if (third_byte == 0x14) {
+      AppendToBuffer("pextrb ");  // reg/m32, xmm, imm8
+      current += PrintRightOperand(current);
+      AppendToBuffer(",%s,%d", NameOfXMMRegister(regop), (*current) & 3);
+      current += 1;
+    } else if (third_byte == 0x15) {
+      AppendToBuffer("pextrw ");  // reg/m32, xmm, imm8
+      current += PrintRightOperand(current);
+      AppendToBuffer(",%s,%d", NameOfXMMRegister(regop), (*current) & 7);
+      current += 1;
+    } else if (third_byte == 0x16) {
+      // reg/m32/reg/m64, xmm, imm8
+      AppendToBuffer("pextr%c ", rex_w() ? 'q' : 'd');
+      current += PrintRightOperand(current);
+      AppendToBuffer(",%s,%d", NameOfXMMRegister(regop), (*current) & 3);
+      current += 1;
+    } else if (third_byte == 0x20) {
+      AppendToBuffer("pinsrb ");  // xmm, reg/m32, imm8
+      AppendToBuffer(" %s,", NameOfXMMRegister(regop));
+      current += PrintRightOperand(current);
+      AppendToBuffer(",%d", (*current) & 3);
+      current += 1;
+    } else if (third_byte == 0x21) {
+      // insertps xmm, xmm/m32, imm8
+      AppendToBuffer("insertps %s,", NameOfXMMRegister(regop));
+      current += PrintRightXMMOperand(current);
+      AppendToBuffer(",0x%x", (*current));
+      current += 1;
+    } else if (third_byte == 0x22) {
+      // xmm, reg/m32/reg/m64, imm8
+      AppendToBuffer("pinsr%c ", rex_w() ? 'q' : 'd');
+      AppendToBuffer(" %s,", NameOfXMMRegister(regop));
+      current += PrintRightOperand(current);
+      AppendToBuffer(",%d", (*current) & 3);
+      current += 1;
+    } else {
+      UnimplementedInstruction();
+    }
+  }
+  return static_cast<int>(current - data);
+}
+
 // Mnemonics for two-byte opcode instructions starting with 0x0F.
 // The argument is the second byte of the two-byte opcode.
 // Returns nullptr if the instruction is not handled here.
@@ -2723,7 +2735,12 @@ int DisassemblerX64::InstructionDecode(v8::internal::Vector<char> out_buffer,
         break;
 
       case 0x0F:
+        // Check for three-byte opcodes, 0x0F38 or 0x0F3A.
+        if (*(data + 1) == 0x38 || *(data + 1) == 0x3A) {
+          data += ThreeByteOpcodeInstruction(data);
+        } else {
           data += TwoByteOpcodeInstruction(data);
+        }
         break;
 
       case 0x8F: {