2

llvm/include/llvm/MC/MCDwarf.h

@@ -530,6 +530,10 @@ class MCCFIInstruction {
     OpGnuArgsSize,
     OpLabel,
     OpValOffset,
+    OpLLVMRegisterPair,
+    OpLLVMVectorRegisters,
+    OpLLVMVectorOffset,
+    OpLLVMVectorRegisterMask,
   };
 
   // Held in ExtraFields for most common OpTypes, exceptions follow.
@@ -548,10 +552,45 @@ class MCCFIInstruction {
   struct LabelFields {
     MCSymbol *CfiLabel = nullptr;
   };
+  /// Held in ExtraFields when OpLLVMRegisterPair.
+  struct RegisterPairFields {
+    unsigned Register;
+    unsigned Reg1, Reg2;
+    unsigned Reg1SizeInBits, Reg2SizeInBits;
+  };
+  struct VectorRegisterWithLane {
+    unsigned Register;
+    unsigned Lane;
+    unsigned SizeInBits;
+  };
+  /// Held in ExtraFields when OpLLVMVectorRegisters.
+  struct VectorRegistersFields {
+    unsigned Register;
+    std::vector<VectorRegisterWithLane> VectorRegisters;
+  };
+  /// Held in ExtraFields when OpLLVMVectorOffset.
+  struct VectorOffsetFields {
+    unsigned Register;
+    unsigned RegisterSizeInBits;
+    int64_t Offset;
+    unsigned MaskRegister;
+    unsigned MaskRegisterSizeInBits;
+  };
+  /// Held in ExtraFields when OpLLVMVectorRegisterMask.
+  struct VectorRegisterMaskFields {
+    unsigned Register;
+    unsigned SpillRegister;
+    unsigned SpillRegisterLaneSizeInBits;
+    unsigned MaskRegister;
+    unsigned MaskRegisterSizeInBits;
+  };
 
 private:
   MCSymbol *Label;
-  std::variant<CommonFields, EscapeFields, LabelFields> ExtraFields;
+  std::variant<CommonFields, EscapeFields, LabelFields, RegisterPairFields,
+               VectorRegistersFields, VectorOffsetFields,
+               VectorRegisterMaskFields>
+      ExtraFields;
   OpType Operation;
   SMLoc Loc;
 
@@ -694,6 +733,57 @@ class MCCFIInstruction {
     return {OpLabel, L, LabelFields{CfiLabel}, Loc};
   }
 
+  /// .cfi_llvm_register_pair Previous value of Register is saved in R1:R2.
+  static MCCFIInstruction
+  createLLVMRegisterPair(MCSymbol *L, unsigned Register, unsigned R1,
+                         unsigned R1SizeInBits, unsigned R2,
+                         unsigned R2SizeInBits, SMLoc Loc = {}) {
+    RegisterPairFields Extra{Register, R1, R2, R1SizeInBits, R2SizeInBits};
+    return {OpLLVMRegisterPair, L, Extra, Loc};
+  }
+
+  /// .cfi_llvm_vector_registers Previous value of Register is saved in lanes of
+  /// vector registers.
+  static MCCFIInstruction
+  createLLVMVectorRegisters(MCSymbol *L, unsigned Register,
+                            std::vector<VectorRegisterWithLane> VectorRegisters,
+                            SMLoc Loc = {}) {
+    VectorRegistersFields Extra{Register, std::move(VectorRegisters)};
+    return {OpLLVMVectorRegisters, L, std::move(Extra), Loc};
+  }
+
+  /// .cfi_llvm_vector_offset Previous value of Register is saved at Offset from
+  /// CFA. MaskRegister specifies the active lanes of register.
+  static MCCFIInstruction
+  createLLVMVectorOffset(MCSymbol *L, unsigned Register,
+                         unsigned RegisterSizeInBits, unsigned MaskRegister,
+                         unsigned MaskRegisterSizeInBits, int64_t Offset,
+                         SMLoc Loc = {}) {
+    VectorOffsetFields Extra{Register, RegisterSizeInBits, Offset, MaskRegister,
+                             MaskRegisterSizeInBits};
+    return MCCFIInstruction(OpLLVMVectorOffset, L, Extra, Loc);
+  }
+
+  /// .cfi_llvm_vector_register_mask Previous value of Register is saved in
+  /// SpillRegister, predicated on the value of MaskRegister.
+  static MCCFIInstruction createLLVMVectorRegisterMask(
+      MCSymbol *L, unsigned Register, unsigned SpillRegister,
+      unsigned SpillRegisterLaneSizeInBits, unsigned MaskRegister,
+      unsigned MaskRegisterSizeInBits, SMLoc Loc = {}) {
+    VectorRegisterMaskFields Extra{
+        Register,     SpillRegister,          SpillRegisterLaneSizeInBits,
+        MaskRegister, MaskRegisterSizeInBits,
+    };
+    return MCCFIInstruction(OpLLVMVectorRegisterMask, L, Extra, Loc);
+  }
+
+  template <class ExtraFieldsTy> ExtraFieldsTy &getExtraFields() {
+    return std::get<ExtraFieldsTy>(ExtraFields);
+  }
+
+  template <class ExtraFieldsTy> const ExtraFieldsTy &getExtraFields() const {
+    return std::get<ExtraFieldsTy>(ExtraFields);
+  }
   /// .cfi_val_offset Previous value of Register is offset Offset from the
   /// current CFA register.
   static MCCFIInstruction createValOffset(MCSymbol *L, unsigned Register,

llvm/include/llvm/MC/MCStreamer.h

@@ -1012,6 +1012,24 @@ class LLVM_ABI MCStreamer {
                                SMLoc Loc = {});
   virtual void emitCFIWindowSave(SMLoc Loc = {});
   virtual void emitCFINegateRAState(SMLoc Loc = {});
+  virtual void emitCFILLVMRegisterPair(int64_t Register, int64_t R1,
+                                       int64_t R1SizeInBits, int64_t R2,
+                                       int64_t R2SizeInBits, SMLoc Loc = {});
+  virtual void emitCFILLVMVectorRegisters(
+      int64_t Register,
+      std::vector<MCCFIInstruction::VectorRegisterWithLane> VRs,
+      SMLoc Loc = {});
+  virtual void emitCFILLVMVectorOffset(int64_t Register,
+                                       int64_t RegisterSizeInBits,
+                                       int64_t MaskRegister,
+                                       int64_t MaskRegisterSizeInBits,
+                                       int64_t Offset, SMLoc Loc = {});
+  virtual void
+  emitCFILLVMVectorRegisterMask(int64_t Register, int64_t SpillRegister,
+                                int64_t SpillRegisterLaneSizeInBits,
+                                int64_t MaskRegister,
+                                int64_t MaskRegisterSizeInBits, SMLoc Loc = {});
+
   virtual void emitCFINegateRAStateWithPC(SMLoc Loc = {});
   virtual void emitCFILabelDirective(SMLoc Loc, StringRef Name);
   virtual void emitCFIValOffset(int64_t Register, int64_t Offset,

llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp

@@ -260,6 +260,39 @@ void AsmPrinter::emitCFIInstruction(const MCCFIInstruction &Inst) const {
   case MCCFIInstruction::OpRestoreState:
     OutStreamer->emitCFIRestoreState(Loc);
     break;
+  case MCCFIInstruction::OpLLVMRegisterPair: {
+    const auto &Fields =
+        Inst.getExtraFields<MCCFIInstruction::RegisterPairFields>();
+    OutStreamer->emitCFILLVMRegisterPair(Fields.Register, Fields.Reg1,
+                                         Fields.Reg1SizeInBits, Fields.Reg2,
+                                         Fields.Reg2SizeInBits, Loc);
+    break;
+  }
+  case MCCFIInstruction::OpLLVMVectorRegisters: {
+    const auto &Fields =
+        Inst.getExtraFields<MCCFIInstruction::VectorRegistersFields>();
+    OutStreamer->emitCFILLVMVectorRegisters(Fields.Register,
+                                            Fields.VectorRegisters, Loc);
+    break;
+  }
+  case MCCFIInstruction::OpLLVMVectorOffset: {
+    const auto &Fields =
+        Inst.getExtraFields<MCCFIInstruction::VectorOffsetFields>();
+    OutStreamer->emitCFILLVMVectorOffset(
+        Fields.Register, Fields.RegisterSizeInBits, Fields.MaskRegister,
+        Fields.MaskRegisterSizeInBits, Fields.Offset, Loc);
+    break;
+  }
+  case MCCFIInstruction::OpLLVMVectorRegisterMask: {
+    const auto &Fields =
+        Inst.getExtraFields<MCCFIInstruction::VectorRegisterMaskFields>();
+    OutStreamer->emitCFILLVMVectorRegisterMask(
+        Fields.Register, Fields.SpillRegister,
+        Fields.SpillRegisterLaneSizeInBits, Fields.MaskRegister,
+        Fields.MaskRegisterSizeInBits);
+    break;
+  }
+
   case MCCFIInstruction::OpValOffset:
     OutStreamer->emitCFIValOffset(Inst.getRegister(), Inst.getOffset(), Loc);
     break;

llvm/lib/CodeGen/CFIInstrInserter.cpp

@@ -262,6 +262,10 @@ void CFIInstrInserter::calculateOutgoingCFAInfo(MBBCFAInfo &MBBInfo) {
       case MCCFIInstruction::OpNegateRAState:
       case MCCFIInstruction::OpNegateRAStateWithPC:
       case MCCFIInstruction::OpGnuArgsSize:
+      case MCCFIInstruction::OpLLVMRegisterPair:
+      case MCCFIInstruction::OpLLVMVectorRegisters:
+      case MCCFIInstruction::OpLLVMVectorOffset:
+      case MCCFIInstruction::OpLLVMVectorRegisterMask:
       case MCCFIInstruction::OpLabel:
       case MCCFIInstruction::OpValOffset:
         break;

llvm/lib/CodeGen/MIRParser/MILexer.cpp

@@ -240,6 +240,11 @@ static MIToken::TokenKind getIdentifierKind(StringRef Identifier) {
       .Case("window_save", MIToken::kw_cfi_window_save)
       .Case("negate_ra_sign_state",
             MIToken::kw_cfi_aarch64_negate_ra_sign_state)
+      .Case("llvm_register_pair", MIToken::kw_cfi_llvm_register_pair)
+      .Case("llvm_vector_registers", MIToken::kw_cfi_llvm_vector_registers)
+      .Case("llvm_vector_offset", MIToken::kw_cfi_llvm_vector_offset)
+      .Case("llvm_vector_register_mask",
+            MIToken::kw_cfi_llvm_vector_register_mask)
       .Case("negate_ra_sign_state_with_pc",
             MIToken::kw_cfi_aarch64_negate_ra_sign_state_with_pc)
       .Case("blockaddress", MIToken::kw_blockaddress)

llvm/lib/CodeGen/MIRParser/MILexer.h

@@ -98,6 +98,10 @@ struct MIToken {
     kw_cfi_undefined,
     kw_cfi_window_save,
     kw_cfi_aarch64_negate_ra_sign_state,
+    kw_cfi_llvm_register_pair,
+    kw_cfi_llvm_vector_registers,
+    kw_cfi_llvm_vector_offset,
+    kw_cfi_llvm_vector_register_mask,
     kw_cfi_aarch64_negate_ra_sign_state_with_pc,
     kw_blockaddress,
     kw_intrinsic,

llvm/lib/CodeGen/MIRParser/MIParser.cpp

@@ -484,6 +484,7 @@ class MIParser {
   bool parseDILocation(MDNode *&Expr);
   bool parseMetadataOperand(MachineOperand &Dest);
   bool parseCFIOffset(int &Offset);
+  bool parseCFIUnsigned(unsigned &Value);
   bool parseCFIRegister(unsigned &Reg);
   bool parseCFIAddressSpace(unsigned &AddressSpace);
   bool parseCFIEscapeValues(std::string& Values);
@@ -2475,6 +2476,14 @@ bool MIParser::parseCFIOffset(int &Offset) {
   return false;
 }
 
+bool MIParser::parseCFIUnsigned(unsigned &Value) {
+  // test
+  if (getUnsigned(Value))
+    return true;
+  lex();
+  return false;
+}
+
 bool MIParser::parseCFIRegister(unsigned &Reg) {
   if (Token.isNot(MIToken::NamedRegister))
     return error("expected a cfi register");
@@ -2608,6 +2617,69 @@ bool MIParser::parseCFIOperand(MachineOperand &Dest) {
   case MIToken::kw_cfi_aarch64_negate_ra_sign_state:
     CFIIndex = MF.addFrameInst(MCCFIInstruction::createNegateRAState(nullptr));
     break;
+  case MIToken::kw_cfi_llvm_register_pair: {
+    unsigned Reg, R1, R2;
+    unsigned R1Size, R2Size;
+    if (parseCFIRegister(Reg) || expectAndConsume(MIToken::comma) ||
+        parseCFIRegister(R1) || expectAndConsume(MIToken::comma) ||
+        parseCFIUnsigned(R1Size) || expectAndConsume(MIToken::comma) ||
+        parseCFIRegister(R2) || expectAndConsume(MIToken::comma) ||
+        parseCFIUnsigned(R2Size))
+      return true;
+
+    CFIIndex = MF.addFrameInst(MCCFIInstruction::createLLVMRegisterPair(
+        nullptr, Reg, R1, R1Size, R2, R2Size));
+    break;
+  }
+  case MIToken::kw_cfi_llvm_vector_registers: {
+    std::vector<MCCFIInstruction::VectorRegisterWithLane> VectorRegisters;
+    if (parseCFIRegister(Reg) || expectAndConsume(MIToken::comma))
+      return true;
+    do {
+      unsigned VR;
+      unsigned Lane, Size;
+      if (parseCFIRegister(VR) || expectAndConsume(MIToken::comma) ||
+          parseCFIUnsigned(Lane) || expectAndConsume(MIToken::comma) ||
+          parseCFIUnsigned(Size))
+        return true;
+      VectorRegisters.push_back({VR, Lane, Size});
+    } while (consumeIfPresent(MIToken::comma));
+
+    CFIIndex = MF.addFrameInst(MCCFIInstruction::createLLVMVectorRegisters(
+        nullptr, Reg, std::move(VectorRegisters)));
+    break;
+  }
+  case MIToken::kw_cfi_llvm_vector_offset: {
+    unsigned Reg, MaskReg;
+    unsigned RegSize, MaskRegSize;
+    int Offset = 0;
+
+    if (parseCFIRegister(Reg) || expectAndConsume(MIToken::comma) ||
+        parseCFIUnsigned(RegSize) || expectAndConsume(MIToken::comma) ||
+        parseCFIRegister(MaskReg) || expectAndConsume(MIToken::comma) ||
+        parseCFIUnsigned(MaskRegSize) || expectAndConsume(MIToken::comma) ||
+        parseCFIOffset(Offset))
+      return true;
+
+    CFIIndex = MF.addFrameInst(MCCFIInstruction::createLLVMVectorOffset(
+        nullptr, Reg, RegSize, MaskReg, MaskRegSize, Offset));
+    break;
+  }
+  case MIToken::kw_cfi_llvm_vector_register_mask: {
+    unsigned Reg, SpillReg, MaskReg;
+    unsigned SpillRegLaneSize, MaskRegSize;
+
+    if (parseCFIRegister(Reg) || expectAndConsume(MIToken::comma) ||
+        parseCFIRegister(SpillReg) || expectAndConsume(MIToken::comma) ||
+        parseCFIUnsigned(SpillRegLaneSize) ||
+        expectAndConsume(MIToken::comma) || parseCFIRegister(MaskReg) ||
+        expectAndConsume(MIToken::comma) || parseCFIUnsigned(MaskRegSize))
+      return true;
+
+    CFIIndex = MF.addFrameInst(MCCFIInstruction::createLLVMVectorRegisterMask(
+        nullptr, Reg, SpillReg, SpillRegLaneSize, MaskReg, MaskRegSize));
+    break;
+  }
   case MIToken::kw_cfi_aarch64_negate_ra_sign_state_with_pc:
     CFIIndex =
         MF.addFrameInst(MCCFIInstruction::createNegateRAStateWithPC(nullptr));
@@ -2962,6 +3034,10 @@ bool MIParser::parseMachineOperand(const unsigned OpCode, const unsigned OpIdx,
   case MIToken::kw_cfi_undefined:
   case MIToken::kw_cfi_window_save:
   case MIToken::kw_cfi_aarch64_negate_ra_sign_state:
+  case MIToken::kw_cfi_llvm_register_pair:
+  case MIToken::kw_cfi_llvm_vector_registers:
+  case MIToken::kw_cfi_llvm_vector_offset:
+  case MIToken::kw_cfi_llvm_vector_register_mask:
   case MIToken::kw_cfi_aarch64_negate_ra_sign_state_with_pc:
     return parseCFIOperand(Dest);
   case MIToken::kw_blockaddress:

llvm/lib/CodeGen/MachineOperand.cpp

@@ -778,6 +778,64 @@ static void printCFI(raw_ostream &OS, const MCCFIInstruction &CFI,
     if (MCSymbol *Label = CFI.getLabel())
       MachineOperand::printSymbol(OS, *Label);
     break;
+  case MCCFIInstruction::OpLLVMRegisterPair: {
+    const auto &Fields =
+        CFI.getExtraFields<MCCFIInstruction::RegisterPairFields>();
+
+    OS << "llvm_register_pair ";
+    if (MCSymbol *Label = CFI.getLabel())
+      MachineOperand::printSymbol(OS, *Label);
+    printCFIRegister(Fields.Register, OS, TRI);
+    OS << ", ";
+    printCFIRegister(Fields.Reg1, OS, TRI);
+    OS << ", " << Fields.Reg1SizeInBits << ", ";
+    printCFIRegister(Fields.Reg2, OS, TRI);
+    OS << ", " << Fields.Reg2SizeInBits;
+    break;
+  }
+  case MCCFIInstruction::OpLLVMVectorRegisters: {
+    const auto &Fields =
+        CFI.getExtraFields<MCCFIInstruction::VectorRegistersFields>();
+
+    OS << "llvm_vector_registers ";
+    if (MCSymbol *Label = CFI.getLabel())
+      MachineOperand::printSymbol(OS, *Label);
+    printCFIRegister(Fields.Register, OS, TRI);
+    for (auto [Reg, Lane, Size] : Fields.VectorRegisters) {
+      OS << ", ";
+      printCFIRegister(Reg, OS, TRI);
+      OS << ", " << Lane << ", " << Size;
+    }
+    break;
+  }
+  case MCCFIInstruction::OpLLVMVectorOffset: {
+    const auto &Fields =
+        CFI.getExtraFields<MCCFIInstruction::VectorOffsetFields>();
+
+    OS << "llvm_vector_offset ";
+    if (MCSymbol *Label = CFI.getLabel())
+      MachineOperand::printSymbol(OS, *Label);
+    printCFIRegister(Fields.Register, OS, TRI);
+    OS << ", " << Fields.RegisterSizeInBits << ", ";
+    printCFIRegister(Fields.MaskRegister, OS, TRI);
+    OS << ", " << Fields.MaskRegisterSizeInBits << ", " << Fields.Offset;
+    break;
+  }
+  case MCCFIInstruction::OpLLVMVectorRegisterMask: {
+    const auto &Fields =
+        CFI.getExtraFields<MCCFIInstruction::VectorRegisterMaskFields>();
+
+    OS << "llvm_vector_register_mask ";
+    if (MCSymbol *Label = CFI.getLabel())
+      MachineOperand::printSymbol(OS, *Label);
+    printCFIRegister(Fields.Register, OS, TRI);
+    OS << ", ";
+    printCFIRegister(Fields.SpillRegister, OS, TRI);
+    OS << ", " << Fields.SpillRegisterLaneSizeInBits << ", ";
+    printCFIRegister(Fields.MaskRegister, OS, TRI);
+    OS << ", " << Fields.MaskRegisterSizeInBits;
+    break;
+  }
   case MCCFIInstruction::OpNegateRAStateWithPC:
     OS << "negate_ra_sign_state_with_pc ";
     if (MCSymbol *Label = CFI.getLabel())