llvm.org GIT mirror llvm / 5aba78b
Update GATHER instructions to support 2 read-write operands. Patch from myself and Manman Ren. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@160110 91177308-0d34-0410-b5e6-96231b3b80d8 Craig Topper 8 years ago
7 changed file(s) with 77 addition(s) and 30 deletion(s). Raw diff Collapse all Expand all
571571 // Classify VEX_B, VEX_4V, VEX_R, VEX_X
572572 unsigned NumOps = Desc.getNumOperands();
573573 unsigned CurOp = 0;
574 if (NumOps > 1 && Desc.getOperandConstraint(1, MCOI::TIED_TO) != -1)
574 if (NumOps > 1 && Desc.getOperandConstraint(1, MCOI::TIED_TO) == 0)
575575 ++CurOp;
576 else if (NumOps > 3 && Desc.getOperandConstraint(2, MCOI::TIED_TO) == 0) {
577 assert(Desc.getOperandConstraint(NumOps - 1, MCOI::TIED_TO) == 1);
578 // Special case for GATHER with 2 TIED_TO operands
579 // Skip the first 2 operands: dst, mask_wb
580 CurOp += 2;
581 }
576582
577583 switch (TSFlags & X86II::FormMask) {
578584 case X86II::MRMInitReg: llvm_unreachable("FIXME: Remove this!");
970976 // FIXME: This should be handled during MCInst lowering.
971977 unsigned NumOps = Desc.getNumOperands();
972978 unsigned CurOp = 0;
973 if (NumOps > 1 && Desc.getOperandConstraint(1, MCOI::TIED_TO) != -1)
979 if (NumOps > 1 && Desc.getOperandConstraint(1, MCOI::TIED_TO) == 0)
974980 ++CurOp;
975 else if (NumOps > 2 && Desc.getOperandConstraint(NumOps-1, MCOI::TIED_TO)== 0)
976 // Skip the last source operand that is tied_to the dest reg. e.g. LXADD32
977 --NumOps;
981 else if (NumOps > 3 && Desc.getOperandConstraint(2, MCOI::TIED_TO) == 0) {
982 assert(Desc.getOperandConstraint(NumOps - 1, MCOI::TIED_TO) == 1);
983 // Special case for GATHER with 2 TIED_TO operands
984 // Skip the first 2 operands: dst, mask_wb
985 CurOp += 2;
986 }
978987
979988 // Keep track of the current byte being emitted.
980989 unsigned CurByte = 0;
934934 // Classify VEX_B, VEX_4V, VEX_R, VEX_X
935935 unsigned NumOps = Desc->getNumOperands();
936936 unsigned CurOp = 0;
937 if (NumOps > 1 && Desc->getOperandConstraint(1, MCOI::TIED_TO) != -1)
937 if (NumOps > 1 && Desc->getOperandConstraint(1, MCOI::TIED_TO) == 0)
938938 ++CurOp;
939 else if (NumOps > 3 && Desc->getOperandConstraint(2, MCOI::TIED_TO) == 0) {
940 assert(Desc->getOperandConstraint(NumOps - 1, MCOI::TIED_TO) == 1);
941 // Special case for GATHER with 2 TIED_TO operands
942 // Skip the first 2 operands: dst, mask_wb
943 CurOp += 2;
944 }
945
939946 switch (TSFlags & X86II::FormMask) {
940947 case X86II::MRMInitReg:
941948 // Duplicate register.
11161123 // If this is a two-address instruction, skip one of the register operands.
11171124 unsigned NumOps = Desc->getNumOperands();
11181125 unsigned CurOp = 0;
1119 if (NumOps > 1 && Desc->getOperandConstraint(1, MCOI::TIED_TO) != -1)
1126 if (NumOps > 1 && Desc->getOperandConstraint(1, MCOI::TIED_TO) == 0)
11201127 ++CurOp;
1121 else if (NumOps > 2 && Desc->getOperandConstraint(NumOps-1,MCOI::TIED_TO)== 0)
1122 // Skip the last source operand that is tied_to the dest reg. e.g. LXADD32
1123 --NumOps;
1128 else if (NumOps > 3 && Desc->getOperandConstraint(2, MCOI::TIED_TO) == 0) {
1129 assert(Desc->getOperandConstraint(NumOps - 1, MCOI::TIED_TO) == 1);
1130 // Special case for GATHER with 2 TIED_TO operands
1131 // Skip the first 2 operands: dst, mask_wb
1132 CurOp += 2;
1133 }
11241134
11251135 uint64_t TSFlags = Desc->TSFlags;
11261136
19651965 if (!Scale)
19661966 return 0;
19671967
1968 SDVTList VTs = CurDAG->getVTList(VSrc.getValueType(), VSrc.getValueType(),
1969 MVT::Other);
1970
19681971 // Memory Operands: Base, Scale, Index, Disp, Segment
19691972 SDValue Disp = CurDAG->getTargetConstant(0, MVT::i32);
19701973 SDValue Segment = CurDAG->getRegister(0, MVT::i32);
19711974 const SDValue Ops[] = { VSrc, Base, getI8Imm(Scale->getSExtValue()), VIdx,
19721975 Disp, Segment, VMask, Chain};
19731976 SDNode *ResNode = CurDAG->getMachineNode(Opc, Node->getDebugLoc(),
1974 VSrc.getValueType(), MVT::Other,
1975 Ops, array_lengthof(Ops));
1977 VTs, Ops, array_lengthof(Ops));
1978 // Node has 2 outputs: VDst and MVT::Other.
1979 // ResNode has 3 outputs: VDst, VMask_wb, and MVT::Other.
1980 // We replace VDst of Node with VDst of ResNode, and Other of Node with Other
1981 // of ResNode.
1982 ReplaceUses(SDValue(Node, 0), SDValue(ResNode, 0));
1983 ReplaceUses(SDValue(Node, 1), SDValue(ResNode, 2));
19761984 return ResNode;
19771985 }
19781986
20332041 }
20342042 SDNode *RetVal = SelectGather(Node, Opc);
20352043 if (RetVal)
2036 return RetVal;
2044 // We already called ReplaceUses inside SelectGather.
2045 return NULL;
20372046 break;
20382047 }
20392048 }
80378037 // VGATHER - GATHER Operations
80388038 multiclass avx2_gather opc, string OpcodeStr,
80398039 RegisterClass RC256, X86MemOperand memop256> {
8040 def rm : AVX28I),
8040 def rm : AVX28I, VR128:$mask_wb),
80418041 (ins VR128:$src1, v128mem:$src2, VR128:$mask),
80428042 !strconcat(OpcodeStr,
80438043 "\t{$mask, $src2, $dst|$dst, $src2, $mask}"),
80448044 []>, VEX_4VOp3;
8045 def Yrm : AVX28I),
8045 def Yrm : AVX28I, RC256:$mask_wb),
80468046 (ins RC256:$src1, memop256:$src2, RC256:$mask),
80478047 !strconcat(OpcodeStr,
80488048 "\t{$mask, $src2, $dst|$dst, $src2, $mask}"),
80498049 []>, VEX_4VOp3, VEX_L;
80508050 }
80518051
8052 let Constraints = "$src1 = $dst" in {
8052 let Constraints = "$src1 = $dst, $mask = $mask_wb" in {
80538053 defm VGATHERDPD : avx2_gather<0x92, "vgatherdpd", VR256, v128mem>, VEX_W;
80548054 defm VGATHERQPD : avx2_gather<0x93, "vgatherqpd", VR256, v256mem>, VEX_W;
80558055 defm VGATHERDPS : avx2_gather<0x92, "vgatherdps", VR256, v256mem>;
11351135 }
11361136 declare <4 x i32> @llvm.x86.avx2.gather.q.d.256(<4 x i32>, i8*,
11371137 <4 x i64>, <4 x i32>, i8) nounwind readonly
1138
1139 ; PR13298
1140 define <8 x float> @test_gather_mask(<8 x float> %a0, float* %a,
1141 <8 x i32> %idx, <8 x float> %mask,
1142 float* nocapture %out) {
1143 ; CHECK: test_gather_mask
1144 ; CHECK: vmovdqa %ymm2, [[DEST:%.*]]
1145 ; CHECK: vgatherdps [[DEST]]
1146 ;; gather with mask
1147 %a_i8 = bitcast float* %a to i8*
1148 %res = call <8 x float> @llvm.x86.avx2.gather.d.ps.256(<8 x float> %a0,
1149 i8* %a_i8, <8 x i32> %idx, <8 x float> %mask, i8 4) ;
1150
1151 ;; for debugging, we'll just dump out the mask
1152 %out_ptr = bitcast float * %out to <8 x float> *
1153 store <8 x float> %mask, <8 x float> * %out_ptr, align 4
1154
1155 ret <8 x float> %res
1156 }
276276 }
277277
278278 void RecognizableInstr::processInstr(DisassemblerTables &tables,
279 const CodeGenInstruction &insn,
280 InstrUID uid)
279 const CodeGenInstruction &insn,
280 InstrUID uid)
281281 {
282282 // Ignore "asm parser only" instructions.
283283 if (insn.TheDef->getValueAsBit("isAsmParserOnly"))
507507 return false;
508508 }
509509
510 void RecognizableInstr::handleOperand(
511 bool optional,
512 unsigned &operandIndex,
513 unsigned &physicalOperandIndex,
514 unsigned &numPhysicalOperands,
515 unsigned *operandMapping,
516 OperandEncoding (*encodingFromString)(const std::string&, bool hasOpSizePrefix)) {
510 void RecognizableInstr::handleOperand(bool optional, unsigned &operandIndex,
511 unsigned &physicalOperandIndex,
512 unsigned &numPhysicalOperands,
513 const unsigned *operandMapping,
514 OperandEncoding (*encodingFromString)
515 (const std::string&,
516 bool hasOpSizePrefix)) {
517517 if (optional) {
518518 if (physicalOperandIndex >= numPhysicalOperands)
519519 return;
562562
563563 const std::vector &OperandList = *Operands;
564564
565 unsigned operandIndex;
566565 unsigned numOperands = OperandList.size();
567566 unsigned numPhysicalOperands = 0;
568567
574573
575574 assert(numOperands <= X86_MAX_OPERANDS && "X86_MAX_OPERANDS is not large enough");
576575
577 for (operandIndex = 0; operandIndex < numOperands; ++operandIndex) {
576 for (unsigned operandIndex = 0; operandIndex < numOperands; ++operandIndex) {
578577 if (OperandList[operandIndex].Constraints.size()) {
579578 const CGIOperandList::ConstraintInfo &Constraint =
580579 OperandList[operandIndex].Constraints[0];
581580 if (Constraint.isTied()) {
582 operandMapping[operandIndex] = Constraint.getTiedOperand();
581 operandMapping[operandIndex] = operandIndex;
582 operandMapping[Constraint.getTiedOperand()] = operandIndex;
583583 } else {
584584 ++numPhysicalOperands;
585585 operandMapping[operandIndex] = operandIndex;
620620 class##EncodingFromString);
621621
622622 // operandIndex should always be < numOperands
623 operandIndex = 0;
623 unsigned operandIndex = 0;
624624 // physicalOperandIndex should always be < numPhysicalOperands
625625 unsigned physicalOperandIndex = 0;
626626
203203 unsigned &operandIndex,
204204 unsigned &physicalOperandIndex,
205205 unsigned &numPhysicalOperands,
206 unsigned *operandMapping,
206 const unsigned *operandMapping,
207207 OperandEncoding (*encodingFromString)
208208 (const std::string&,
209209 bool hasOpSizePrefix));