llvm.org GIT mirror llvm / 9008ca6
2nd attempt, fixing SSE4.1 issues and implementing feedback from duncan. PR2957 ISD::VECTOR_SHUFFLE now stores an array of integers representing the shuffle mask internal to the node, rather than taking a BUILD_VECTOR of ConstantSDNodes as the shuffle mask. A value of -1 represents UNDEF. In addition to eliminating the creation of illegal BUILD_VECTORS just to represent shuffle masks, we are better about canonicalizing the shuffle mask, resulting in substantially better code for some classes of shuffles. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@70225 91177308-0d34-0410-b5e6-96231b3b80d8 Nate Begeman 10 years ago
26 changed file(s) with 1760 addition(s) and 2322 deletion(s). Raw diff Collapse all Expand all
352352 SDValue getConvertRndSat(MVT VT, DebugLoc dl, SDValue Val, SDValue DTy,
353353 SDValue STy,
354354 SDValue Rnd, SDValue Sat, ISD::CvtCode Code);
355
356 /// getVectorShuffle - Return an ISD::VECTOR_SHUFFLE node. The number of
357 /// elements in VT, which must be a vector type, must match the number of
358 /// mask elements NumElts. A negative integer mask element is treated as
359 /// undefined.
360 SDValue getVectorShuffle(MVT VT, DebugLoc dl, SDValue N1, SDValue N2,
361 const int *MaskElts);
355362
356363 /// getZeroExtendInReg - Return the expression required to zero extend the Op
357364 /// value assuming it was the smaller SrcTy value.
2323 #include "llvm/ADT/GraphTraits.h"
2424 #include "llvm/ADT/iterator.h"
2525 #include "llvm/ADT/ilist_node.h"
26 #include "llvm/ADT/SmallVector.h"
2627 #include "llvm/ADT/STLExtras.h"
2728 #include "llvm/CodeGen/ValueTypes.h"
2829 #include "llvm/CodeGen/MachineMemOperand.h"
17021703 }
17031704 };
17041705
1706 class ShuffleVectorSDNode : public SDNode {
1707 SDUse Ops[2];
1708 int *Mask;
1709 protected:
1710 friend class SelectionDAG;
1711 ShuffleVectorSDNode(MVT VT, DebugLoc dl, SDValue N1, SDValue N2, int *M)
1712 : SDNode(ISD::VECTOR_SHUFFLE, dl, getSDVTList(VT)), Mask(M) {
1713 InitOperands(Ops, N1, N2);
1714 }
1715 public:
1716
1717 void getMask(SmallVectorImpl &M) const {
1718 MVT VT = getValueType(0);
1719 M.clear();
1720 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1721 M.push_back(Mask[i]);
1722 }
1723 int getMaskElt(unsigned Idx) const {
1724 assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!");
1725 return Mask[Idx];
1726 }
1727
1728 bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }
1729 int getSplatIndex() const {
1730 assert(isSplat() && "Cannot get splat index for non-splat!");
1731 return Mask[0];
1732 }
1733 static bool isSplatMask(const int *Mask, MVT VT);
1734
1735 static bool classof(const ShuffleVectorSDNode *) { return true; }
1736 static bool classof(const SDNode *N) {
1737 return N->getOpcode() == ISD::VECTOR_SHUFFLE;
1738 }
1739 };
1740
17051741 class ConstantSDNode : public SDNode {
17061742 const ConstantInt *Value;
17071743 friend class SelectionDAG;
20832119 return N->getOpcode() == ISD::CONDCODE;
20842120 }
20852121 };
2086
2122
20872123 /// CvtRndSatSDNode - NOTE: avoid using this node as this may disappear in the
20882124 /// future and most targets don't support it.
20892125 class CvtRndSatSDNode : public SDNode {
2727 #include "llvm/ADT/APFloat.h"
2828 #include "llvm/ADT/DenseMap.h"
2929 #include "llvm/ADT/SmallSet.h"
30 #include "llvm/ADT/SmallVector.h"
3031 #include "llvm/ADT/STLExtras.h"
3132 #include "llvm/CodeGen/DebugLoc.h"
3233 #include "llvm/Target/TargetMachine.h"
327328 /// support *some* VECTOR_SHUFFLE operations, those with specific masks.
328329 /// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
329330 /// are assumed to be legal.
330 virtual bool isShuffleMaskLegal(SDValue Mask, MVT VT) const {
331 virtual bool isShuffleMaskLegal(SmallVectorImpl &Mask, MVT VT) const {
331332 return true;
332333 }
333334
335336 /// used by Targets can use this to indicate if there is a suitable
336337 /// VECTOR_SHUFFLE that can be used to replace a VAND with a constant
337338 /// pool entry.
338 virtual bool isVectorClearMaskLegal(const std::vector &BVOps,
339 MVT EVT,
340 SelectionDAG &DAG) const {
339 virtual bool isVectorClearMaskLegal(SmallVectorImpl &M, MVT VT) const {
341340 return false;
342341 }
343342
4848
4949 class SDTCisOpSmallerThanOp : SDTypeConstraint{
5050 int BigOperandNum = BigOp;
51 }
52
53 /// SDTCisIntVectorOfSameSize - This indicates that ThisOp and OtherOp are
54 /// vector types, and that ThisOp is the result of
55 /// MVT::getIntVectorWithNumElements with the number of elements
56 /// that ThisOp has.
57 class SDTCisIntVectorOfSameSize
58 : SDTypeConstraint {
59 int OtherOpNum = OtherOp;
6051 }
6152
6253 /// SDTCisEltOfVec - This indicates that ThisOp is a scalar type of the same
174165 SDTCisSameAs<0, 2>, SDTCisPtrTy<0>, SDTCisPtrTy<3>
175166 ]>;
176167
177 def SDTVecShuffle : SDTypeProfile<1, 3, [
178 SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2>, SDTCisIntVectorOfSameSize<3, 0>
168 def SDTVecShuffle : SDTypeProfile<1, 2, [
169 SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2>
179170 ]>;
180171 def SDTVecExtract : SDTypeProfile<1, 2, [ // vector extract
181172 SDTCisEltOfVec<0, 1>, SDTCisPtrTy<2>
51015101 return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(),
51025102 InVec.getValueType(), &Ops[0], Ops.size());
51035103 }
5104
5104 // If the invec is an UNDEF and if EltNo is a constant, create a new
5105 // BUILD_VECTOR with undef elements and the inserted element.
5106 if (!LegalOperations && InVec.getOpcode() == ISD::UNDEF &&
5107 isa(EltNo)) {
5108 MVT VT = InVec.getValueType();
5109 MVT EVT = VT.getVectorElementType();
5110 unsigned NElts = VT.getVectorNumElements();
5111 SmallVector Ops(NElts, DAG.getUNDEF(EVT));
5112
5113 unsigned Elt = cast(EltNo)->getZExtValue();
5114 if (Elt < Ops.size())
5115 Ops[Elt] = InVal;
5116 return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(),
5117 InVec.getValueType(), &Ops[0], Ops.size());
5118 }
51055119 return SDValue();
51065120 }
51075121
51635177 // to examine the mask.
51645178 if (BCNumEltsChanged)
51655179 return SDValue();
5166 unsigned Idx = cast(InVec.getOperand(2).
5167 getOperand(Elt))->getZExtValue();
5168 unsigned NumElems = InVec.getOperand(2).getNumOperands();
5180 int Idx = cast(InVec)->getMaskElt(Elt);
5181 int NumElems = InVec.getValueType().getVectorNumElements();
51695182 InVec = (Idx < NumElems) ? InVec.getOperand(0) : InVec.getOperand(1);
51705183 if (InVec.getOpcode() == ISD::BIT_CONVERT)
51715184 InVec = InVec.getOperand(0);
52125225 SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
52135226 unsigned NumInScalars = N->getNumOperands();
52145227 MVT VT = N->getValueType(0);
5215 unsigned NumElts = VT.getVectorNumElements();
52165228 MVT EltType = VT.getVectorElementType();
52175229
52185230 // Check to see if this is a BUILD_VECTOR of a bunch of EXTRACT_VECTOR_ELT
52555267 }
52565268
52575269 // If everything is good, we can make a shuffle operation.
5258 MVT IndexVT = MVT::i32;
52595270 if (VecIn1.getNode()) {
5260 SmallVector<SDValue, 8> BuildVecIndices;
5271 SmallVector<int, 8> Mask;
52615272 for (unsigned i = 0; i != NumInScalars; ++i) {
52625273 if (N->getOperand(i).getOpcode() == ISD::UNDEF) {
5263 BuildVecIndices.push_back(DAG.getUNDEF(IndexVT));
5274 Mask.push_back(-1);
52645275 continue;
52655276 }
52665277
5278 // If extracting from the first vector, just use the index directly.
52675279 SDValue Extract = N->getOperand(i);
5268
5269 // If extracting from the first vector, just use the index directly.
52705280 SDValue ExtVal = Extract.getOperand(1);
52715281 if (Extract.getOperand(0) == VecIn1) {
5272 if (ExtVal.getValueType() == IndexVT)
5273 BuildVecIndices.push_back(ExtVal);
5274 else {
5275 unsigned Idx = cast(ExtVal)->getZExtValue();
5276 BuildVecIndices.push_back(DAG.getConstant(Idx, IndexVT));
5277 }
5282 Mask.push_back(cast(ExtVal)->getZExtValue());
52785283 continue;
52795284 }
52805285
52815286 // Otherwise, use InIdx + VecSize
52825287 unsigned Idx = cast(ExtVal)->getZExtValue();
5283 BuildVecIndices.push_back(DAG.getConstant(Idx+NumInScalars, IndexVT));
5288 Mask.push_back(Idx+NumInScalars);
52845289 }
52855290
52865291 // Add count and size info.
5287 MVT BuildVecVT = MVT::getVectorVT(IndexVT, NumElts);
5288 if (!TLI.isTypeLegal(BuildVecVT) && LegalTypes)
5292 if (!TLI.isTypeLegal(VT) && LegalTypes)
52895293 return SDValue();
52905294
52915295 // Return the new VECTOR_SHUFFLE node.
5292 SDValue Ops[5];
5296 SDValue Ops[2];
52935297 Ops[0] = VecIn1;
5294 if (VecIn2.getNode()) {
5295 Ops[1] = VecIn2;
5296 } else {
5297 // Use an undef build_vector as input for the second operand.
5298 std::vector UnOps(NumInScalars,
5299 DAG.getUNDEF(EltType));
5300 Ops[1] = DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), VT,
5301 &UnOps[0], UnOps.size());
5302 AddToWorkList(Ops[1].getNode());
5303 }
5304
5305 Ops[2] = DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), BuildVecVT,
5306 &BuildVecIndices[0], BuildVecIndices.size());
5307 return DAG.getNode(ISD::VECTOR_SHUFFLE, N->getDebugLoc(), VT, Ops, 3);
5298 Ops[1] = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT);
5299 return DAG.getVectorShuffle(VT, N->getDebugLoc(), Ops[0], Ops[1], &Mask[0]);
53085300 }
53095301
53105302 return SDValue();
53245316 }
53255317
53265318 SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) {
5327 SDValue ShufMask = N->getOperand(2);
5328 unsigned NumElts = ShufMask.getNumOperands();
5319 return SDValue();
5320
5321 MVT VT = N->getValueType(0);
5322 unsigned NumElts = VT.getVectorNumElements();
53295323
53305324 SDValue N0 = N->getOperand(0);
53315325 SDValue N1 = N->getOperand(1);
53335327 assert(N0.getValueType().getVectorNumElements() == NumElts &&
53345328 "Vector shuffle must be normalized in DAG");
53355329
5336 // If the shuffle mask is an identity operation on the LHS, return the LHS.
5337 bool isIdentity = true;
5338 for (unsigned i = 0; i != NumElts; ++i) {
5339 if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF &&
5340 cast(ShufMask.getOperand(i))->getZExtValue() != i) {
5341 isIdentity = false;
5342 break;
5343 }
5344 }
5345 if (isIdentity) return N->getOperand(0);
5346
5347 // If the shuffle mask is an identity operation on the RHS, return the RHS.
5348 isIdentity = true;
5349 for (unsigned i = 0; i != NumElts; ++i) {
5350 if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF &&
5351 cast(ShufMask.getOperand(i))->getZExtValue() !=
5352 i+NumElts) {
5353 isIdentity = false;
5354 break;
5355 }
5356 }
5357 if (isIdentity) return N->getOperand(1);
5358
5359 // Check if the shuffle is a unary shuffle, i.e. one of the vectors is not
5360 // needed at all.
5361 bool isUnary = true;
5362 bool isSplat = true;
5363 int VecNum = -1;
5364 unsigned BaseIdx = 0;
5365 for (unsigned i = 0; i != NumElts; ++i)
5366 if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF) {
5367 unsigned Idx=cast(ShufMask.getOperand(i))->getZExtValue();
5368 int V = (Idx < NumElts) ? 0 : 1;
5369 if (VecNum == -1) {
5370 VecNum = V;
5371 BaseIdx = Idx;
5372 } else {
5373 if (BaseIdx != Idx)
5374 isSplat = false;
5375 if (VecNum != V) {
5376 isUnary = false;
5377 break;
5378 }
5379 }
5380 }
5381
5382 // Normalize unary shuffle so the RHS is undef.
5383 if (isUnary && VecNum == 1)
5384 std::swap(N0, N1);
5330 // FIXME: implement canonicalizations from DAG.getVectorShuffle()
53855331
53865332 // If it is a splat, check if the argument vector is a build_vector with
53875333 // all scalar elements the same.
5388 if (isSplat) {
5334 if (cast(N)->isSplat()) {
53895335 SDNode *V = N0.getNode();
5336
53905337
53915338 // If this is a bit convert that changes the element type of the vector but
53925339 // not the number of vector elements, look through it. Be careful not to
54005347
54015348 if (V->getOpcode() == ISD::BUILD_VECTOR) {
54025349 unsigned NumElems = V->getNumOperands();
5350 unsigned BaseIdx = cast(N)->getSplatIndex();
54035351 if (NumElems > BaseIdx) {
54045352 SDValue Base;
54055353 bool AllSame = true;
54245372 }
54255373 }
54265374 }
5427
5428 // If it is a unary or the LHS and the RHS are the same node, turn the RHS
5429 // into an undef.
5430 if (isUnary || N0 == N1) {
5431 // Check the SHUFFLE mask, mapping any inputs from the 2nd operand into the
5432 // first operand.
5433 SmallVector MappedOps;
5434
5435 for (unsigned i = 0; i != NumElts; ++i) {
5436 if (ShufMask.getOperand(i).getOpcode() == ISD::UNDEF ||
5437 cast(ShufMask.getOperand(i))->getZExtValue() <
5438 NumElts) {
5439 MappedOps.push_back(ShufMask.getOperand(i));
5440 } else {
5441 unsigned NewIdx =
5442 cast(ShufMask.getOperand(i))->getZExtValue() -
5443 NumElts;
5444 MappedOps.push_back(DAG.getConstant(NewIdx,
5445 ShufMask.getOperand(i).getValueType()));
5446 }
5447 }
5448
5449 ShufMask = DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(),
5450 ShufMask.getValueType(),
5451 &MappedOps[0], MappedOps.size());
5452 AddToWorkList(ShufMask.getNode());
5453 return DAG.getNode(ISD::VECTOR_SHUFFLE, N->getDebugLoc(),
5454 N->getValueType(0), N0,
5455 DAG.getUNDEF(N->getValueType(0)),
5456 ShufMask);
5457 }
5458
54595375 return SDValue();
54605376 }
54615377
54645380 /// e.g. AND V, <0xffffffff, 0, 0xffffffff, 0>. ==>
54655381 /// vector_shuffle V, Zero, <0, 4, 2, 4>
54665382 SDValue DAGCombiner::XformToShuffleWithZero(SDNode *N) {
5383 MVT VT = N->getValueType(0);
5384 DebugLoc dl = N->getDebugLoc();
54675385 SDValue LHS = N->getOperand(0);
54685386 SDValue RHS = N->getOperand(1);
54695387 if (N->getOpcode() == ISD::AND) {
54705388 if (RHS.getOpcode() == ISD::BIT_CONVERT)
54715389 RHS = RHS.getOperand(0);
54725390 if (RHS.getOpcode() == ISD::BUILD_VECTOR) {
5473 std::vector IdxOps;
5474 unsigned NumOps = RHS.getNumOperands();
5475 unsigned NumElts = NumOps;
5391 SmallVector Indices;
5392 unsigned NumElts = RHS.getNumOperands();
54765393 for (unsigned i = 0; i != NumElts; ++i) {
54775394 SDValue Elt = RHS.getOperand(i);
54785395 if (!isa(Elt))
54795396 return SDValue();
54805397 else if (cast(Elt)->isAllOnesValue())
5481 IdxOps.push_back(DAG.getIntPtrConstant(i));
5398 Indices.push_back(i);
54825399 else if (cast(Elt)->isNullValue())
5483 IdxOps.push_back(DAG.getIntPtrConstant(NumElts));
5400 Indices.push_back(NumElts);
54845401 else
54855402 return SDValue();
54865403 }
54875404
54885405 // Let's see if the target supports this vector_shuffle.
5489 if (!TLI.isVectorClearMaskLegal(IdxOps, TLI.getPointerTy(), DAG))
5406 MVT RVT = RHS.getValueType();
5407 if (!TLI.isVectorClearMaskLegal(Indices, RVT))
54905408 return SDValue();
54915409
54925410 // Return the new VECTOR_SHUFFLE node.
5493 MVT EVT = RHS.getValueType().getVectorElementType();
5494 MVT VT = MVT::getVectorVT(EVT, NumElts);
5495 MVT MaskVT = MVT::getVectorVT(TLI.getPointerTy(), NumElts);
5496 std::vector Ops;
5497 LHS = DAG.getNode(ISD::BIT_CONVERT, LHS.getDebugLoc(), VT, LHS);
5498 Ops.push_back(LHS);
5499 AddToWorkList(LHS.getNode());
5500 std::vector ZeroOps(NumElts, DAG.getConstant(0, EVT));
5501 Ops.push_back(DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(),
5502 VT, &ZeroOps[0], ZeroOps.size()));
5503 Ops.push_back(DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(),
5504 MaskVT, &IdxOps[0], IdxOps.size()));
5505 SDValue Result = DAG.getNode(ISD::VECTOR_SHUFFLE, N->getDebugLoc(),
5506 VT, &Ops[0], Ops.size());
5507
5508 if (VT != N->getValueType(0))
5509 Result = DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(),
5510 N->getValueType(0), Result);
5511
5512 return Result;
5411 MVT EVT = RVT.getVectorElementType();
5412 SmallVector ZeroOps(RVT.getVectorNumElements(),
5413 DAG.getConstant(0, EVT));
5414 SDValue Zero = DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(),
5415 RVT, &ZeroOps[0], ZeroOps.size());
5416 LHS = DAG.getNode(ISD::BIT_CONVERT, dl, RVT, LHS);
5417 SDValue Shuf = DAG.getVectorShuffle(RVT, dl, LHS, Zero, &Indices[0]);
5418 return DAG.getNode(ISD::BIT_CONVERT, dl, VT, Shuf);
55135419 }
55145420 }
55155421
266266 bool isVolatile, SDValue ValOp,
267267 unsigned StWidth, DebugLoc dl);
268268
269 /// isShuffleLegal - Return non-null if a vector shuffle is legal with the
270 /// specified mask and type. Targets can specify exactly which masks they
271 /// support and the code generator is tasked with not creating illegal masks.
272 ///
273 /// Note that this will also return true for shuffles that are promoted to a
274 /// different type.
275 ///
276 /// If this is a legal shuffle, this method returns the (possibly promoted)
277 /// build_vector Mask. If it's not a legal shuffle, it returns null.
278 SDNode *isShuffleLegal(MVT VT, SDValue Mask) const;
269 /// promoteShuffle - Promote a shuffle mask of a vector VT to perform the
270 /// same shuffle on a vector of NVT. Must not create an illegal shuffle mask.
271 SDValue promoteShuffle(MVT NVT, MVT VT, DebugLoc dl, SDValue N1, SDValue N2,
272 SmallVectorImpl &Mask) const;
279273
280274 bool LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest,
281275 SmallPtrSet &NodesLeadingTo);
318312 };
319313 }
320314
321 /// isVectorShuffleLegal - Return true if a vector shuffle is legal with the
322 /// specified mask and type. Targets can specify exactly which masks they
323 /// support and the code generator is tasked with not creating illegal masks.
324 ///
325 /// Note that this will also return true for shuffles that are promoted to a
326 /// different type.
327 SDNode *SelectionDAGLegalize::isShuffleLegal(MVT VT, SDValue Mask) const {
328 switch (TLI.getOperationAction(ISD::VECTOR_SHUFFLE, VT)) {
329 default: return 0;
330 case TargetLowering::Legal:
331 case TargetLowering::Custom:
332 break;
333 case TargetLowering::Promote: {
334 // If this is promoted to a different type, convert the shuffle mask and
335 // ask if it is legal in the promoted type!
336 MVT NVT = TLI.getTypeToPromoteTo(ISD::VECTOR_SHUFFLE, VT);
337 MVT EltVT = NVT.getVectorElementType();
338
339 // If we changed # elements, change the shuffle mask.
340 unsigned NumEltsGrowth =
341 NVT.getVectorNumElements() / VT.getVectorNumElements();
342 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
343 if (NumEltsGrowth > 1) {
344 // Renumber the elements.
345 SmallVector Ops;
346 for (unsigned i = 0, e = Mask.getNumOperands(); i != e; ++i) {
347 SDValue InOp = Mask.getOperand(i);
348 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
349 if (InOp.getOpcode() == ISD::UNDEF)
350 Ops.push_back(DAG.getUNDEF(EltVT));
351 else {
352 unsigned InEltNo = cast(InOp)->getZExtValue();
353 Ops.push_back(DAG.getConstant(InEltNo*NumEltsGrowth+j, EltVT));
354 }
355 }
356 }
357 Mask = DAG.getNode(ISD::BUILD_VECTOR, Mask.getDebugLoc(),
358 NVT, &Ops[0], Ops.size());
359 }
360 VT = NVT;
361 break;
362 }
363 }
364 return TLI.isShuffleMaskLegal(Mask, VT) ? Mask.getNode() : 0;
315 /// promoteShuffle - Promote a shuffle mask of a vector VT to perform the
316 /// same shuffle on a vector of NVT. Must not create an illegal shuffle mask.
317 /// e.g. <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
318 SDValue SelectionDAGLegalize::promoteShuffle(MVT NVT, MVT VT, DebugLoc dl,
319 SDValue N1, SDValue N2,
320 SmallVectorImpl &Mask) const {
321 MVT EltVT = NVT.getVectorElementType();
322 int NumMaskElts = VT.getVectorNumElements();
323 int NumDestElts = NVT.getVectorNumElements();
324 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
325
326 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
327
328 if (NumEltsGrowth == 1)
329 return DAG.getVectorShuffle(NVT, dl, N1, N2, &Mask[0]);
330
331 SmallVector NewMask;
332 for (int i = 0; i != NumMaskElts; ++i) {
333 int Idx = Mask[i];
334 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
335 if (Idx < 0)
336 NewMask.push_back(-1);
337 else
338 NewMask.push_back(Idx * NumEltsGrowth + j);
339 }
340 }
341 assert((int)NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
342 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
343 return DAG.getVectorShuffle(NVT, dl, N1, N2, &NewMask[0]);
365344 }
366345
367346 SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag,
16511630 Tmp1.getValueType(), Tmp2);
16521631
16531632 unsigned NumElts = Tmp1.getValueType().getVectorNumElements();
1654 MVT ShufMaskVT =
1655 MVT::getIntVectorWithNumElements(NumElts);
1656 MVT ShufMaskEltVT = ShufMaskVT.getVectorElementType();
1657
16581633 // We generate a shuffle of InVec and ScVec, so the shuffle mask
16591634 // should be 0,1,2,3,4,5... with the appropriate element replaced with
16601635 // elt 0 of the RHS.
1661 SmallVector ShufOps;
1662 for (unsigned i = 0; i != NumElts; ++i) {
1663 if (i != InsertPos->getZExtValue())
1664 ShufOps.push_back(DAG.getConstant(i, ShufMaskEltVT));
1665 else
1666 ShufOps.push_back(DAG.getConstant(NumElts, ShufMaskEltVT));
1667 }
1668 SDValue ShufMask = DAG.getNode(ISD::BUILD_VECTOR, dl, ShufMaskVT,
1669 &ShufOps[0], ShufOps.size());
1670
1671 Result = DAG.getNode(ISD::VECTOR_SHUFFLE, dl, Tmp1.getValueType(),
1672 Tmp1, ScVec, ShufMask);
1636 SmallVector ShufOps;
1637 for (unsigned i = 0; i != NumElts; ++i)
1638 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
1639
1640 Result = DAG.getVectorShuffle(Tmp1.getValueType(), dl, Tmp1, ScVec,
1641 &ShufOps[0]);
16731642 Result = LegalizeOp(Result);
16741643 break;
16751644 }
17041673 break;
17051674 }
17061675 break;
1707 case ISD::VECTOR_SHUFFLE:
1676 case ISD::VECTOR_SHUFFLE: {
17081677 Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the input vectors,
17091678 Tmp2 = LegalizeOp(Node->getOperand(1)); // but not the shuffle mask.
1710 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2));
1679 Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
1680 MVT VT = Result.getValueType();
1681
1682 // Copy the Mask to a local SmallVector for use wi
1683 SmallVector Mask;
1684 cast(Result)->getMask(Mask);
17111685
17121686 // Allow targets to custom lower the SHUFFLEs they support.
1713 switch (TLI.getOperationAction(ISD::VECTOR_SHUFFLE, Result.getValueType())){
1687 switch (TLI.getOperationAction(ISD::VECTOR_SHUFFLE, VT)) {
17141688 default: assert(0 && "Unknown operation action!");
17151689 case TargetLowering::Legal:
1716 assert(isShuffleLegal(Result.getValueType(), Node->getOperand(2)) &&
1690 assert(TLI.isShuffleMaskLegal(Mask, VT) &&
17171691 "vector shuffle should not be created if not legal!");
17181692 break;
17191693 case TargetLowering::Custom:
17241698 }
17251699 // FALLTHROUGH
17261700 case TargetLowering::Expand: {
1727 MVT VT = Node->getValueType(0);
17281701 MVT EltVT = VT.getVectorElementType();
1729 MVT PtrVT = TLI.getPointerTy();
1730 SDValue Mask = Node->getOperand(2);
1731 unsigned NumElems = Mask.getNumOperands();
1702 int NumElems = VT.getVectorNumElements();
17321703 SmallVector Ops;
1733 for (unsigned i = 0; i != NumElems; ++i) {
1734 SDValue Arg = Mask.getOperand(i);
1735 if (Arg.getOpcode() == ISD::UNDEF) {
1704 for (int i = 0; i != NumElems; ++i) {
1705 if (Mask[i] < 0) {
17361706 Ops.push_back(DAG.getUNDEF(EltVT));
1737 } else {
1738 assert(isa(Arg) && "Invalid VECTOR_SHUFFLE mask!");
1739 unsigned Idx = cast(Arg)->getZExtValue();
1740 if (Idx < NumElems)
1741 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Tmp1,
1742 DAG.getConstant(Idx, PtrVT)));
1743 else
1744 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Tmp2,
1745 DAG.getConstant(Idx - NumElems, PtrVT)));
1707 continue;
17461708 }
1709 int Idx = Mask[i];
1710 if (Idx < NumElems)
1711 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Tmp1,
1712 DAG.getIntPtrConstant(Idx)));
1713 else
1714 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Tmp2,
1715 DAG.getIntPtrConstant(Idx - NumElems)));
17471716 }
17481717 Result = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], Ops.size());
17491718 break;
17581727 Tmp2 = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Tmp2);
17591728
17601729 // Convert the shuffle mask to the right # elements.
1761 Tmp3 = SDValue(isShuffleLegal(OVT, Node->getOperand(2)), 0);
1762 assert(Tmp3.getNode() && "Shuffle not legal?");
1763 Result = DAG.getNode(ISD::VECTOR_SHUFFLE, dl, NVT, Tmp1, Tmp2, Tmp3);
1730 Result = promoteShuffle(NVT, OVT, dl, Tmp1, Tmp2, Mask);
17641731 Result = DAG.getNode(ISD::BIT_CONVERT, dl, OVT, Result);
17651732 break;
17661733 }
17671734 }
17681735 break;
1769
1736 }
17701737 case ISD::EXTRACT_VECTOR_ELT:
17711738 Tmp1 = Node->getOperand(0);
17721739 Tmp2 = LegalizeOp(Node->getOperand(1));
54895456
54905457 // FIXME: it would be far nicer to change this into map
54915458 // and use a bitmask instead of a list of elements.
5459 // FIXME: this doesn't treat <0, u, 0, u> for example, as a splat.
54925460 std::map > Values;
54935461 Values[SplatValue].push_back(0);
54945462 bool isConstant = true;
55455513
55465514 if (SplatValue.getNode()) { // Splat of one value?
55475515 // Build the shuffle constant vector: <0, 0, 0, 0>
5548 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems);
5549 SDValue Zero = DAG.getConstant(0, MaskVT.getVectorElementType());
5550 std::vector ZeroVec(NumElems, Zero);
5551 SDValue SplatMask = DAG.getNode(ISD::BUILD_VECTOR, dl, MaskVT,
5552 &ZeroVec[0], ZeroVec.size());
5516 SmallVector ZeroVec(NumElems, 0);
55535517
55545518 // If the target supports VECTOR_SHUFFLE and this shuffle mask, use it.
5555 if (isShuffleLegal(VT, SplatMask)) {
5519 if (TLI.isShuffleMaskLegal(ZeroVec, Node->getValueType(0))) {
55565520 // Get the splatted value into the low element of a vector register.
55575521 SDValue LowValVec =
55585522 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, SplatValue);
55595523
55605524 // Return shuffle(LowValVec, undef, <0,0,0,0>)
5561 return DAG.getNode(ISD::VECTOR_SHUFFLE, dl, VT, LowValVec,
5562 DAG.getUNDEF(VT), SplatMask);
5525 return DAG.getVectorShuffle(VT, dl, LowValVec, DAG.getUNDEF(VT),
5526 &ZeroVec[0]);
55635527 }
55645528 }
55655529
55815545 std::swap(Val1, Val2);
55825546
55835547 // Build the shuffle constant vector: e.g. <0, 4, 0, 4>
5584 MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems);
5585 MVT MaskEltVT = MaskVT.getVectorElementType();
5586 std::vector MaskVec(NumElems);
5548 SmallVector ShuffleMask(NumElems, -1);
55875549
55885550 // Set elements of the shuffle mask for Val1.
55895551 std::vector &Val1Elts = Values[Val1];
55905552 for (unsigned i = 0, e = Val1Elts.size(); i != e; ++i)
5591 MaskVec[Val1Elts[i]] = DAG.getConstant(0, MaskEltVT);
5553 ShuffleMask[Val1Elts[i]] = 0;
55925554
55935555 // Set elements of the shuffle mask for Val2.
55945556 std::vector &Val2Elts = Values[Val2];
55955557 for (unsigned i = 0, e = Val2Elts.size(); i != e; ++i)
55965558 if (Val2.getOpcode() != ISD::UNDEF)
5597 MaskVec[Val2Elts[i]] = DAG.getConstant(NumElems, MaskEltVT);
5598 else
5599 MaskVec[Val2Elts[i]] = DAG.getUNDEF(MaskEltVT);
5600
5601 SDValue ShuffleMask = DAG.getNode(ISD::BUILD_VECTOR, dl, MaskVT,
5602 &MaskVec[0], MaskVec.size());
5559 ShuffleMask[Val2Elts[i]] = NumElems;
56035560
56045561 // If the target supports SCALAR_TO_VECTOR and this shuffle mask, use it.
56055562 if (TLI.isOperationLegalOrCustom(ISD::SCALAR_TO_VECTOR, VT) &&
5606 isShuffleLegal(VT, ShuffleMask)) {
5563 TLI.isShuffleMaskLegal(ShuffleMask, VT)) {
56075564 Val1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Val1);
56085565 Val2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Val2);
5609 SDValue Ops[] = { Val1, Val2, ShuffleMask };
5610
5611 // Return shuffle(LoValVec, HiValVec, <0,1,0,1>)
5612 return DAG.getNode(ISD::VECTOR_SHUFFLE, dl, VT, Ops, 3);
5566 return DAG.getVectorShuffle(VT, dl, Val1, Val2, &ShuffleMask[0]);
56135567 }
56145568 }
56155569
80658019 case ISD::VECTOR_SHUFFLE: {
80668020 SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT);
80678021 SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), WidenVT);
8068 // VECTOR_SHUFFLE 3rd operand must be a constant build vector that is
8069 // used as permutation array. We build the vector here instead of widening
8070 // because we don't want to legalize and have it turned to something else.
8071 SDValue PermOp = Node->getOperand(2);
8072 SDValueVector NewOps;
8073 MVT PVT = PermOp.getValueType().getVectorElementType();
8022 ShuffleVectorSDNode *SVOp = cast(Node);
8023 SmallVector NewMask;
80748024 for (unsigned i = 0; i < NumElts; ++i) {
8075 if (PermOp.getOperand(i).getOpcode() == ISD::UNDEF) {
8076 NewOps.push_back(PermOp.getOperand(i));
8077 } else {
8078 unsigned Idx =
8079 cast(PermOp.getOperand(i))->getZExtValue();
8080 if (Idx < NumElts) {
8081 NewOps.push_back(PermOp.getOperand(i));
8082 }
8083 else {
8084 NewOps.push_back(DAG.getConstant(Idx + NewNumElts - NumElts,
8085 PermOp.getOperand(i).getValueType()));
8086 }
8087 }
8088 }
8089 for (unsigned i = NumElts; i < NewNumElts; ++i) {
8090 NewOps.push_back(DAG.getUNDEF(PVT));
8091 }
8092
8093 SDValue Tmp3 = DAG.getNode(ISD::BUILD_VECTOR, dl,
8094 MVT::getVectorVT(PVT, NewOps.size()),
8095 &NewOps[0], NewOps.size());
8096
8097 Result = DAG.getNode(ISD::VECTOR_SHUFFLE, dl, WidenVT, Tmp1, Tmp2, Tmp3);
8025 int Idx = SVOp->getMaskElt(i);
8026 if (Idx < (int)NumElts)
8027 NewMask.push_back(Idx);
8028 else
8029 NewMask.push_back(Idx + NewNumElts - NumElts);
8030 }
8031 for (unsigned i = NumElts; i < NewNumElts; ++i)
8032 NewMask.push_back(-1);
8033
8034 Result = DAG.getVectorShuffle(WidenVT, dl, Tmp1, Tmp2, &NewMask[0]);
80988035 break;
80998036 }
81008037 case ISD::LOAD: {
760760 SDValue &Hi) {
761761 // The low and high parts of the original input give four input vectors.
762762 SDValue Inputs[4];
763 ShuffleVectorSDNode *SVN = cast(N);
763764 DebugLoc dl = N->getDebugLoc();
764765 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
765766 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
771772 // If Lo or Hi uses elements from at most two of the four input vectors, then
772773 // express it as a vector shuffle of those two inputs. Otherwise extract the
773774 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
774 SDValue Mask = N->getOperand(2);
775 MVT IdxVT = Mask.getValueType().getVectorElementType();
776 SmallVector Ops;
777 Ops.reserve(NewElts);
775 SmallVector Ops;
778776 for (unsigned High = 0; High < 2; ++High) {
779777 SDValue &Output = High ? Hi : Lo;
780778
786784 unsigned FirstMaskIdx = High * NewElts;
787785 bool useBuildVector = false;
788786 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
789 SDValue Arg = Mask.getOperand(FirstMaskIdx + MaskOffset);
790
791787 // The mask element. This indexes into the input.
792 unsigned Idx = Arg.getOpcode() == ISD::UNDEF ?
793 -1U : cast(Arg)->getZExtValue();
788 int Idx = SVN->getMaskElt(FirstMaskIdx + MaskOffset);
794789
795790 // The input vector this mask element indexes into.
796 unsigned Input = Idx / NewElts;
791 unsigned Input = (unsigned)Idx / NewElts;
797792
798793 if (Input >= array_lengthof(Inputs)) {
799794 // The mask element does not index into any input vector.
800 Ops.push_back(DAG.getUNDEF(IdxVT));
795 Ops.push_back(-1);
801796 continue;
802797 }
803798
825820 }
826821
827822 // Add the mask index for the new shuffle vector.
828 Ops.push_back(DAG.getConstant(Idx + OpNo * NewElts, IdxVT));
823 Ops.push_back(Idx + OpNo * NewElts);
829824 }
830825
831826 if (useBuildVector) {
832827 MVT EltVT = NewVT.getVectorElementType();
833 Ops.clear();
828 SmallVector SVOps;
834829
835830 // Extract the input elements by hand.
836831 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
837 SDValue Arg = Mask.getOperand(FirstMaskIdx + MaskOffset);
838
839832 // The mask element. This indexes into the input.
840 unsigned Idx = Arg.getOpcode() == ISD::UNDEF ?
841 -1U : cast(Arg)->getZExtValue();
833 int Idx = SVN->getMaskElt(FirstMaskIdx + MaskOffset);
842834
843835 // The input vector this mask element indexes into.
844 unsigned Input = Idx / NewElts;
836 unsigned Input = (unsigned)Idx / NewElts;
845837
846838 if (Input >= array_lengthof(Inputs)) {
847839 // The mask element is "undef" or indexes off the end of the input.
848 Ops.push_back(DAG.getUNDEF(EltVT));
840 SVOps.push_back(DAG.getUNDEF(EltVT));
849841 continue;
850842 }
851843
853845 Idx -= Input * NewElts;
854846
855847 // Extract the vector element by hand.
856 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
857 Inputs[Input], DAG.getIntPtrConstant(Idx)));
848 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
849 Inputs[Input], DAG.getIntPtrConstant(Idx)));
858850 }
859851
860852 // Construct the Lo/Hi output using a BUILD_VECTOR.
861 Output = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, &Ops[0], Ops.size());
853 Output = DAG.getNode(ISD::BUILD_VECTOR,dl,NewVT, &SVOps[0], SVOps.size());
862854 } else if (InputUsed[0] == -1U) {
863855 // No input vectors were used! The result is undefined.
864856 Output = DAG.getUNDEF(NewVT);
865857 } else {
866 // At least one input vector was used. Create a new shuffle vector.
867 SDValue NewMask = DAG.getNode(ISD::BUILD_VECTOR, dl,
868 MVT::getVectorVT(IdxVT, Ops.size()),
869 &Ops[0], Ops.size());
870858 SDValue Op0 = Inputs[InputUsed[0]];
871859 // If only one input was used, use an undefined vector for the other.
872860 SDValue Op1 = InputUsed[1] == -1U ?
873861 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
874 Output = DAG.getNode(ISD::VECTOR_SHUFFLE, dl, NewVT, Op0, Op1, NewMask);
862 // At least one input vector was used. Create a new shuffle vector.
863 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]);
875864 }
876865
877866 Ops.clear();
14721461
14731462 if (NumOperands == 2) {
14741463 // Replace concat of two operands with a shuffle.
1475 MVT PtrVT = TLI.getPointerTy();
1476 SmallVector<SDValue, 16> MaskOps(WidenNumElts);
1464 SmallVector<int, 16> MaskOps(WidenNumElts);
14771465 for (unsigned i=0; i < WidenNumElts/2; ++i) {
1478 MaskOps[i] = DAG.getConstant(i, PtrVT);
1479 MaskOps[i+WidenNumElts/2] = DAG.getConstant(i+WidenNumElts, PtrVT);
1466 MaskOps[i] = i;
1467 MaskOps[i+WidenNumElts/2] = i+WidenNumElts;
14801468 }
1481 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, dl,
1482 MVT::getVectorVT(PtrVT, WidenNumElts),
1483 &MaskOps[0], WidenNumElts);
1484 return DAG.getNode(ISD::VECTOR_SHUFFLE, dl, WidenVT,
1485 GetWidenedVector(N->getOperand(0)),
1486 GetWidenedVector(N->getOperand(1)), Mask);
1469 return DAG.getVectorShuffle(WidenVT, dl,
1470 GetWidenedVector(N->getOperand(0)),
1471 GetWidenedVector(N->getOperand(1)),
1472 &MaskOps[0]);
14871473 }
14881474 }
14891475 }
17601746 }
17611747
17621748 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(SDNode *N) {
1749 ShuffleVectorSDNode *SVN = cast(N);
17631750 MVT VT = N->getValueType(0);
1764 unsigned NumElts = VT.getVectorNumElements();
1751 int NumElts = VT.getVectorNumElements();
17651752 DebugLoc dl = N->getDebugLoc();
17661753
17671754 MVT WidenVT = TLI.getTypeToTransformTo(VT);
17711758 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
17721759
17731760 // Adjust mask based on new input vector length.
1774 SDValue Mask = N->getOperand(2);
1775 SmallVector MaskOps(WidenNumElts);
1776 MVT IdxVT = Mask.getValueType().getVectorElementType();
1777 for (unsigned i = 0; i < NumElts; ++i) {
1778 SDValue Arg = Mask.getOperand(i);
1779 if (Arg.getOpcode() == ISD::UNDEF)
1780 MaskOps[i] = Arg;
1781 else {
1782 unsigned Idx = cast(Arg)->getZExtValue();
1783 if (Idx < NumElts)
1784 MaskOps[i] = Arg;
1785 else
1786 MaskOps[i] = DAG.getConstant(Idx - NumElts + WidenNumElts, IdxVT);
1787 }
1761 SmallVector NewMask;
1762 for (int i = 0; i < NumElts; ++i) {
1763 int Idx = SVN->getMaskElt(i);
1764 if (Idx < NumElts)
1765 NewMask.push_back(Idx);
1766 else
1767 NewMask.push_back(Idx - NumElts + WidenNumElts);
17881768 }
17891769 for (unsigned i = NumElts; i < WidenNumElts; ++i)
1790 MaskOps[i] = DAG.getUNDEF(IdxVT);
1791 SDValue NewMask = DAG.getNode(ISD::BUILD_VECTOR, dl,
1792 MVT::getVectorVT(IdxVT, WidenNumElts),
1793 &MaskOps[0], WidenNumElts);
1794
1795 return DAG.getNode(ISD::VECTOR_SHUFFLE, dl, WidenVT, InOp1, InOp2, NewMask);
1770 NewMask.push_back(-1);
1771 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]);
17961772 }
17971773
17981774 SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) {
455455 ID.AddInteger(AT->getRawSubclassData());
456456 break;
457457 }
458 case ISD::VECTOR_SHUFFLE: {
459 const ShuffleVectorSDNode *SVN = cast(N);
460 for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
461 i != e; ++i)
462 ID.AddInteger(SVN->getMaskElt(i));
463 break;
464 }
458465 } // end switch (N->getOpcode())
459466 }
460467
764771 MVT EltVT = N->getValueType(0).getVectorElementType();
765772 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I)
766773 assert((I->getValueType() == EltVT ||
767 (EltVT.isInteger() && I->getValueType().isInteger() &&
768 EltVT.bitsLE(I->getValueType()))) &&
769 "Wrong operand type!");
774 (EltVT.isInteger() && I->getValueType().isInteger() &&
775 EltVT.bitsLE(I->getValueType()))) &&
776 "Wrong operand type!");
770777 break;
771778 }
772779 }
11231130 AllNodes.push_back(N);
11241131 }
11251132 return SDValue(CondCodeNodes[Cond], 0);
1133 }
1134
1135 static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl &M) {
1136 std::swap(N1, N2);
1137 int NElts = M.size();
1138 for (int i = 0; i != NElts; ++i) {
1139 if (M[i] >= NElts)
1140 M[i] -= NElts;
1141 else if (M[i] >= 0)
1142 M[i] += NElts;
1143 }
1144 }
1145
1146 SDValue SelectionDAG::getVectorShuffle(MVT VT, DebugLoc dl, SDValue N1,
1147 SDValue N2, const int *Mask) {
1148 assert(N1.getValueType() == N2.getValueType() && "Invalid VECTOR_SHUFFLE");
1149 assert(VT.isVector() && N1.getValueType().isVector() &&
1150 "Vector Shuffle VTs must be a vectors");
1151 assert(VT.getVectorElementType() == N1.getValueType().getVectorElementType()
1152 && "Vector Shuffle VTs must have same element type");
1153
1154 // Canonicalize shuffle undef, undef -> undef
1155 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF)
1156 return N1;
1157
1158 // Validate that all indices in Mask are within the range of the elements
1159 // input to the shuffle.
1160 int NElts = VT.getVectorNumElements();
1161 SmallVector MaskVec;
1162 for (int i = 0; i != NElts; ++i) {
1163 if (Mask[i] >= (NElts * 2)) {
1164 assert(0 && "Index out of range");
1165 return SDValue();
1166 }
1167 MaskVec.push_back(Mask[i]);
1168 }
1169
1170 // Canonicalize shuffle v, v -> v, undef
1171 if (N1 == N2) {
1172 N2 = getUNDEF(VT);
1173 for (int i = 0; i != NElts; ++i)
1174 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
1175 }
1176
1177 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
1178 if (N1.getOpcode() == ISD::UNDEF)
1179 commuteShuffle(N1, N2, MaskVec);
1180
1181 // Canonicalize all index into lhs, -> shuffle lhs, undef
1182 // Canonicalize all index into rhs, -> shuffle rhs, undef
1183 bool AllLHS = true, AllRHS = true;
1184 bool N2Undef = N2.getOpcode() == ISD::UNDEF;
1185 for (int i = 0; i != NElts; ++i) {
1186 if (MaskVec[i] >= NElts) {
1187 if (N2Undef)
1188 MaskVec[i] = -1;
1189 else
1190 AllLHS = false;
1191 } else if (MaskVec[i] >= 0) {
1192 AllRHS = false;
1193 }
1194 }
1195 if (AllLHS && AllRHS)
1196 return getUNDEF(VT);
1197 if (AllLHS)
1198 N2 = getUNDEF(VT);
1199 if (AllRHS) {
1200 N1 = getUNDEF(VT);
1201 commuteShuffle(N1, N2, MaskVec);
1202 }
1203
1204 // If Identity shuffle, or all shuffle in to undef, return that node.
1205 bool AllUndef = true;
1206 bool Identity = true;
1207 for (int i = 0; i < NElts; ++i) {
1208 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
1209 if (MaskVec[i] >= 0) AllUndef = false;
1210 }
1211 if (Identity)
1212 return N1;
1213 if (AllUndef)
1214 return getUNDEF(VT);
1215
1216 FoldingSetNodeID ID;
1217 SDValue Ops[2] = { N1, N2 };
1218 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops, 2);
1219 for (int i = 0; i != NElts; ++i)
1220 ID.AddInteger(MaskVec[i]);
1221
1222 void* IP = 0;
1223 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1224 return SDValue(E, 0);
1225
1226 // Allocate the mask array for the node out of the BumpPtrAllocator, since
1227 // SDNode doesn't have access to it. This memory will be "leaked" when
1228 // the node is deallocated, but recovered when the NodeAllocator is released.
1229 int *MaskAlloc = OperandAllocator.Allocate(NElts);
1230 memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int));
1231
1232 ShuffleVectorSDNode *N = NodeAllocator.Allocate();
1233 new (N) ShuffleVectorSDNode(VT, dl, N1, N2, MaskAlloc);
1234 CSEMap.InsertNode(N, IP);
1235 AllNodes.push_back(N);
1236 return SDValue(N, 0);
11261237 }
11271238
11281239 SDValue SelectionDAG::getConvertRndSat(MVT VT, DebugLoc dl,
20862197 SDValue SelectionDAG::getShuffleScalarElt(const SDNode *N, unsigned i) {
20872198 MVT VT = N->getValueType(0);
20882199 DebugLoc dl = N->getDebugLoc();
2089 SDValue PermMask = N->getOperand(2);
2090 SDValue Idx = PermMask.getOperand(i);
2091 if (Idx.getOpcode() == ISD::UNDEF)
2200 const ShuffleVectorSDNode *SVN = cast(N);
2201 int Index = SVN->getMaskElt(i);
2202 if (Index < 0)
20922203 return getUNDEF(VT.getVectorElementType());
2093 unsigned Index = cast(Idx)->getZExtValue();
2094 unsigned NumElems = PermMask.getNumOperands();
2204 int NumElems = VT.getVectorNumElements();
20952205 SDValue V = (Index < NumElems) ? N->getOperand(0) : N->getOperand(1);
20962206 Index %= NumElems;
20972207
20982208 if (V.getOpcode() == ISD::BIT_CONVERT) {
20992209 V = V.getOperand(0);
21002210 MVT VVT = V.getValueType();
2101 if (!VVT.isVector() || VVT.getVectorNumElements() != NumElems)
2211 if (!VVT.isVector() || VVT.getVectorNumElements() != (unsigned)NumElems)
21022212 return SDValue();
21032213 }
21042214 if (V.getOpcode() == ISD::SCALAR_TO_VECTOR)
27932903 }
27942904 break;
27952905 case ISD::VECTOR_SHUFFLE:
2796 assert(N1.getValueType() == N2.getValueType() &&
2797 N1.getValueType().isVector() &&
2798 VT.isVector() && N3.getValueType().isVector() &&
2799 N3.getOpcode() == ISD::BUILD_VECTOR &&
2800 VT.getVectorNumElements() == N3.getNumOperands() &&
2801 "Illegal VECTOR_SHUFFLE node!");
2906 assert(0 && "should use getVectorShuffle constructor!");
28022907 break;
28032908 case ISD::BIT_CONVERT:
28042909 // Fold bit_convert nodes from a type to themselves.
53225427
53235428 void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const {
53245429 if (!isTargetOpcode() && getOpcode() == ISD::VECTOR_SHUFFLE) {
5325 SDNode *Mask = getOperand(2).getNode();
5430 const ShuffleVectorSDNode *SVN = cast(this);
53265431 OS << "<";
5327 for (unsigned i = 0, e = Mask->getNumOperands(); i != e; ++i) {
5432 for (unsigned i = 0, e = ValueList[0].getVectorNumElements(); i != e; ++i) {
5433 int Idx = SVN->getMaskElt(i);
53285434 if (i) OS << ",";
5329 if (Mask->getOperand(i).getOpcode() == ISD::UNDEF)
5435 if (Idx < 0)
53305436 OS << "u";
53315437 else
5332 OS << cast(Mask->getOperand(i))->getZExtValue();
5438 OS << Idx;
53335439 }
53345440 OS << ">";
53355441 }
56105716 SplatBitSize = sz;
56115717 return true;
56125718 }
5719
5720 bool ShuffleVectorSDNode::isSplatMask(const int *Mask, MVT VT) {
5721 int Idx = -1;
5722 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
5723 if (Idx < 0) Idx = Mask[i];
5724 if (Mask[i] >= 0 && Mask[i] != Idx)
5725 return false;
5726 }
5727 return true;
5728 }
869869 if (ConstantFP *CFP = dyn_cast(C))
870870 return N = DAG.getConstantFP(*CFP, VT);
871871
872 if (isa(C) && !isa(V->getType()) &&
873 !V->getType()->isAggregateType())
872 if (isa(C) && !V->getType()->isAggregateType())
874873 return N = DAG.getUNDEF(VT);
875874
876875 if (ConstantExpr *CE = dyn_cast(C)) {
924923 for (unsigned i = 0; i != NumElements; ++i)
925924 Ops.push_back(getValue(CP->getOperand(i)));
926925 } else {
927 assert((isa(C) || isa(C)) &&
928 "Unknown vector constant!");
926 assert(isa(C) && "Unknown vector constant!");
929927 MVT EltVT = TLI.getValueType(VecTy->getElementType());
930928
931929 SDValue Op;
932 if (isa(C))
933 Op = DAG.getUNDEF(EltVT);
934 else if (EltVT.isFloatingPoint())
930 if (EltVT.isFloatingPoint())
935931 Op = DAG.getConstantFP(0, EltVT);
936932 else
937933 Op = DAG.getConstant(0, EltVT);
24342430
24352431 // Utility for visitShuffleVector - Returns true if the mask is mask starting
24362432 // from SIndx and increasing to the element length (undefs are allowed).
2437 static bool SequentialMask(SDValue Mask, unsigned SIndx) {
2438 unsigned MaskNumElts = Mask.getNumOperands();
2439 for (unsigned i = 0; i != MaskNumElts; ++i) {
2440 if (Mask.getOperand(i).getOpcode() != ISD::UNDEF) {
2441 unsigned Idx = cast(Mask.getOperand(i))->getZExtValue();
2442 if (Idx != i + SIndx)
2443 return false;
2444 }
2445 }
2433 static bool SequentialMask(SmallVectorImpl &Mask, int SIndx) {
2434 int MaskNumElts = Mask.size();
2435 for (int i = 0; i != MaskNumElts; ++i)
2436 if ((Mask[i] >= 0) && (Mask[i] != i + SIndx))
2437 return false;
24462438 return true;
24472439 }
24482440
24492441 void SelectionDAGLowering::visitShuffleVector(User &I) {
2442 SmallVector Mask;
24502443 SDValue Src1 = getValue(I.getOperand(0));
24512444 SDValue Src2 = getValue(I.getOperand(1));
2452 SDValue Mask = getValue(I.getOperand(2));
2453
2445
2446 // Convert the ConstantVector mask operand into an array of ints, with -1
2447 // representing undef values.
2448 SmallVector MaskElts;
2449 cast(I.getOperand(2))->getVectorElements(MaskElts);
2450 int MaskNumElts = MaskElts.size();
2451 for (int i = 0; i != MaskNumElts; ++i) {
2452 if (isa(MaskElts[i]))
2453 Mask.push_back(-1);
2454 else
2455 Mask.push_back(cast(MaskElts[i])->getSExtValue());
2456 }
2457
24542458 MVT VT = TLI.getValueType(I.getType());
24552459 MVT SrcVT = Src1.getValueType();
2456 int MaskNumElts = Mask.getNumOperands();
24572460 int SrcNumElts = SrcVT.getVectorNumElements();
24582461
24592462 if (SrcNumElts == MaskNumElts) {
2460 setValue(&I, DAG.getNode(ISD::VECTOR_SHUFFLE, getCurDebugLoc(),
2461 VT, Src1, Src2, Mask));
2463 setValue(&I, DAG.getVectorShuffle(VT, getCurDebugLoc(), Src1, Src2,
2464 &Mask[0]));
24622465 return;
24632466 }
24642467
24652468 // Normalize the shuffle vector since mask and vector length don't match.
2466 MVT MaskEltVT = Mask.getValueType().getVectorElementType();
2467
24682469 if (SrcNumElts < MaskNumElts && MaskNumElts % SrcNumElts == 0) {
24692470 // Mask is longer than the source vectors and is a multiple of the source
24702471 // vectors. We can use concatenate vector to make the mask and vectors
24782479
24792480 // Pad both vectors with undefs to make them the same length as the mask.
24802481 unsigned NumConcat = MaskNumElts / SrcNumElts;
2482 bool Src1U = Src1.getOpcode() == ISD::UNDEF;
2483 bool Src2U = Src2.getOpcode() == ISD::UNDEF;
24812484 SDValue UndefVal = DAG.getUNDEF(SrcVT);
24822485
2483 SDValue* MOps1 = new SDValue[NumConcat];
2484 SDValue* MOps2 = new SDValue[NumConcat];
2486 SmallVector MOps1(NumConcat, UndefVal);
2487 SmallVector MOps2(NumConcat, UndefVal);
24852488 MOps1[0] = Src1;
24862489 MOps2[0] = Src2;
2487 for (unsigned i = 1; i != NumConcat; ++i) {
2488 MOps1[i] = UndefVal;
2489 MOps2[i] = UndefVal;
2490 }
2491 Src1 = DAG.getNode(ISD::CONCAT_VECTORS, getCurDebugLoc(),
2492 VT, MOps1, NumConcat);
2493 Src2 = DAG.getNode(ISD::CONCAT_VECTORS, getCurDebugLoc(),
2494 VT, MOps2, NumConcat);
2495
2496 delete [] MOps1;
2497 delete [] MOps2;
2490
2491 Src1 = Src1U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
2492 getCurDebugLoc(), VT,
2493 &MOps1[0], NumConcat);
2494 Src2 = Src2U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
2495 getCurDebugLoc(), VT,
2496 &MOps2[0], NumConcat);
24982497
24992498 // Readjust mask for new input vector length.
2500 SmallVector<SDValue, 8> MappedOps;
2499 SmallVector<int, 8> MappedOps;
25012500 for (int i = 0; i != MaskNumElts; ++i) {
2502 if (Mask.getOperand(i).getOpcode() == ISD::UNDEF) {
2503 MappedOps.push_back(Mask.getOperand(i));
2504 } else {
2505 int Idx = cast(Mask.getOperand(i))->getZExtValue();
2506 if (Idx < SrcNumElts)
2507 MappedOps.push_back(DAG.getConstant(Idx, MaskEltVT));
2508 else
2509 MappedOps.push_back(DAG.getConstant(Idx + MaskNumElts - SrcNumElts,
2510 MaskEltVT));
2511 }
2512 }
2513 Mask = DAG.getNode(ISD::BUILD_VECTOR, getCurDebugLoc(),
2514 Mask.getValueType(),
2515 &MappedOps[0], MappedOps.size());
2516
2517 setValue(&I, DAG.getNode(ISD::VECTOR_SHUFFLE, getCurDebugLoc(),
2518 VT, Src1, Src2, Mask));
2501 int Idx = Mask[i];
2502 if (Idx < SrcNumElts)
2503 MappedOps.push_back(Idx);
2504 else
2505 MappedOps.push_back(Idx + MaskNumElts - SrcNumElts);
2506 }
2507 setValue(&I, DAG.getVectorShuffle(VT, getCurDebugLoc(), Src1, Src2,
2508 &MappedOps[0]));
25192509 return;
25202510 }
25212511
25402530 int MaxRange[2] = {-1, -1};
25412531
25422532 for (int i = 0; i != MaskNumElts; ++i) {
2543 SDValue Arg = Mask.getOperand(i);
2544 if (Arg.getOpcode() != ISD::UNDEF) {
2545 assert(isa(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2546 int Idx = cast(Arg)->getZExtValue();
2547 int Input = 0;
2548 if (Idx >= SrcNumElts) {
2549 Input = 1;
2550 Idx -= SrcNumElts;
2551 }
2552 if (Idx > MaxRange[Input])
2553 MaxRange[Input] = Idx;
2554 if (Idx < MinRange[Input])
2555 MinRange[Input] = Idx;
2556 }
2533 int Idx = Mask[i];
2534 int Input = 0;
2535 if (Idx < 0)
2536 continue;
2537
2538 if (Idx >= SrcNumElts) {
2539 Input = 1;
2540 Idx -= SrcNumElts;
2541 }
2542 if (Idx > MaxRange[Input])
2543 MaxRange[Input] = Idx;
2544 if (Idx < MinRange[Input])
2545 MinRange[Input] = Idx;
25572546 }
25582547
25592548 // Check if the access is smaller than the vector size and can we find
25952584 }
25962585 }
25972586 // Calculate new mask.
2598 SmallVector<SDValue, 8> MappedOps;
2587 SmallVector<int, 8> MappedOps;
25992588 for (int i = 0; i != MaskNumElts; ++i) {
2600 SDValue Arg = Mask.getOperand(i);
2601 if (Arg.getOpcode() == ISD::UNDEF) {
2602 MappedOps.push_back(Arg);
2603 } else {
2604 int Idx = cast(Arg)->getZExtValue();
2605 if (Idx < SrcNumElts)
2606 MappedOps.push_back(DAG.getConstant(Idx - StartIdx[0], MaskEltVT));
2607 else {
2608 Idx = Idx - SrcNumElts - StartIdx[1] + MaskNumElts;
2609 MappedOps.push_back(DAG.getConstant(Idx, MaskEltVT));
2610 }
2611 }
2612 }
2613 Mask = DAG.getNode(ISD::BUILD_VECTOR, getCurDebugLoc(),
2614 Mask.getValueType(),
2615 &MappedOps[0], MappedOps.size());
2616 setValue(&I, DAG.getNode(ISD::VECTOR_SHUFFLE, getCurDebugLoc(),
2617 VT, Src1, Src2, Mask));
2589 int Idx = Mask[i];
2590 if (Idx < 0)
2591 MappedOps.push_back(Idx);
2592 else if (Idx < SrcNumElts)
2593 MappedOps.push_back(Idx - StartIdx[0]);
2594 else
2595 MappedOps.push_back(Idx - SrcNumElts - StartIdx[1] + MaskNumElts);
2596 }
2597 setValue(&I, DAG.getVectorShuffle(VT, getCurDebugLoc(), Src1, Src2,
2598 &MappedOps[0]));
26182599 return;
26192600 }
26202601 }
26262607 MVT PtrVT = TLI.getPointerTy();
26272608 SmallVector Ops;
26282609 for (int i = 0; i != MaskNumElts; ++i) {
2629 SDValue Arg = Mask.getOperand(i);
2630 if (Arg.getOpcode() == ISD::UNDEF) {
2610 if (Mask[i] < 0) {
26312611 Ops.push_back(DAG.getUNDEF(EltVT));
26322612 } else {
2633 assert(isa(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2634 int Idx = cast(Arg)->getZExtValue();
2613 int Idx = Mask[i];
26352614 if (Idx < SrcNumElts)
26362615 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurDebugLoc(),
26372616 EltVT, Src1, DAG.getConstant(Idx, PtrVT)));
16691669 /// \note
16701670 /// SPUISD::SHUFB is eventually selected as Cell's shufb instructions.
16711671 static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) {
1672 const ShuffleVectorSDNode *SVN = cast(Op);
16721673 SDValue V1 = Op.getOperand(0);
16731674 SDValue V2 = Op.getOperand(1);
1674 SDValue PermMask = Op.getOperand(2);
16751675 DebugLoc dl = Op.getDebugLoc();
16761676
16771677 if (V2.getOpcode() == ISD::UNDEF) V2 = V1;
17021702 } else
17031703 assert(0 && "Unhandled vector type in LowerVECTOR_SHUFFLE");
17041704
1705 for (unsigned i = 0; i != PermMask.getNumOperands(); ++i) {
1706 if (PermMask.getOperand(i).getOpcode() != ISD::UNDEF) {
1707 unsigned SrcElt = cast (PermMask.getOperand(i))->getZExtValue();
1708
1709 if (monotonic) {
1710 if (SrcElt >= V2EltIdx0) {
1711 if (1 >= (++EltsFromV2)) {
1712 V2Elt = (V2EltIdx0 - SrcElt) << 2;
1713 }
1714 } else if (CurrElt != SrcElt) {
1715 monotonic = false;
1705 for (unsigned i = 0; i != MaxElts; ++i) {
1706 if (SVN->getMaskElt(i) < 0)
1707 continue;
1708
1709 unsigned SrcElt = SVN->getMaskElt(i);
1710
1711 if (monotonic) {
1712 if (SrcElt >= V2EltIdx0) {
1713 if (1 >= (++EltsFromV2)) {
1714 V2Elt = (V2EltIdx0 - SrcElt) << 2;
17161715 }
1717
1718 ++CurrElt;
1716 } else if (CurrElt != SrcElt) {
1717 monotonic = false;
17191718 }
17201719
1721 if (rotate) {
1722 if (PrevElt > 0 && SrcElt < MaxElts) {
1723 if ((PrevElt == SrcElt - 1)
1724 || (PrevElt == MaxElts - 1 && SrcElt == 0)) {
1725 PrevElt = SrcElt;
1726 if (SrcElt == 0)
1727 V0Elt = i;
1728 } else {
1729 rotate = false;
1730 }
1731 } else if (PrevElt == 0) {
1732 // First time through, need to keep track of previous element
1720 ++CurrElt;
1721 }
1722
1723 if (rotate) {
1724 if (PrevElt > 0 && SrcElt < MaxElts) {
1725 if ((PrevElt == SrcElt - 1)
1726 || (PrevElt == MaxElts - 1 && SrcElt == 0)) {
17331727 PrevElt = SrcElt;
1728 if (SrcElt == 0)
1729 V0Elt = i;
17341730 } else {
1735 // This isn't a rotation, takes elements from vector 2
17361731 rotate = false;
17371732 }
1733 } else if (PrevElt == 0) {
1734 // First time through, need to keep track of previous element
1735 PrevElt = SrcElt;
1736 } else {
1737 // This isn't a rotation, takes elements from vector 2
1738 rotate = false;
17381739 }
17391740 }
17401741 }
17671768 unsigned BytesPerElement = EltVT.getSizeInBits()/8;
17681769
17691770 SmallVector ResultMask;
1770 for (unsigned i = 0, e = PermMask.getNumOperands(); i != e; ++i) {
1771 unsigned SrcElt;
1772 if (PermMask.getOperand(i).getOpcode() == ISD::UNDEF)
1773 SrcElt = 0;
1774 else
1775 SrcElt = cast(PermMask.getOperand(i))->getZExtValue();
1776
1777 for (unsigned j = 0; j < BytesPerElement; ++j) {
1778 ResultMask.push_back(DAG.getConstant(SrcElt*BytesPerElement+j,
1779 MVT::i8));
1780 }
1771 for (unsigned i = 0, e = MaxElts; i != e; ++i) {
1772 unsigned SrcElt = SVN->getMaskElt(i) < 0 ? 0 : SVN->getMaskElt(i);
1773
1774 for (unsigned j = 0; j < BytesPerElement; ++j)
1775 ResultMask.push_back(DAG.getConstant(SrcElt*BytesPerElement+j,MVT::i8));
17811776 }
17821777
17831778 SDValue VPermMask = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i8,
455455
456456 /// isConstantOrUndef - Op is either an undef node or a ConstantSDNode. Return
457457 /// true if Op is undef or if it matches the specified value.
458 static bool isConstantOrUndef(SDValue Op, unsigned Val) {
459 return Op.getOpcode() == ISD::UNDEF ||
460 cast(Op)->getZExtValue() == Val;
458 static bool isConstantOrUndef(int Op, int Val) {
459 return Op < 0 || Op == Val;
461460 }
462461
463462 /// isVPKUHUMShuffleMask - Return true if this is the shuffle mask for a
464463 /// VPKUHUM instruction.
465 bool PPC::isVPKUHUMShuffleMask(SDNode *N, bool isUnary) {
464 bool PPC::isVPKUHUMShuffleMask(ShuffleVectorSDNode *N, bool isUnary) {
466465 if (!isUnary) {
467466 for (unsigned i = 0; i != 16; ++i)
468 if (!isConstantOrUndef(N->getOperand(i), i*2+1))
467 if (!isConstantOrUndef(N->getMaskElt(i), i*2+1))
469468 return false;
470469 } else {
471470 for (unsigned i = 0; i != 8; ++i)
472 if (!isConstantOrUndef(N->getOperand(i), i*2+1) ||
473 !isConstantOrUndef(N->getOperand(i+8), i*2+1))
471 if (!isConstantOrUndef(N->getMaskElt(i), i*2+1) ||
472 !isConstantOrUndef(N->getMaskElt(i+8), i*2+1))
474473 return false;
475474 }
476475 return true;
478477
479478 /// isVPKUWUMShuffleMask - Return true if this is the shuffle mask for a
480479 /// VPKUWUM instruction.
481 bool PPC::isVPKUWUMShuffleMask(SDNode *N, bool isUnary) {
480 bool PPC::isVPKUWUMShuffleMask(ShuffleVectorSDNode *N, bool isUnary) {
482481 if (!isUnary) {
483482 for (unsigned i = 0; i != 16; i += 2)
484 if (!isConstantOrUndef(N->getOperand(i ), i*2+2) ||
485 !isConstantOrUndef(N->getOperand(i+1), i*2+3))
483 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+2) ||
484 !isConstantOrUndef(N->getMaskElt(i+1), i*2+3))
486485 return false;
487486 } else {
488487 for (unsigned i = 0; i != 8; i += 2)
489 if (!isConstantOrUndef(N->getOperand(i ), i*2+2) ||
490 !isConstantOrUndef(N->getOperand(i+1), i*2+3) ||
491 !isConstantOrUndef(N->getOperand(i+8), i*2+2) ||
492 !isConstantOrUndef(N->getOperand(i+9), i*2+3))
488 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+2) ||
489 !isConstantOrUndef(N->getMaskElt(i+1), i*2+3) ||
490 !isConstantOrUndef(N->getMaskElt(i+8), i*2+2) ||
491 !isConstantOrUndef(N->getMaskElt(i+9), i*2+3))
493492 return false;
494493 }
495494 return true;
497496
498497 /// isVMerge - Common function, used to match vmrg* shuffles.
499498 ///
500 static bool isVMerge(SDNode *N, unsigned UnitSize,
499 static bool isVMerge(ShuffleVectorSDNode *N, unsigned UnitSize,
501500 unsigned LHSStart, unsigned RHSStart) {
502 assert(N->getOpcode() == ISD::BUILD_VECTOR &&
503 N->getNumOperands() == 16 && "PPC only supports shuffles by bytes!");
501 assert(N->getValueType(0) == MVT::v16i8 &&
502 "PPC only supports shuffles by bytes!");
504503 assert((UnitSize == 1 || UnitSize == 2 || UnitSize == 4) &&
505504 "Unsupported merge size!");
506505
507506 for (unsigned i = 0; i != 8/UnitSize; ++i) // Step over units
508507 for (unsigned j = 0; j != UnitSize; ++j) { // Step over bytes within unit
509 if (!isConstantOrUndef(N->getOperand(i*UnitSize*2+j),
508 if (!isConstantOrUndef(N->getMaskElt(i*UnitSize*2+j),
510509 LHSStart+j+i*UnitSize) ||
511 !isConstantOrUndef(N->getOperand(i*UnitSize*2+UnitSize+j),
510 !isConstantOrUndef(N->getMaskElt(i*UnitSize*2+UnitSize+j),
512511 RHSStart+j+i*UnitSize))
513512 return false;
514513 }
515 return true;
514 return true;
516515 }
517516
518517 /// isVMRGLShuffleMask - Return true if this is a shuffle mask suitable for
519518 /// a VRGL* instruction with the specified unit size (1,2 or 4 bytes).
520 bool PPC::isVMRGLShuffleMask(SDNode *N, unsigned UnitSize, bool isUnary) {
519 bool PPC::isVMRGLShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
520 bool isUnary) {
521521 if (!isUnary)
522522 return isVMerge(N, UnitSize, 8, 24);
523523 return isVMerge(N, UnitSize, 8, 8);
525525
526526 /// isVMRGHShuffleMask - Return true if this is a shuffle mask suitable for
527527 /// a VRGH* instruction with the specified unit size (1,2 or 4 bytes).
528 bool PPC::isVMRGHShuffleMask(SDNode *N, unsigned UnitSize, bool isUnary) {
528 bool PPC::isVMRGHShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
529 bool isUnary) {
529530 if (!isUnary)
530531 return isVMerge(N, UnitSize, 0, 16);
531532 return isVMerge(N, UnitSize, 0, 0);
535536 /// isVSLDOIShuffleMask - If this is a vsldoi shuffle mask, return the shift
536537 /// amount, otherwise return -1.
537538 int PPC::isVSLDOIShuffleMask(SDNode *N, bool isUnary) {
538 assert(N->getOpcode() == ISD::BUILD_VECTOR &&
539 N->getNumOperands() == 16 && "PPC only supports shuffles by bytes!");
539 assert(N->getValueType(0) == MVT::v16i8 &&
540 "PPC only supports shuffles by bytes!");
541
542 ShuffleVectorSDNode *SVOp = cast(N);
543
540544 // Find the first non-undef value in the shuffle mask.
541545 unsigned i;
542 for (i = 0; i != 16 && N->getOperand(i).getOpcode() == ISD::UNDEF; ++i)
546 for (i = 0; i != 16 && SVOp->getMaskElt(i) < 0; ++i)
543547 /*search*/;
544548
545549 if (i == 16) return -1; // all undef.
546550
547 // Otherwise, check to see if the rest of the elements are consequtively
551 // Otherwise, check to see if the rest of the elements are consecutively
548552 // numbered from this value.
549 unsigned ShiftAmt = cast(N->getOperand(i))->getZExtValue();
553 unsigned ShiftAmt = SVOp->getMaskElt(i);
550554 if (ShiftAmt < i) return -1;
551555 ShiftAmt -= i;
552556
553557 if (!isUnary) {
554 // Check the rest of the elements to see if they are consequtive.
558 // Check the rest of the elements to see if they are consecutive.
555559 for (++i; i != 16; ++i)
556 if (!isConstantOrUndef(N->getOperand(i), ShiftAmt+i))
560 if (!isConstantOrUndef(SVOp->getMaskElt(i), ShiftAmt+i))
557561 return -1;
558562 } else {
559 // Check the rest of the elements to see if they are consequtive.
563 // Check the rest of the elements to see if they are consecutive.
560564 for (++i; i != 16; ++i)
561 if (!isConstantOrUndef(N->getOperand(i), (ShiftAmt+i) & 15))
565 if (!isConstantOrUndef(SVOp->getMaskElt(i), (ShiftAmt+i) & 15))
562566 return -1;
563567 }
564
565568 return ShiftAmt;
566569 }
567570
568571 /// isSplatShuffleMask - Return true if the specified VECTOR_SHUFFLE operand
569572 /// specifies a splat of a single element that is suitable for input to
570573 /// VSPLTB/VSPLTH/VSPLTW.
571 bool PPC::isSplatShuffleMask(SDNode *N, unsigned EltSize) {
572 assert(N->getOpcode() == ISD::BUILD_VECTOR &&
573 N->getNumOperands() == 16 &&
574 bool PPC::isSplatShuffleMask(ShuffleVectorSDNode *N, unsigned EltSize) {
575 assert(N->getValueType(0) == MVT::v16i8 &&
574576 (EltSize == 1 || EltSize == 2 || EltSize == 4));
575577
576578 // This is a splat operation if each element of the permute is the same, and
577579 // if the value doesn't reference the second vector.
578 unsigned ElementBase = 0;
579 SDValue Elt = N->getOperand(0);
580 if (ConstantSDNode *EltV = dyn_cast(Elt))
581 ElementBase = EltV->getZExtValue();
582 else
583 return false; // FIXME: Handle UNDEF elements too!
584
585 if (cast(Elt)->getZExtValue() >= 16)
580 unsigned ElementBase = N->getMaskElt(0);
581
582 // FIXME: Handle UNDEF elements too!
583 if (ElementBase >= 16)
586584 return false;
587585
588 // Check that they are consequtive.
589 for (unsigned i = 1; i != EltSize; ++i) {
590 if (!isa(N->getOperand(i)) ||
591 cast(N->getOperand(i))->getZExtValue() != i+ElementBase)
586 // Check that the indices are consecutive, in the case of a multi-byte element
587 // splatted with a v16i8 mask.
588 for (unsigned i = 1; i != EltSize; ++i)
589 if (N->getMaskElt(i) < 0 || N->getMaskElt(i) != (int)(i+ElementBase))
592590 return false;
593 }
594
595 assert(isa(Elt) && "Invalid VECTOR_SHUFFLE mask!");
591
596592 for (unsigned i = EltSize, e = 16; i != e; i += EltSize) {
597 if (N->getOperand(i).getOpcode() == ISD::UNDEF) continue;
598 assert(isa(N->getOperand(i)) &&
599 "Invalid VECTOR_SHUFFLE mask!");
593 if (N->getMaskElt(i) < 0) continue;
600594 for (unsigned j = 0; j != EltSize; ++j)
601 if (N->getOperand(i+j) != N->getOperand(j))
595 if (N->getMaskElt(i+j) != N->getMaskElt(j))
602596 return false;
603597 }
604
605598 return true;
606599 }
607600
608601 /// isAllNegativeZeroVector - Returns true if all elements of build_vector
609602 /// are -0.0.
610603 bool PPC::isAllNegativeZeroVector(SDNode *N) {
611 assert(N->getOpcode() == ISD::BUILD_VECTOR);
612 if (PPC::isSplatShuffleMask(N, N->getNumOperands()))
613 if (ConstantFPSDNode *CFP = dyn_cast(N))
604 BuildVectorSDNode *BV = cast(N);
605
606 APInt APVal, APUndef;
607 unsigned BitSize;
608 bool HasAnyUndefs;
609
610 if (BV->isConstantSplat(APVal, APUndef, BitSize, HasAnyUndefs, 32))
611 if (ConstantFPSDNode *CFP = dyn_cast(N->getOperand(0)))
614612 return CFP->getValueAPF().isNegZero();
613
615614 return false;
616615 }
617616
618617 /// getVSPLTImmediate - Return the appropriate VSPLT* immediate to splat the
619618 /// specified isSplatShuffleMask VECTOR_SHUFFLE mask.
620619 unsigned PPC::getVSPLTImmediate(SDNode *N, unsigned EltSize) {
621 assert(isSplatShuffleMask(N, EltSize));
622 return cast(N->getOperand(0))->getZExtValue() / EltSize;
620 ShuffleVectorSDNode *SVOp = cast(N);
621 assert(isSplatShuffleMask(SVOp, EltSize));
622 return SVOp->getMaskElt(0) / EltSize;
623623 }
624624
625625 /// get_VSPLTI_elt - If this is a build_vector of constants which can be formed
31483148 LHS = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v16i8, LHS);
31493149 RHS = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v16i8, RHS);
31503150
3151 SDValue Ops[16];
3151 int Ops[16];
31523152 for (unsigned i = 0; i != 16; ++i)
3153 Ops[i] = DAG.getConstant(i+Amt, MVT::i8);
3154 SDValue T = DAG.getNode(ISD::VECTOR_SHUFFLE, dl, MVT::v16i8, LHS, RHS,
3155 DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i8, Ops,16));
3153 Ops[i] = i + Amt;
3154 SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, LHS, RHS, Ops);
31563155 return DAG.getNode(ISD::BIT_CONVERT, dl, VT, T);
31573156 }
31583157
33533352 OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
33543353 OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl);
33553354
3356 unsigned ShufIdxs[16];
3355 int ShufIdxs[16];
33573356 switch (OpNum) {
33583357 default: assert(0 && "Unknown i32 permute!");
33593358 case OP_VMRGHW:
33913390 case OP_VSLDOI12:
33923391 return BuildVSLDOI(OpLHS, OpRHS, 12, OpLHS.getValueType(), DAG, dl);
33933392 }
3394 SDValue Ops[16];
3395 for (unsigned i = 0; i != 16; ++i)
3396 Ops[i] = DAG.getConstant(ShufIdxs[i], MVT::i8);
3397
3398 return DAG.getNode(ISD::VECTOR_SHUFFLE, dl, OpLHS.getValueType(),
3399 OpLHS, OpRHS,
3400 DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i8, Ops, 16));
3393 MVT VT = OpLHS.getValueType();
3394 OpLHS = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v16i8, OpLHS);
3395 OpRHS = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v16i8, OpRHS);
3396 SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, OpLHS, OpRHS, ShufIdxs);
3397 return DAG.getNode(ISD::BIT_CONVERT, dl, VT, T);
34013398 }
34023399
34033400 /// LowerVECTOR_SHUFFLE - Return the code we lower for VECTOR_SHUFFLE. If this
34053402 /// return the code it can be lowered into. Worst case, it can always be
34063403 /// lowered into a vperm.
34073404 SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
3408 SelectionDAG &DAG) {
3405 SelectionDAG &DAG) {
34093406 DebugLoc dl = Op.getDebugLoc();
34103407 SDValue V1 = Op.getOperand(0);
34113408 SDValue V2 = Op.getOperand(1);
3412 SDValue PermMask = Op.getOperand(2);
3409 ShuffleVectorSDNode *SVOp = cast(Op);
3410 MVT VT = Op.getValueType();
34133411
34143412 // Cases that are handled by instructions that take permute immediates
34153413 // (such as vsplt*) should be left as VECTOR_SHUFFLE nodes so they can be
34163414 // selected by the instruction selector.
34173415 if (V2.getOpcode() == ISD::UNDEF) {
3418 if (PPC::isSplatShuffleMask(PermMask.getNode(), 1) ||
3419 PPC::isSplatShuffleMask(PermMask.getNode(), 2) ||
3420 PPC::isSplatShuffleMask(PermMask.getNode(), 4) ||
3421 PPC::isVPKUWUMShuffleMask(PermMask.getNode(), true) ||
3422 PPC::isVPKUHUMShuffleMask(PermMask.getNode(), true) ||
3423 PPC::isVSLDOIShuffleMask(PermMask.getNode(), true) != -1 ||
3424 PPC::isVMRGLShuffleMask(PermMask.getNode(), 1, true) ||
3425 PPC::isVMRGLShuffleMask(PermMask.getNode(), 2, true) ||
3426 PPC::isVMRGLShuffleMask(PermMask.getNode(), 4, true) ||
3427 PPC::isVMRGHShuffleMask(PermMask.getNode(), 1, true) ||
3428 PPC::isVMRGHShuffleMask(PermMask.getNode(), 2, true) ||
3429 PPC::isVMRGHShuffleMask(PermMask.getNode(), 4, true)) {
3416 if (PPC::isSplatShuffleMask(SVOp, 1) ||
3417 PPC::isSplatShuffleMask(SVOp, 2) ||
3418 PPC::isSplatShuffleMask(SVOp, 4) ||
3419 PPC::isVPKUWUMShuffleMask(SVOp, true) ||
3420 PPC::isVPKUHUMShuffleMask(SVOp, true) ||
3421 PPC::isVSLDOIShuffleMask(SVOp, true) != -1 ||
3422 PPC::isVMRGLShuffleMask(SVOp, 1, true) ||
3423 PPC::isVMRGLShuffleMask(SVOp, 2, true) ||
3424 PPC::isVMRGLShuffleMask(SVOp, 4, true) ||
3425 PPC::isVMRGHShuffleMask(SVOp, 1, true) ||
3426 PPC::isVMRGHShuffleMask(SVOp, 2, true) ||
3427 PPC::isVMRGHShuffleMask(SVOp, 4, true)) {
34303428 return Op;
34313429 }
34323430 }
34343432 // Altivec has a variety of "shuffle immediates" that take two vector inputs
34353433 // and produce a fixed permutation. If any of these match, do not lower to
34363434 // VPERM.
3437 if (PPC::isVPKUWUMShuffleMask(PermMask.getNode(), false) ||
3438 PPC::isVPKUHUMShuffleMask(PermMask.getNode(), false) ||
3439 PPC::isVSLDOIShuffleMask(PermMask.getNode(), false) != -1 ||
3440 PPC::isVMRGLShuffleMask(PermMask.getNode(), 1, false) ||
3441 PPC::isVMRGLShuffleMask(PermMask.getNode(), 2, false) ||
3442 PPC::isVMRGLShuffleMask(PermMask.getNode(), 4, false) ||
3443 PPC::isVMRGHShuffleMask(PermMask.getNode(), 1, false) ||
3444 PPC::isVMRGHShuffleMask(PermMask.getNode(), 2, false) ||
3445 PPC::isVMRGHShuffleMask(PermMask.getNode(), 4, false))
3435 if (PPC::isVPKUWUMShuffleMask(SVOp, false) ||
3436 PPC::isVPKUHUMShuffleMask(SVOp, false) ||
3437 PPC::isVSLDOIShuffleMask(SVOp, false) != -1 ||
3438 PPC::isVMRGLShuffleMask(SVOp, 1, false) ||
3439 PPC::isVMRGLShuffleMask(SVOp, 2, false) ||
3440 PPC::isVMRGLShuffleMask(SVOp, 4, false) ||
3441 PPC::isVMRGHShuffleMask(SVOp, 1, false) ||
3442 PPC::isVMRGHShuffleMask(SVOp, 2, false) ||
3443 PPC::isVMRGHShuffleMask(SVOp, 4, false))
34463444 return Op;
34473445
34483446 // Check to see if this is a shuffle of 4-byte values. If so, we can use our
34493447 // perfect shuffle table to emit an optimal matching sequence.
3448 SmallVector PermMask;
3449 SVOp->getMask(PermMask);
3450
34503451 unsigned PFIndexes[4];
34513452 bool isFourElementShuffle = true;
34523453 for (unsigned i = 0; i != 4 && isFourElementShuffle; ++i) { // Element number
34533454 unsigned EltNo = 8; // Start out undef.
34543455 for (unsigned j = 0; j != 4; ++j) { // Intra-element byte.
3455 if (PermMask.getOperand(i*4+j).getOpcode() == ISD::UNDEF)
3456 if (PermMask[i*4+j] < 0)
34563457 continue; // Undef, ignore it.
34573458
3458 unsigned ByteSource =
3459 cast(PermMask.getOperand(i*4+j))->getZExtValue();
3459 unsigned ByteSource = PermMask[i*4+j];
34603460 if ((ByteSource & 3) != j) {
34613461 isFourElementShuffle = false;
34623462 break;
35083508 unsigned BytesPerElement = EltVT.getSizeInBits()/8;
35093509
35103510 SmallVector ResultMask;
3511 for (unsigned i = 0, e = PermMask.getNumOperands(); i != e; ++i) {
3512 unsigned SrcElt;
3513 if (PermMask.getOperand(i).getOpcode() == ISD::UNDEF)
3514 SrcElt = 0;
3515 else
3516 SrcElt = cast(PermMask.getOperand(i))->getZExtValue();
3511 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
3512 unsigned SrcElt = PermMask[i] < 0 ? 0 : PermMask[i];
35173513
35183514 for (unsigned j = 0; j != BytesPerElement; ++j)
35193515 ResultMask.push_back(DAG.getConstant(SrcElt*BytesPerElement+j,
37033699 OddParts = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v16i8, OddParts);
37043700
37053701 // Merge the results together.
3706 SDValue Ops[16];
3702 int Ops[16];
37073703 for (unsigned i = 0; i != 8; ++i) {
3708 Ops[i*2 ] = DAG.getConstant(2*i+1, MVT::i8);
3709 Ops[i*2+1] = DAG.getConstant(2*i+1+16, MVT::i8);
3710 }
3711 return DAG.getNode(ISD::VECTOR_SHUFFLE, dl, MVT::v16i8, EvenParts, OddParts,
3712 DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i8, Ops, 16));
3704 Ops[i*2 ] = 2*i+1;
3705 Ops[i*2+1] = 2*i+1+16;
3706 }
3707 return DAG.getVectorShuffle(MVT::v16i8, dl, EvenParts, OddParts, Ops);
37133708 } else {
37143709 assert(0 && "Unknown mul to lower!");
37153710 abort();
174174 namespace PPC {
175175 /// isVPKUHUMShuffleMask - Return true if this is the shuffle mask for a
176176 /// VPKUHUM instruction.
177 bool isVPKUHUMShuffleMask(SDNode *N, bool isUnary);
177 bool isVPKUHUMShuffleMask(ShuffleVectorSDNode *N, bool isUnary);
178178
179179 /// isVPKUWUMShuffleMask - Return true if this is the shuffle mask for a
180180 /// VPKUWUM instruction.
181 bool isVPKUWUMShuffleMask(SDNode *N, bool isUnary);
181 bool isVPKUWUMShuffleMask(ShuffleVectorSDNode *N, bool isUnary);
182182
183183 /// isVMRGLShuffleMask - Return true if this is a shuffle mask suitable for
184184 /// a VRGL* instruction with the specified unit size (1,2 or 4 bytes).
185 bool isVMRGLShuffleMask(SDNode *N, unsigned UnitSize, bool isUnary);
185 bool isVMRGLShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
186 bool isUnary);
186187
187188 /// isVMRGHShuffleMask - Return true if this is a shuffle mask suitable for
188189 /// a VRGH* instruction with the specified unit size (1,2 or 4 bytes).
189 bool isVMRGHShuffleMask(SDNode *N, unsigned UnitSize, bool isUnary);
190 bool isVMRGHShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
191 bool isUnary);
190192
191193 /// isVSLDOIShuffleMask - If this is a vsldoi shuffle mask, return the shift
192194 /// amount, otherwise return -1.
195197 /// isSplatShuffleMask - Return true if the specified VECTOR_SHUFFLE operand
196198 /// specifies a splat of a single element that is suitable for input to
197199 /// VSPLTB/VSPLTH/VSPLTW.
198 bool isSplatShuffleMask(SDNode *N, unsigned EltSize);
200 bool isSplatShuffleMask(ShuffleVectorSDNode *N, unsigned EltSize);
199201
200202 /// isAllNegativeZeroVector - Returns true if all elements of build_vector
201203 /// are -0.0.
1414 // Altivec transformation functions and pattern fragments.
1515 //
1616
17 /// VPKUHUM_shuffle_mask/VPKUWUM_shuffle_mask - Return true if this is a valid
18 /// shuffle mask for the VPKUHUM or VPKUWUM instructions.
19 def VPKUHUM_shuffle_mask : PatLeaf<(build_vector), [{
20 return PPC::isVPKUHUMShuffleMask(N, false);
21 }]>;
22 def VPKUWUM_shuffle_mask : PatLeaf<(build_vector), [{
23 return PPC::isVPKUWUMShuffleMask(N, false);
24 }]>;
25
26 def VPKUHUM_unary_shuffle_mask : PatLeaf<(build_vector), [{
27 return PPC::isVPKUHUMShuffleMask(N, true);
28 }]>;
29 def VPKUWUM_unary_shuffle_mask : PatLeaf<(build_vector), [{
30 return PPC::isVPKUWUMShuffleMask(N, true);
31 }]>;
32
33
34 def VMRGLB_shuffle_mask : PatLeaf<(build_vector), [{
35 return PPC::isVMRGLShuffleMask(N, 1, false);
36 }]>;
37 def VMRGLH_shuffle_mask : PatLeaf<(build_vector), [{
38 return PPC::isVMRGLShuffleMask(N, 2, false);
39 }]>;
40 def VMRGLW_shuffle_mask : PatLeaf<(build_vector), [{
41 return PPC::isVMRGLShuffleMask(N, 4, false);
42 }]>;
43 def VMRGHB_shuffle_mask : PatLeaf<(build_vector), [{
44 return PPC::isVMRGHShuffleMask(N, 1, false);
45 }]>;
46 def VMRGHH_shuffle_mask : PatLeaf<(build_vector), [{
47 return PPC::isVMRGHShuffleMask(N, 2, false);
48 }]>;
49 def VMRGHW_shuffle_mask : PatLeaf<(build_vector), [{
50 return PPC::isVMRGHShuffleMask(N, 4, false);
51 }]>;
52
53 def VMRGLB_unary_shuffle_mask : PatLeaf<(build_vector), [{
54 return PPC::isVMRGLShuffleMask(N, 1, true);
55 }]>;
56 def VMRGLH_unary_shuffle_mask : PatLeaf<(build_vector), [{
57 return PPC::isVMRGLShuffleMask(N, 2, true);
58 }]>;
59 def VMRGLW_unary_shuffle_mask : PatLeaf<(build_vector), [{
60 return PPC::isVMRGLShuffleMask(N, 4, true);
61 }]>;
62 def VMRGHB_unary_shuffle_mask : PatLeaf<(build_vector), [{
63 return PPC::isVMRGHShuffleMask(N, 1, true);
64 }]>;
65 def VMRGHH_unary_shuffle_mask : PatLeaf<(build_vector), [{
66 return PPC::isVMRGHShuffleMask(N, 2, true);
67 }]>;
68 def VMRGHW_unary_shuffle_mask : PatLeaf<(build_vector), [{
69 return PPC::isVMRGHShuffleMask(N, 4, true);
70 }]>;
71
72 def VSLDOI_get_imm : SDNodeXForm
17
18 def vpkuhum_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
19 (vector_shuffle node:$lhs, node:$rhs), [{
20 return PPC::isVPKUHUMShuffleMask(cast(N), false);
21 }]>;
22 def vpkuwum_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
23 (vector_shuffle node:$lhs, node:$rhs), [{
24 return PPC::isVPKUWUMShuffleMask(cast(N), false);
25 }]>;
26 def vpkuhum_unary_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
27 (vector_shuffle node:$lhs, node:$rhs), [{
28 return PPC::isVPKUHUMShuffleMask(cast(N), true);
29 }]>;
30 def vpkuwum_unary_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
31 (vector_shuffle node:$lhs, node:$rhs), [{
32 return PPC::isVPKUWUMShuffleMask(cast(N), true);
33 }]>;
34
35
36 def vmrglb_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
37 (vector_shuffle node:$lhs, node:$rhs), [{
38 return PPC::isVMRGLShuffleMask(cast(N), 1, false);
39 }]>;
40 def vmrglh_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
41 (vector_shuffle node:$lhs, node:$rhs), [{
42 return PPC::isVMRGLShuffleMask(cast(N), 2, false);
43 }]>;
44 def vmrglw_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
45 (vector_shuffle node:$lhs, node:$rhs), [{
46 return PPC::isVMRGLShuffleMask(cast(N), 4, false);
47 }]>;
48 def vmrghb_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
49 (vector_shuffle node:$lhs, node:$rhs), [{
50 return PPC::isVMRGHShuffleMask(cast(N), 1, false);
51 }]>;
52 def vmrghh_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
53 (vector_shuffle node:$lhs, node:$rhs), [{
54 return PPC::isVMRGHShuffleMask(cast(N), 2, false);
55 }]>;
56 def vmrghw_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
57 (vector_shuffle node:$lhs, node:$rhs), [{
58 return PPC::isVMRGHShuffleMask(cast(N), 4, false);
59 }]>;
60
61
62 def vmrglb_unary_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
63 (vector_shuffle node:$lhs, node:$rhs), [{
64 return PPC::isVMRGLShuffleMask(cast(N), 1, true);
65 }]>;
66 def vmrglh_unary_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
67 (vector_shuffle node:$lhs, node:$rhs), [{
68 return PPC::isVMRGLShuffleMask(cast(N), 2, true);
69 }]>;
70 def vmrglw_unary_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
71 (vector_shuffle node:$lhs, node:$rhs), [{
72 return PPC::isVMRGLShuffleMask(cast(N), 4, true);
73 }]>;
74 def vmrghb_unary_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
75 (vector_shuffle node:$lhs, node:$rhs), [{
76 return PPC::isVMRGHShuffleMask(cast(N), 1, true);
77 }]>;
78 def vmrghh_unary_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
79 (vector_shuffle node:$lhs, node:$rhs), [{
80 return PPC::isVMRGHShuffleMask(cast(N), 2, true);
81 }]>;
82 def vmrghw_unary_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
83 (vector_shuffle node:$lhs, node:$rhs), [{
84 return PPC::isVMRGHShuffleMask(cast(N), 4, true);
85 }]>;
86
87
88 def VSLDOI_get_imm : SDNodeXForm
7389 return getI32Imm(PPC::isVSLDOIShuffleMask(N, false));
7490 }]>;
75 def VSLDOI_shuffle_mask : PatLeaf<(build_vector), [{
91 def vsldoi_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
92 (vector_shuffle node:$lhs, node:$rhs), [{
7693 return PPC::isVSLDOIShuffleMask(N, false) != -1;
7794 }], VSLDOI_get_imm>;
7895
96
7997 /// VSLDOI_unary* - These are used to match vsldoi(X,X), which is turned into
8098 /// vector_shuffle(X,undef,mask) by the dag combiner.
81 def VSLDOI_unary_get_imm : SDNodeXForm<build_vector, [{
99 def VSLDOI_unary_get_imm : SDNodeXForm<vector_shuffle, [{
82100 return getI32Imm(PPC::isVSLDOIShuffleMask(N, true));
83101 }]>;
84 def VSLDOI_unary_shuffle_mask : PatLeaf<(build_vector), [{
102 def vsldoi_unary_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
103 (vector_shuffle node:$lhs, node:$rhs), [{
85104 return PPC::isVSLDOIShuffleMask(N, true) != -1;
86105 }], VSLDOI_unary_get_imm>;
87106
88107
89108 // VSPLT*_get_imm xform function: convert vector_shuffle mask to VSPLT* imm.
90 def VSPLTB_get_imm : SDNodeXForm<build_vector, [{
109 def VSPLTB_get_imm : SDNodeXForm<vector_shuffle, [{
91110 return getI32Imm(PPC::getVSPLTImmediate(N, 1));
92111 }]>;
93 def VSPLTB_shuffle_mask : PatLeaf<(build_vector), [{
94 return PPC::isSplatShuffleMask(N, 1);
112 def vspltb_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
113 (vector_shuffle node:$lhs, node:$rhs), [{
114 return PPC::isSplatShuffleMask(cast(N), 1);
95115 }], VSPLTB_get_imm>;
96 def VSPLTH_get_imm : SDNodeXForm<build_vector, [{
116 def VSPLTH_get_imm : SDNodeXForm<vector_shuffle, [{
97117 return getI32Imm(PPC::getVSPLTImmediate(N, 2));
98118 }]>;
99 def VSPLTH_shuffle_mask : PatLeaf<(build_vector), [{
100 return PPC::isSplatShuffleMask(N, 2);
119 def vsplth_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
120 (vector_shuffle node:$lhs, node:$rhs), [{
121 return PPC::isSplatShuffleMask(cast(N), 2);
101122 }], VSPLTH_get_imm>;
102 def VSPLTW_get_imm : SDNodeXForm<build_vector, [{
123 def VSPLTW_get_imm : SDNodeXForm<vector_shuffle, [{
103124 return getI32Imm(PPC::getVSPLTImmediate(N, 4));
104125 }]>;
105 def VSPLTW_shuffle_mask : PatLeaf<(build_vector), [{
106 return PPC::isSplatShuffleMask(N, 4);
126 def vspltw_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
127 (vector_shuffle node:$lhs, node:$rhs), [{
128 return PPC::isSplatShuffleMask(cast(N), 4);
107129 }], VSPLTW_get_imm>;
108130
109131
267289 def VSLDOI : VAForm_2<44, (outs VRRC:$vD), (ins VRRC:$vA, VRRC:$vB, u5imm:$SH),
268290 "vsldoi $vD, $vA, $vB, $SH", VecFP,
269291 [(set VRRC:$vD,
270 (vector_shuffle (v16i8 VRRC:$vA), VRRC:$vB,
271 VSLDOI_shuffle_mask:$SH))]>;
292 (vsldoi_shuffle:$SH (v16i8 VRRC:$vA), VRRC:$vB))]>;
272293
273294 // VX-Form instructions. AltiVec arithmetic ops.
274295 def VADDFP : VXForm_1<10, (outs VRRC:$vD), (ins VRRC:$vA, VRRC:$vB),
344365
345366 def VMRGHB : VXForm_1< 12, (outs VRRC:$vD), (ins VRRC:$vA, VRRC:$vB),
346367 "vmrghb $vD, $vA, $vB", VecFP,
347 [(set VRRC:$vD, (vector_shuffle (v16i8 VRRC:$vA),
348 VRRC:$vB, VMRGHB_shuffle_mask))]>;
368 [(set VRRC:$vD, (vmrghb_shuffle VRRC:$vA, VRRC:$vB))]>;
349369 def VMRGHH : VXForm_1< 76, (outs VRRC:$vD), (ins VRRC:$vA, VRRC:$vB),
350370 "vmrghh $vD, $vA, $vB", VecFP,
351 [(set VRRC:$vD, (vector_shuffle (v16i8 VRRC:$vA),
352 VRRC:$vB, VMRGHH_shuffle_mask))]>;
371 [(set VRRC:$vD, (vmrghh_shuffle VRRC:$vA, VRRC:$vB))]>;
353372 def VMRGHW : VXForm_1<140, (outs VRRC:$vD), (ins VRRC:$vA, VRRC:$vB),
354373 "vmrghw $vD, $vA, $vB", VecFP,
355 [(set VRRC:$vD, (vector_shuffle (v16i8 VRRC:$vA),
356 VRRC:$vB, VMRGHW_shuffle_mask))]>;
374 [(set VRRC:$vD, (vmrghw_shuffle VRRC:$vA, VRRC:$vB))]>;
357375 def VMRGLB : VXForm_1<268, (outs VRRC:$vD), (ins VRRC:$vA, VRRC:$vB),
358376 "vmrglb $vD, $vA, $vB", VecFP,
359 [(set VRRC:$vD, (vector_shuffle (v16i8 VRRC:$vA),
360 VRRC:$vB, VMRGLB_shuffle_mask))]>;
377 [(set VRRC:$vD, (vmrglb_shuffle VRRC:$vA, VRRC:$vB))]>;
361378 def VMRGLH : VXForm_1<332, (outs VRRC:$vD), (ins VRRC:$vA, VRRC:$vB),
362379 "vmrglh $vD, $vA, $vB", VecFP,
363 [(set VRRC:$vD, (vector_shuffle (v16i8 VRRC:$vA),
364 VRRC:$vB, VMRGLH_shuffle_mask))]>;
380 [(set VRRC:$vD, (vmrglh_shuffle VRRC:$vA, VRRC:$vB))]>;
365381 def VMRGLW : VXForm_1<396, (outs VRRC:$vD), (ins VRRC:$vA, VRRC:$vB),
366382 "vmrglw $vD, $vA, $vB", VecFP,
367 [(set VRRC:$vD, (vector_shuffle (v16i8 VRRC:$vA),
368 VRRC:$vB, VMRGLW_shuffle_mask))]>;
383 [(set VRRC:$vD, (vmrglw_shuffle VRRC:$vA, VRRC:$vB))]>;
369384
370385 def VMSUMMBM : VA1a_Int<37, "vmsummbm", int_ppc_altivec_vmsummbm>;
371386 def VMSUMSHM : VA1a_Int<40, "vmsumshm", int_ppc_altivec_vmsumshm>;
439454
440455 def VSPLTB : VXForm_1<524, (outs VRRC:$vD), (ins u5imm:$UIMM, VRRC:$vB),
441456 "vspltb $vD, $vB, $UIMM", VecPerm,
442 [(set VRRC:$vD, (vector_shuffle (v16i8 VRRC:$vB), (undef),
443 VSPLTB_shuffle_mask:$UIMM))]>;
457 [(set VRRC:$vD,
458 (vspltb_shuffle:$UIMM (v16i8 VRRC:$vB), (undef)))]>;
444459 def VSPLTH : VXForm_1<588, (outs VRRC:$vD), (ins u5imm:$UIMM, VRRC:$vB),
445460 "vsplth $vD, $vB, $UIMM", VecPerm,
446 [(set VRRC:$vD, (vector_shuffle (v16i8 VRRC:$vB), (undef),
447 VSPLTH_shuffle_mask:$UIMM))]>;
461 [(set VRRC:$vD,
462 (vsplth_shuffle:$UIMM (v16i8 VRRC:$vB), (undef)))]>;
448463 def VSPLTW : VXForm_1<652, (outs VRRC:$vD), (ins u5imm:$UIMM, VRRC:$vB),
449464 "vspltw $vD, $vB, $UIMM", VecPerm,
450 [(set VRRC:$vD, (vector_shuffle (v16i8 VRRC:$vB), (undef),
451 VSPLTW_shuffle_mask:$UIMM))]>;
465 [(set VRRC:$vD,
466 (vspltw_shuffle:$UIMM (v16i8 VRRC:$vB), (undef)))]>;
452467
453468 def VSR : VX1_Int< 708, "vsr" , int_ppc_altivec_vsr>;
454469 def VSRO : VX1_Int<1100, "vsro" , int_ppc_altivec_vsro>;
478493 def VPKSWUS : VX1_Int<334, "vpkswus", int_ppc_altivec_vpkswus>;
479494 def VPKUHUM : VXForm_1<14, (outs VRRC:$vD), (ins VRRC:$vA, VRRC:$vB),
480495 "vpkuhum $vD, $vA, $vB", VecFP,
481 [(set VRRC:$vD, (vector_shuffle (v16i8 VRRC:$vA),
482 VRRC:$vB, VPKUHUM_shuffle_mask))]>;
496 [(set VRRC:$vD,
497 (vpkuhum_shuffle (v16i8 VRRC:$vA), VRRC:$vB))]>;
483498 def VPKUHUS : VX1_Int<142, "vpkuhus", int_ppc_altivec_vpkuhus>;
484499 def VPKUWUM : VXForm_1<78, (outs VRRC:$vD), (ins VRRC:$vA, VRRC:$vB),
485500 "vpkuwum $vD, $vA, $vB", VecFP,
486 [(set VRRC:$vD, (vector_shuffle (v16i8 VRRC:$vA),
487 VRRC:$vB, VPKUWUM_shuffle_mask))]>;
501 [(set VRRC:$vD,
502 (vpkuwum_shuffle (v16i8 VRRC:$vA), VRRC:$vB))]>;
488503 def VPKUWUS : VX1_Int<206, "vpkuwus", int_ppc_altivec_vpkuwus>;
489504
490505 // Vector Unpack.
602617 // Shuffles.
603618
604619 // Match vsldoi(x,x), vpkuwum(x,x), vpkuhum(x,x)
605 def:Pat<(vector_shuffle (v16i8 VRRC:$vA), undef, VSLDOI_unary_shuffle_mask:$in),
606 (VSLDOI VRRC:$vA, VRRC:$vA, VSLDOI_unary_shuffle_mask:$in)>;
607 def:Pat<(vector_shuffle (v16i8 VRRC:$vA), undef,VPKUWUM_unary_shuffle_mask:$in),
620 def:Pat<(vsldoi_unary_shuffle:$in (v16i8 VRRC:$vA), undef),
621 (VSLDOI VRRC:$vA, VRRC:$vA, (VSLDOI_unary_get_imm VRRC:$in))>;
622 def:Pat<(vpkuwum_unary_shuffle (v16i8 VRRC:$vA), undef),
608623 (VPKUWUM VRRC:$vA, VRRC:$vA)>;
609 def:Pat<(vector_shuffle (v16i8 VRRC:$vA), undef,VPKUHUM_unary_shuffle_mask:$in),
624 def:Pat<(vpkuhum_unary_shuffle (v16i8 VRRC:$vA), undef),
610625 (VPKUHUM VRRC:$vA, VRRC:$vA)>;
611626
612627 // Match vmrg*(x,x)
613 def:Pat<(vector_shuffle (v16i8 VRRC:$vA), undef, VMRGLB_unary_shuffle_mask:$in),
628 def:Pat<(vmrglb_unary_shuffle (v16i8 VRRC:$vA), undef),
614629 (VMRGLB VRRC:$vA, VRRC:$vA)>;
615 def:Pat<(vector_shuffle (v16i8 VRRC:$vA), undef, VMRGLH_unary_shuffle_mask:$in),
630 def:Pat<(vmrglh_unary_shuffle (v16i8 VRRC:$vA), undef),
616631 (VMRGLH VRRC:$vA, VRRC:$vA)>;
617 def:Pat<(vector_shuffle (v16i8 VRRC:$vA), undef, VMRGLW_unary_shuffle_mask:$in),
632 def:Pat<(vmrglw_unary_shuffle (v16i8 VRRC:$vA), undef),
618633 (VMRGLW VRRC:$vA, VRRC:$vA)>;
619 def:Pat<(vector_shuffle (v16i8 VRRC:$vA), undef, VMRGHB_unary_shuffle_mask:$in),
634 def:Pat<(vmrghb_unary_shuffle (v16i8 VRRC:$vA), undef),
620635 (VMRGHB VRRC:$vA, VRRC:$vA)>;
621 def:Pat<(vector_shuffle (v16i8 VRRC:$vA), undef, VMRGHH_unary_shuffle_mask:$in),
636 def:Pat<(vmrghh_unary_shuffle (v16i8 VRRC:$vA), undef),
622637 (VMRGHH VRRC:$vA, VRRC:$vA)>;
623 def:Pat<(vector_shuffle (v16i8 VRRC:$vA), undef, VMRGHW_unary_shuffle_mask:$in),
638 def:Pat<(vmrghw_unary_shuffle (v16i8 VRRC:$vA), undef),
624639 (VMRGHW VRRC:$vA, VRRC:$vA)>;
625640
626641 // Logical Operations
4444 DisableMMX("disable-mmx", cl::Hidden, cl::desc("Disable use of MMX"));
4545
4646 // Forward declarations.
47 static SDValue getMOVLMask(unsigned NumElems, SelectionDAG &DAG, DebugLoc dl);
47 static SDValue getMOVL(SelectionDAG &DAG, DebugLoc dl, MVT VT, SDValue V1,
48 SDValue V2);
4849
4950 X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
5051 : TargetLowering(TM) {
16661667 // Special case: passing MMX values in XMM registers.
16671668 Arg = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i64, Arg);
16681669 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Arg);
1669 Arg = DAG.getNode(ISD::VECTOR_SHUFFLE, dl, MVT::v2i64,
1670 DAG.getUNDEF(MVT::v2i64), Arg,
1671 getMOVLMask(2, DAG, dl));
1670 Arg = getMOVL(DAG, dl, MVT::v2i64, DAG.getUNDEF(MVT::v2i64), Arg);
16721671 break;
16731672 }
16741673 }
21372136 }
21382137 }
21392138
2140 /// isUndefOrInRange - Op is either an undef node or a ConstantSDNode. Return
2141 /// true if Op is undef or if its value falls within the specified range (L, H].
2142 static bool isUndefOrInRange(SDValue Op, unsigned Low, unsigned Hi) {
2143 if (Op.getOpcode() == ISD::UNDEF)
2139 /// isUndefOrInRange - Return true if Val is undef or if its value falls within
2140 /// the specified range (L, H].
2141 static bool isUndefOrInRange(int Val, int Low, int Hi) {
2142 return (Val < 0) || (Val >= Low && Val < Hi);
2143 }
2144
2145 /// isUndefOrEqual - Val is either less than zero (undef) or equal to the
2146 /// specified value.
2147 static bool isUndefOrEqual(int Val, int CmpVal) {
2148 if (Val < 0 || Val == CmpVal)
21442149 return true;
2145
2146 unsigned Val = cast(Op)->getZExtValue();
2147 return (Val >= Low && Val < Hi);
2148 }
2149
2150 /// isUndefOrEqual - Op is either an undef node or a ConstantSDNode. Return
2151 /// true if Op is undef or if its value equal to the specified value.
2152 static bool isUndefOrEqual(SDValue Op, unsigned Val) {
2153 if (Op.getOpcode() == ISD::UNDEF)
2154 return true;
2155 return cast(Op)->getZExtValue() == Val;
2156 }
2157
2158 /// isPSHUFDMask - Return true if the specified VECTOR_SHUFFLE operand
2159 /// specifies a shuffle of elements that is suitable for input to PSHUFD.
2160 bool X86::isPSHUFDMask(SDNode *N) {
2161 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2162
2163 if (N->getNumOperands() != 2 && N->getNumOperands() != 4)
2150 return false;
2151 }
2152
2153 /// isPSHUFDMask - Return true if the node specifies a shuffle of elements that
2154 /// is suitable for input to PSHUFD or PSHUFW. That is, it doesn't reference
2155 /// the second operand.
2156 static bool isPSHUFDMask(SmallVectorImpl &Mask, MVT VT) {
2157 if (VT == MVT::v4f32 || VT == MVT::v4i32 || VT == MVT::v4i16)
2158 return (Mask[0] < 4 && Mask[1] < 4 && Mask[2] < 4 && Mask[3] < 4);
2159 if (VT == MVT::v2f64 || VT == MVT::v2i64)
2160 return (Mask[0] < 2 && Mask[1] < 2);
2161 return false;
2162 }
2163
2164 bool X86::isPSHUFDMask(ShuffleVectorSDNode *N) {
2165 SmallVector M;
2166 N->getMask(M);
2167 return ::isPSHUFDMask(M, N->getValueType(0));
2168 }
2169
2170 /// isPSHUFHWMask - Return true if the node specifies a shuffle of elements that
2171 /// is suitable for input to PSHUFHW.
2172 static bool isPSHUFHWMask(SmallVectorImpl &Mask, MVT VT) {
2173 if (VT != MVT::v8i16)
21642174 return false;
2165
2166 // Check if the value doesn't reference the second vector.
2167 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
2168 SDValue Arg = N->getOperand(i);
2169 if (Arg.getOpcode() == ISD::UNDEF) continue;
2170 assert(isa(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2171 if (cast(Arg)->getZExtValue() >= e)
2175
2176 // Lower quadword copied in order or undef.
2177 for (int i = 0; i != 4; ++i)
2178 if (Mask[i] >= 0 && Mask[i] != i)
21722179 return false;
2173 }
2174
2180
2181 // Upper quadword shuffled.
2182 for (int i = 4; i != 8; ++i)
2183 if (Mask[i] >= 0 && (Mask[i] < 4 || Mask[i] > 7))
2184 return false;
2185
21752186 return true;
21762187 }
21772188
2178 /// isPSHUFHWMask - Return true if the specified VECTOR_SHUFFLE operand
2179 /// specifies a shuffle of elements that is suitable for input to PSHUFHW.
2180 bool X86::isPSHUFHWMask(SDNode *N) {
2181 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2182
2183 if (N->getNumOperands() != 8)
2189 bool X86::isPSHUFHWMask(ShuffleVectorSDNode *N) {
2190 SmallVector M;
2191 N->getMask(M);
2192 return ::isPSHUFHWMask(M, N->getValueType(0));
2193 }
2194
2195 /// isPSHUFLWMask - Return true if the node specifies a shuffle of elements that
2196 /// is suitable for input to PSHUFLW.
2197 static bool isPSHUFLWMask(SmallVectorImpl &Mask, MVT VT) {
2198 if (VT != MVT::v8i16)
21842199 return false;
2185
2186 // Lower quadword copied in order.
2187 for (unsigned i = 0; i != 4; ++i) {
2188 SDValue Arg = N->getOperand(i);
2189 if (Arg.getOpcode() == ISD::UNDEF) continue;
2190 assert(isa(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2191 if (cast(Arg)->getZExtValue() != i)
2200
2201 // Upper quadword copied in order.
2202 for (int i = 4; i != 8; ++i)
2203 if (Mask[i] >= 0 && Mask[i] != i)
21922204 return false;
2193 }
2194
2195 // Upper quadword shuffled.
2196 for (unsigned i = 4; i != 8; ++i) {
2197 SDValue Arg = N->getOperand(i);
2198 if (Arg.getOpcode() == ISD::UNDEF) continue;
2199 assert(isa(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2200 unsigned Val = cast(Arg)->getZExtValue();
2201 if (Val < 4 || Val > 7)
2205
2206 // Lower quadword shuffled.
2207 for (int i = 0; i != 4; ++i)
2208 if (Mask[i] >= 4)
22022209 return false;
2203 }
2204
2210
22052211 return true;
22062212 }
22072213
2208 /// isPSHUFLWMask - Return true if the specified VECTOR_SHUFFLE operand
2209 /// specifies a shuffle of elements that is suitable for input to PSHUFLW.
2210 bool X86::isPSHUFLWMask(SDNode *N) {
2211 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2212
2213 if (N->getNumOperands() != 8)
2214 return false;
2215
2216 // Upper quadword copied in order.
2217 for (unsigned i = 4; i != 8; ++i)
2218 if (!isUndefOrEqual(N->getOperand(i), i))
2219 return false;
2220
2221 // Lower quadword shuffled.
2222 for (unsigned i = 0; i != 4; ++i)
2223 if (!isUndefOrInRange(N->getOperand(i), 0, 4))
2224 return false;
2225
2226 return true;
2214 bool X86::isPSHUFLWMask(ShuffleVectorSDNode *N) {
2215 SmallVector M;
2216 N->getMask(M);
2217 return ::isPSHUFLWMask(M, N->getValueType(0));
22272218 }
22282219
22292220 /// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand
22302221 /// specifies a shuffle of elements that is suitable for input to SHUFP*.
2231 template
2232 static bool isSHUFPMask(SDOperand *Elems, unsigned NumElems) {
2233 if (NumElems != 2 && NumElems != 4) return false;
2234
2235 unsigned Half = NumElems / 2;
2236 for (unsigned i = 0; i < Half; ++i)
2237 if (!isUndefOrInRange(Elems[i], 0, NumElems))
2222 static bool isSHUFPMask(SmallVectorImpl &Mask, MVT VT) {
2223 int NumElems = VT.getVectorNumElements();
2224 if (NumElems != 2 && NumElems != 4)
2225 return false;
2226
2227 int Half = NumElems / 2;
2228 for (int i = 0; i < Half; ++i)
2229 if (!isUndefOrInRange(Mask[i], 0, NumElems))
22382230 return false;
2239 for (unsigned i = Half; i < NumElems; ++i)
2240 if (!isUndefOrInRange(Elems[i], NumElems, NumElems*2))
2231 for (int i = Half; i < NumElems; ++i)
2232 if (!isUndefOrInRange(Mask[i], NumElems, NumElems*2))
22412233 return false;
2242
2234
22432235 return true;
22442236 }
22452237
2246 bool X86::isSHUFPMask(SDNode *N) {
2247 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2248 return ::isSHUFPMask(N->op_begin(), N->getNumOperands());
2238 bool X86::isSHUFPMask(ShuffleVectorSDNode *N) {
2239 SmallVector M;
2240 N->getMask(M);
2241 return ::isSHUFPMask(M, N->getValueType(0));
22492242 }
22502243
22512244 /// isCommutedSHUFP - Returns true if the shuffle mask is exactly
22522245 /// the reverse of what x86 shuffles want. x86 shuffles requires the lower
22532246 /// half elements to come from vector 1 (which would equal the dest.) and
22542247 /// the upper half to come from vector 2.
2255 template
2256 static bool isCommutedSHUFP(SDOperand *Ops, unsigned NumOps) {
2257 if (NumOps != 2 && NumOps != 4) return false;
2258
2259 unsigned Half = NumOps / 2;
2260 for (unsigned i = 0; i < Half; ++i)
2261 if (!isUndefOrInRange(Ops[i], NumOps, NumOps*2))
2248 static bool isCommutedSHUFPMask(SmallVectorImpl &Mask, MVT VT) {
2249 int NumElems = VT.getVectorNumElements();
2250
2251 if (NumElems != 2 && NumElems != 4)
2252 return false;
2253
2254 int Half = NumElems / 2;
2255 for (int i = 0; i < Half; ++i)
2256 if (!isUndefOrInRange(Mask[i], NumElems, NumElems*2))
22622257 return false;
2263 for (unsigned i = Half; i < NumOps; ++i)
2264 if (!isUndefOrInRange(Ops[i], 0, NumOps))
2258 for (int i = Half; i < NumElems; ++i)
2259 if (!isUndefOrInRange(Mask[i], 0, NumElems))
22652260 return false;
22662261 return true;
22672262 }
22682263
2269 static bool isCommutedSHUFP(SDNode *N) {
2270 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2271 return isCommutedSHUFP(N->op_begin(), N->getNumOperands());
2264 static bool isCommutedSHUFP(ShuffleVectorSDNode *N) {
2265 SmallVector M;
2266 N->getMask(M);
2267 return isCommutedSHUFPMask(M, N->getValueType(0));
22722268 }
22732269
22742270 /// isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand
22752271 /// specifies a shuffle of elements that is suitable for input to MOVHLPS.
2276 bool X86::isMOVHLPSMask(SDNode *N) {
2277 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2278
2279 if (N->getNumOperands() != 4)
2272 bool X86::isMOVHLPSMask(ShuffleVectorSDNode *N) {
2273 if (N->getValueType(0).getVectorNumElements() != 4)
22802274 return false;
22812275
22822276 // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3
2283 return isUndefOrEqual(N->getOperand(0), 6) &&
2284 isUndefOrEqual(N->getOperand(1), 7) &&
2285 isUndefOrEqual(N->getOperand(2), 2) &&
2286 isUndefOrEqual(N->getOperand(3), 3);
2277 return isUndefOrEqual(N->getMaskElt(0), 6) &&
2278 isUndefOrEqual(N->getMaskElt(1), 7) &&
2279 isUndefOrEqual(N->getMaskElt(2), 2) &&
2280 isUndefOrEqual(N->getMaskElt(3), 3);
2281 }
2282
2283 /// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand
2284 /// specifies a shuffle of elements that is suitable for input to MOVLP{S|D}.
2285 bool X86::isMOVLPMask(ShuffleVectorSDNode *N) {
2286 unsigned NumElems = N->getValueType(0).getVectorNumElements();
2287
2288 if (NumElems != 2 && NumElems != 4)
2289 return false;
2290
2291 for (unsigned i = 0; i < NumElems/2; ++i)
2292 if (!isUndefOrEqual(N->getMaskElt(i), i + NumElems))
2293 return false;
2294
2295 for (unsigned i = NumElems/2; i < NumElems; ++i)
2296 if (!isUndefOrEqual(N->getMaskElt(i), i))
2297 return false;
2298
2299 return true;
2300 }
2301
2302 /// isMOVHPMask - Return true if the specified VECTOR_SHUFFLE operand
2303 /// specifies a shuffle of elements that is suitable for input to MOVHP{S|D}
2304 /// and MOVLHPS.
2305 bool X86::isMOVHPMask(ShuffleVectorSDNode *N) {
2306 unsigned NumElems = N->getValueType(0).getVectorNumElements();
2307
2308 if (NumElems != 2 && NumElems != 4)
2309 return false;
2310
2311 for (unsigned i = 0; i < NumElems/2; ++i)
2312 if (!isUndefOrEqual(N->getMaskElt(i), i))
2313 return false;
2314
2315 for (unsigned i = 0; i < NumElems/2; ++i)
2316 if (!isUndefOrEqual(N->getMaskElt(i + NumElems/2), i + NumElems))
2317 return false;
2318
2319 return true;
22872320 }
22882321
22892322 /// isMOVHLPS_v_undef_Mask - Special case of isMOVHLPSMask for canonical form
22902323 /// of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef,
22912324 /// <2, 3, 2, 3>
2292 bool X86::isMOVHLPS_v_undef_Mask(SDNode *N) {
2293 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2294
2295 if (N->getNumOperands() != 4)
2325 bool X86::isMOVHLPS_v_undef_Mask(ShuffleVectorSDNode *N) {
2326 unsigned NumElems = N->getValueType(0).getVectorNumElements();
2327
2328 if (NumElems != 4)
22962329 return false;
2297
2298 // Expect bit0 == 2, bit1 == 3, bit2 == 2, bit3 == 3
2299 return isUndefOrEqual(N->getOperand(0), 2) &&
2300 isUndefOrEqual(N->getOperand(1), 3) &&
2301 isUndefOrEqual(N->getOperand(2), 2) &&
2302 isUndefOrEqual(N->getOperand(3), 3);
2303 }
2304
2305 /// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand
2306 /// specifies a shuffle of elements that is suitable for input to MOVLP{S|D}.
2307 bool X86::isMOVLPMask(SDNode *N) {
2308 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2309
2310 unsigned NumElems = N->getNumOperands();
2311 if (NumElems != 2 && NumElems != 4)
2312 return false;
2313
2314 for (unsigned i = 0; i < NumElems/2; ++i)
2315 if (!isUndefOrEqual(N->getOperand(i), i + NumElems))
2316 return false;
2317
2318 for (unsigned i = NumElems/2; i < NumElems; ++i)
2319 if (!isUndefOrEqual(N->getOperand(i), i))
2320 return false;
2321
2322 return true;
2323 }
2324
2325 /// isMOVHPMask - Return true if the specified VECTOR_SHUFFLE operand
2326 /// specifies a shuffle of elements that is suitable for input to MOVHP{S|D}
2327 /// and MOVLHPS.
2328 bool X86::isMOVHPMask(SDNode *N) {
2329 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2330
2331 unsigned NumElems = N->getNumOperands();
2332 if (NumElems != 2 && NumElems != 4)
2333 return false;
2334
2335 for (unsigned i = 0; i < NumElems/2; ++i)
2336 if (!isUndefOrEqual(N->getOperand(i), i))
2337 return false;
2338
2339 for (unsigned i = 0; i < NumElems/2; ++i) {
2340 SDValue Arg = N->getOperand(i + NumElems/2);
2341 if (!isUndefOrEqual(Arg, i + NumElems))
2342 return false;
2343 }
2344
2345 return true;
2330
2331 return isUndefOrEqual(N->getMaskElt(0), 2) &&
2332 isUndefOrEqual(N->getMaskElt(1), 3) &&
2333 isUndefOrEqual(N->getMaskElt(2), 2) &&
2334 isUndefOrEqual(N->getMaskElt(3), 3);
23462335 }
23472336
23482337 /// isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand
23492338 /// specifies a shuffle of elements that is suitable for input to UNPCKL.
2350 template
2351 bool static isUNPCKLMask(SDOperand *Elts, unsigned NumElts,
2339 static bool isUNPCKLMask(SmallVectorImpl &Mask, MVT VT,
23522340 bool V2IsSplat = false) {
2341 int NumElts = VT.getVectorNumElements();
23532342 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
23542343 return false;
2355
2356 for (unsigned i = 0, j = 0; i != NumElts; i += 2, ++j) {
2357 SDValue BitI = Elts[i];
2358 SDValue BitI1 = Elts[i+1];
2344
2345 for (int i = 0, j = 0; i != NumElts; i += 2, ++j) {
2346 int BitI = Mask[i];
2347 int BitI1 = Mask[i+1];
23592348 if (!isUndefOrEqual(BitI, j))
23602349 return false;
23612350 if (V2IsSplat) {
23662355 return false;
23672356 }
23682357 }
2369
23702358 return true;
23712359 }
23722360
2373 bool X86::isUNPCKLMask(SDNode *N, bool V2IsSplat) {
2374 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2375 return ::isUNPCKLMask(N->op_begin(), N->getNumOperands(), V2IsSplat);
2361 bool X86::isUNPCKLMask(ShuffleVectorSDNode *N, bool V2IsSplat) {
2362 SmallVector M;
2363 N->getMask(M);
2364 return ::isUNPCKLMask(M, N->getValueType(0), V2IsSplat);
23762365 }
23772366
23782367 /// isUNPCKHMask - Return true if the specified VECTOR_SHUFFLE operand
23792368 /// specifies a shuffle of elements that is suitable for input to UNPCKH.
2380 template
2381 bool static isUNPCKHMask(SDOperand *Elts, unsigned NumElts,
2369 static bool isUNPCKHMask(SmallVectorImpl &Mask, MVT VT,
23822370 bool V2IsSplat = false) {
2371 int NumElts = VT.getVectorNumElements();
23832372 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
23842373 return false;
2385
2386 for (unsigned i = 0, j = 0; i != NumElts; i += 2, ++j) {
2387 SDValue BitI = Elts[i];
2388 SDValue BitI1 = Elts[i+1];
2374
2375 for (int i = 0, j = 0; i != NumElts; i += 2, ++j) {
2376 int BitI = Mask[i];
2377 int BitI1 = Mask[i+1];
23892378 if (!isUndefOrEqual(BitI, j + NumElts/2))
23902379 return false;
23912380 if (V2IsSplat) {
23962385 return false;
23972386 }
23982387 }
2399
24002388 return true;
24012389 }
24022390
2403 bool X86::isUNPCKHMask(SDNode *N, bool V2IsSplat) {
2404 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2405 return ::isUNPCKHMask(N->op_begin(), N->getNumOperands(), V2IsSplat);
2391 bool X86::isUNPCKHMask(ShuffleVectorSDNode *N, bool V2IsSplat) {
2392 SmallVector M;
2393 N->getMask(M);
2394 return ::isUNPCKHMask(M, N->getValueType(0), V2IsSplat);
24062395 }
24072396
24082397 /// isUNPCKL_v_undef_Mask - Special case of isUNPCKLMask for canonical form
24092398 /// of vector_shuffle v, v, <0, 4, 1, 5>, i.e. vector_shuffle v, undef,
24102399 /// <0, 0, 1, 1>
2411 bool X86::isUNPCKL_v_undef_Mask(SDNode *N) {
2412 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2413
2414 unsigned NumElems = N->getNumOperands();
2400 static bool isUNPCKL_v_undef_Mask(SmallVectorImpl &Mask, MVT VT) {
2401 int NumElems = VT.getVectorNumElements();
24152402 if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16)
24162403 return false;
2417
2418 for (unsigned i = 0, j = 0; i != NumElems; i += 2, ++j) {
2419 SDValue BitI = N->getOperand(i);
2420 SDValue BitI1 = N->getOperand(i+1);
2421
2404
2405 for (int i = 0, j = 0; i != NumElems; i += 2, ++j) {
2406 int BitI = Mask[i];
2407 int BitI1 = Mask[i+1];
24222408 if (!isUndefOrEqual(BitI, j))
24232409 return false;
24242410 if (!isUndefOrEqual(BitI1, j))
24252411 return false;
24262412 }
2427
24282413 return true;
2414 }
2415
2416 bool X86::isUNPCKL_v_undef_Mask(ShuffleVectorSDNode *N) {
2417 SmallVector M;
2418 N->getMask(M);
2419 return ::isUNPCKL_v_undef_Mask(M, N->getValueType(0));
24292420 }
24302421
24312422 /// isUNPCKH_v_undef_Mask - Special case of isUNPCKHMask for canonical form
24322423 /// of vector_shuffle v, v, <2, 6, 3, 7>, i.e. vector_shuffle v, undef,
24332424 /// <2, 2, 3, 3>
2434 bool X86::isUNPCKH_v_undef_Mask(SDNode *N) {
2435 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2436
2437 unsigned NumElems = N->getNumOperands();
2425 static bool isUNPCKH_v_undef_Mask(SmallVectorImpl &Mask, MVT VT) {
2426 int NumElems = VT.getVectorNumElements();
24382427 if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16)
24392428 return false;
2440
2441 for (unsigned i = 0, j = NumElems / 2; i != NumElems; i += 2, ++j) {
2442 SDValue BitI = N->getOperand(i);
2443 SDValue BitI1 = N->getOperand(i + 1);
2444
2429
2430 for (int i = 0, j = NumElems / 2; i != NumElems; i += 2, ++j) {
2431 int BitI = Mask[i];
2432 int BitI1 = Mask[i+1];
24452433 if (!isUndefOrEqual(BitI, j))
24462434 return false;
24472435 if (!isUndefOrEqual(BitI1, j))
24482436 return false;
24492437 }
2450
24512438 return true;
2439 }
2440
2441 bool X86::isUNPCKH_v_undef_Mask(ShuffleVectorSDNode *N) {
2442 SmallVector M;
2443 N->getMask(M);
2444 return ::isUNPCKH_v_undef_Mask(M, N->getValueType(0));
24522445 }
24532446
24542447 /// isMOVLMask - Return true if the specified VECTOR_SHUFFLE operand
24552448 /// specifies a shuffle of elements that is suitable for input to MOVSS,
24562449 /// MOVSD, and MOVD, i.e. setting the lowest element.
2457 template
2458 static bool isMOVLMask(SDOperand *Elts, unsigned NumElts) {
2450 static bool isMOVLMask(SmallVectorImpl &Mask, MVT VT) {
2451 int NumElts = VT.getVectorNumElements();
24592452 if (NumElts != 2 && NumElts != 4)
24602453 return false;
2461
2462 if (!isUndefOrEqual(Elts[0], NumElts))
2454
2455 if (!isUndefOrEqual(Mask[0], NumElts))
24632456 return false;
2464
2465 for (unsigned i = 1; i < NumElts; ++i) {
2466 if (!isUndefOrEqual(Elts[i], i))
2457
2458 for (int i = 1; i < NumElts; ++i)
2459 if (!isUndefOrEqual(Mask[i], i))
24672460 return false;
2468 }
2469
2461
24702462 return true;
24712463 }
24722464
2473 bool X86::isMOVLMask(SDNode *N) {
2474 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2475 return ::isMOVLMask(N->op_begin(), N->getNumOperands());
2465 bool X86::isMOVLMask(ShuffleVectorSDNode *N) {
2466 SmallVector M;
2467 N->getMask(M);
2468 return ::isMOVLMask(M, N->getValueType(0));
24762469 }
24772470
24782471 /// isCommutedMOVL - Returns true if the shuffle mask is except the reverse
24792472 /// of what x86 movss want. X86 movs requires the lowest element to be lowest
24802473 /// element of vector 2 and the other elements to come from vector 1 in order.
2481 template
2482 static bool isCommutedMOVL(SDOperand *Ops, unsigned NumOps,
2483 bool V2IsSplat = false,
2484 bool V2IsUndef = false) {
2474 static bool isCommutedMOVLMask(SmallVectorImpl &Mask, MVT VT,
2475 bool V2IsSplat = false, bool V2IsUndef = false) {
2476 int NumOps = VT.getVectorNumElements();
24852477 if (NumOps != 2 && NumOps != 4 && NumOps != 8 && NumOps != 16)
24862478 return false;
2487
2488 if (!isUndefOrEqual(Ops[0], 0))
2479
2480 if (!isUndefOrEqual(Mask[0], 0))
24892481 return false;
2490
2491 for (unsigned i = 1; i < NumOps; ++i) {
2492 SDValue Arg = Ops[i];
2493 if (!(isUndefOrEqual(Arg, i+NumOps) ||
2494 (V2IsUndef && isUndefOrInRange(Arg, NumOps, NumOps*2)) ||
2495 (V2IsSplat && isUndefOrEqual(Arg, NumOps))))
2482
2483 for (int i = 1; i < NumOps; ++i)
2484 if (!(isUndefOrEqual(Mask[i], i+NumOps) ||
2485 (V2IsUndef && isUndefOrInRange(Mask[i], NumOps, NumOps*2)) ||
2486 (V2IsSplat && isUndefOrEqual(Mask[i], NumOps))))
24962487 return false;
2497 }
2498
2488
24992489 return true;
25002490 }
25012491
2502 static bool isCommutedMOVL(SDNode *N, bool V2IsSplat = false,
2492 static bool isCommutedMOVL(ShuffleVectorSDNode *N, bool V2IsSplat = false,
25032493 bool V2IsUndef = false) {
2504 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2505 return isCommutedMOVL(N->op_begin(), N->getNumOperands(),
2506 V2IsSplat, V2IsUndef);
2494 SmallVector M;
2495 N->getMask(M);
2496 return isCommutedMOVLMask(M, N->getValueType(0), V2IsSplat, V2IsUndef);
25072497 }
25082498
25092499 /// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand
25102500 /// specifies a shuffle of elements that is suitable for input to MOVSHDUP.
2511 bool X86::isMOVSHDUPMask(SDNode *N) {
2512 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2513
2514 if (N->getNumOperands() != 4)
2501 bool X86::isMOVSHDUPMask(ShuffleVectorSDNode *N) {
2502 if (N->getValueType(0).getVectorNumElements() != 4)
25152503 return false;
25162504
25172505 // Expect 1, 1, 3, 3
25182506 for (unsigned i = 0; i < 2; ++i) {
2519 SDValue Arg = N->getOperand(i);
2520 if (Arg.getOpcode() == ISD::UNDEF) continue;
2521 assert(isa(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2522 unsigned Val = cast(Arg)->getZExtValue();
2523 if (Val != 1) return false;
2507 int Elt = N->getMaskElt(i);
2508 if (Elt >= 0 && Elt != 1)
2509 return false;
25242510 }
25252511
25262512 bool HasHi = false;
25272513 for (unsigned i = 2; i < 4; ++i) {
2528 SDValue Arg = N->getOperand(i);
2529 if (Arg.getOpcode() == ISD::UNDEF) continue;
2530 assert(isa(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2531 unsigned Val = cast(Arg)->getZExtValue();
2532 if (Val != 3) return false;
2533 HasHi = true;
2534 }
2535
2514 int Elt = N->getMaskElt(i);
2515 if (Elt >= 0 && Elt != 3)
2516 return false;
2517 if (Elt == 3)
2518 HasHi = true;
2519 }
25362520 // Don't use movshdup if it can be done with a shufps.
2521 // FIXME: verify that matching u, u, 3, 3 is what we want.
25372522 return HasHi;
25382523 }
25392524
25402525 /// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand
25412526 /// specifies a shuffle of elements that is suitable for input to MOVSLDUP.
2542 bool X86::isMOVSLDUPMask(SDNode *N) {
2543 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2544
2545 if (N->getNumOperands() != 4)
2527 bool X86::isMOVSLDUPMask(ShuffleVectorSDNode *N) {
2528 if (N->getValueType(0).getVectorNumElements() != 4)
25462529 return false;
25472530
25482531 // Expect 0, 0, 2, 2
2549 for (unsigned i = 0; i < 2; ++i) {
2550 SDValue Arg = N->getOperand(i);
2551 if (Arg.getOpcode() == ISD::UNDEF) continue;
2552 assert(isa(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2553 unsigned Val = cast(Arg)->getZExtValue();
2554 if (Val != 0) return false;
2555 }
2532 for (unsigned i = 0; i < 2; ++i)
2533 if (N->getMaskElt(i) > 0)
2534 return false;
25562535
25572536 bool HasHi = false;
25582537 for (unsigned i = 2; i < 4; ++i) {
2559 SDValue Arg = N->getOperand(i);
2560 if (Arg.getOpcode() == ISD::UNDEF) continue;
2561 assert(isa(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2562 unsigned Val = cast(Arg)->getZExtValue();
2563 if (Val != 2) return false;
2564 HasHi = true;
2565 }
2566
2567 // Don't use movshdup if it can be done with a shufps.
2538 int Elt = N->getMaskElt(i);
2539 if (Elt >= 0 && Elt != 2)
2540 return false;
2541 if (Elt == 2)
2542 HasHi = true;
2543 }
2544 // Don't use movsldup if it can be done with a shufps.
25682545 return HasHi;
2569 }
2570
2571 /// isIdentityMask - Return true if the specified VECTOR_SHUFFLE operand
2572 /// specifies a identity operation on the LHS or RHS.
2573 static bool isIdentityMask(SDNode *N, bool RHS = false) {
2574 unsigned NumElems = N->getNumOperands();
2575 for (unsigned i = 0; i < NumElems; ++i)
2576 if (!isUndefOrEqual(N->getOperand(i), i + (RHS ? NumElems : 0)))
2577 return false;
2578 return true;
2579 }
2580
2581 /// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies
2582 /// a splat of a single element.
2583 static bool isSplatMask(SDNode *N) {
2584 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2585
2586 // This is a splat operation if each element of the permute is the same, and
2587 // if the value doesn't reference the second vector.
2588 unsigned NumElems = N->getNumOperands();
2589 SDValue ElementBase;
2590 unsigned i = 0;
2591 for (; i != NumElems; ++i) {
2592 SDValue Elt = N->getOperand(i);
2593 if (isa(Elt)) {
2594 ElementBase = Elt;
2595 break;
2596 }
2597 }
2598
2599 if (!ElementBase.getNode())
2600 return false;
2601
2602 for (; i != NumElems; ++i) {
2603 SDValue Arg = N->getOperand(i);
2604 if (Arg.getOpcode() == ISD::UNDEF) continue;
2605 assert(isa(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2606 if (Arg != ElementBase) return false;
2607 }
2608
2609 // Make sure it is a splat of the first vector operand.
2610 return cast(ElementBase)->getZExtValue() < NumElems;
2611 }
2612
2613 /// getSplatMaskEltNo - Given a splat mask, return the index to the element
2614 /// we want to splat.
2615 static SDValue getSplatMaskEltNo(SDNode *N) {
2616 assert(isSplatMask(N) && "Not a splat mask");
2617 unsigned NumElems = N->getNumOperands();
2618 SDValue ElementBase;
2619 unsigned i = 0;
2620 for (; i != NumElems; ++i) {
2621 SDValue Elt = N->getOperand(i);
2622 if (isa(Elt))
2623 return Elt;
2624 }
2625 assert(0 && " No splat value found!");
2626 return SDValue();
2627 }
2628
2629
2630 /// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies
2631 /// a splat of a single element and it's a 2 or 4 element mask.
2632 bool X86::isSplatMask(SDNode *N) {
2633 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2634
2635 // We can only splat 64-bit, and 32-bit quantities with a single instruction.
2636 if (N->getNumOperands() != 4 && N->getNumOperands() != 2)
2637 return false;
2638 return ::isSplatMask(N);
2639 }
2640
2641 /// isSplatLoMask - Return true if the specified VECTOR_SHUFFLE operand
2642 /// specifies a splat of zero element.
2643 bool X86::isSplatLoMask(SDNode *N) {
2644 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2645
2646 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
2647 if (!isUndefOrEqual(N->getOperand(i), 0))
2648 return false;
2649 return true;
26502546 }
26512547
26522548 /// isMOVDDUPMask - Return true if the specified VECTOR_SHUFFLE operand
26532549 /// specifies a shuffle of elements that is suitable for input to MOVDDUP.
2654 bool X86::isMOVDDUPMask(SDNode *N) {
2655 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2656
2657 unsigned e = N->getNumOperands() / 2;
2658 for (unsigned i = 0; i < e; ++i)
2659 if (!isUndefOrEqual(N->getOperand(i), i))
2550 bool X86::isMOVDDUPMask(ShuffleVectorSDNode *N) {
2551 int e = N->getValueType(0).getVectorNumElements() / 2;
2552
2553 for (int i = 0; i < e; ++i)
2554 if (!isUndefOrEqual(N->getMaskElt(i), i))
26602555 return false;
2661 for (unsigned i = 0; i < e; ++i)
2662 if (!isUndefOrEqual(N->getOperand(e+i), i))
2556 for (int i = 0; i < e; ++i)
2557 if (!isUndefOrEqual(N->getMaskElt(e+i), i))
26632558 return false;
26642559 return true;
26652560 }
26682563 /// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUF* and SHUFP*
26692564 /// instructions.
26702565 unsigned X86::getShuffleSHUFImmediate(SDNode *N) {
2671 unsigned NumOperands = N->getNumOperands();
2566 ShuffleVectorSDNode *SVOp = cast(N);
2567 int NumOperands = SVOp->getValueType(0).getVectorNumElements();
2568
26722569 unsigned Shift = (NumOperands == 4) ? 2 : 1;
26732570 unsigned Mask = 0;
2674 for (unsigned i = 0; i < NumOperands; ++i) {
2675 unsigned Val = 0;
2676 SDValue Arg = N->getOperand(NumOperands-i-1);
2677 if (Arg.getOpcode() != ISD::UNDEF)
2678 Val = cast(Arg)->getZExtValue();
2571 for (int i = 0; i < NumOperands; ++i) {
2572 int Val = SVOp->getMaskElt(NumOperands-i-1);
2573 if (Val < 0) Val = 0;
26792574 if (Val >= NumOperands) Val -= NumOperands;
26802575 Mask |= Val;
26812576 if (i != NumOperands - 1)
26822577 Mask <<= Shift;
26832578 }
2684
26852579 return Mask;
26862580 }
26872581
26892583 /// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFHW
26902584 /// instructions.
26912585 unsigned X86::getShufflePSHUFHWImmediate(SDNode *N) {
2586 ShuffleVectorSDNode *SVOp = cast(N);
26922587 unsigned Mask = 0;
26932588 // 8 nodes, but we only care about the last 4.
26942589 for (unsigned i = 7; i >= 4; --i) {
2695 unsigned Val = 0;
2696 SDValue Arg = N->getOperand(i);
2697 if (Arg.getOpcode() != ISD::UNDEF) {
2698 Val = cast(Arg)->getZExtValue();
2590 int Val = SVOp->getMaskElt(i);
2591 if (Val >= 0)
26992592 Mask |= (Val - 4);
2700 }
27012593 if (i != 4)
27022594 Mask <<= 2;
27032595 }
2704
27052596 return Mask;
27062597 }
27072598
27092600 /// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFLW
27102601 /// instructions.
27112602 unsigned X86::getShufflePSHUFLWImmediate(SDNode *N) {
2603 ShuffleVectorSDNode *SVOp = cast(N);
27122604 unsigned Mask = 0;
27132605 // 8 nodes, but we only care about the first 4.
27142606 for (int i = 3; i >= 0; --i) {
2715 unsigned Val = 0;
2716 SDValue Arg = N->getOperand(i);
2717 if (Arg.getOpcode() != ISD::UNDEF)
2718 Val = cast(Arg)->getZExtValue();
2719 Mask |= Val;
2607 int Val = SVOp->getMaskElt(i);
2608 if (Val >= 0)
2609 Mask |= Val;
27202610 if (i != 0)
27212611 Mask <<= 2;
27222612 }
2723
27242613 return Mask;
27252614 }
27262615
2727 /// CommuteVectorShuffle - Swap vector_shuffle operands as well as
2728 /// values in ther permute mask.
2729 static SDValue CommuteVectorShuffle(SDValue Op, SDValue &V1,
2730 SDValue &V2, SDValue &Mask,
2731 SelectionDAG &DAG) {
2732 MVT VT = Op.getValueType();
2733 MVT MaskVT = Mask.getValueType();
2734 MVT EltVT = MaskVT.getVectorElementType();
2735 unsigned NumElems = Mask.getNumOperands();
2736 SmallVector MaskVec;
2737 DebugLoc dl = Op.getDebugLoc();
2738
2739 for (unsigned i = 0; i != NumElems; ++i) {
2740 SDValue Arg = Mask.getOperand(i);
2741 if (Arg.getOpcode() == ISD::UNDEF) {
2742 MaskVec.push_back(DAG.getUNDEF(EltVT));
2743 continue;
2744 }
2745 assert(isa(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2746 unsigned Val = cast(Arg)->getZExtValue();
2747 if (Val < NumElems)
2748 MaskVec.push_back(DAG.getConstant(Val + NumElems, EltVT));
2616 /// CommuteVectorShuffle - Swap vector_shuffle operands as well as values in
2617 /// their permute mask.
2618 static SDValue CommuteVectorShuffle(ShuffleVectorSDNode *SVOp,
2619 SelectionDAG &DAG) {
2620 MVT VT = SVOp->getValueType(0);
2621 int NumElems = VT.getVectorNumElements();
2622 SmallVector MaskVec;
2623
2624 for (int i = 0; i != NumElems; ++i) {
2625 int idx = SVOp->getMaskElt(i);
2626 if (idx < 0)
2627 MaskVec.push_back(idx);
2628 else if (idx < NumElems)
2629 MaskVec.push_back(idx + NumElems);
27492630 else
2750 MaskVec.push_back(DAG.getConstant(Val - NumElems, EltVT));
2751 }
2752
2753 std::swap(V1, V2);
2754 Mask = DAG.getNode(ISD::BUILD_VECTOR, dl, MaskVT, &MaskVec[0], NumElems);
2755 return DAG.getNode(ISD::VECTOR_SHUFFLE, dl, VT, V1, V2, Mask);
2631 MaskVec.push_back(idx - NumElems);
2632 }
2633 return DAG.getVectorShuffle(VT, SVOp->getDebugLoc(), SVOp->getOperand(1),
2634 SVOp->getOperand(0), &MaskVec[0]);
27562635 }
27572636
27582637 /// CommuteVectorShuffleMask - Change values in a shuffle permute mask assuming
27592638 /// the two vector operands have swapped position.
2760 static
2761 SDValue CommuteVectorShuffleMask(SDValue Mask, SelectionDAG &DAG, DebugLoc dl) {
2762 MVT MaskVT = Mask.getValueType();
2763 MVT EltVT = MaskVT.getVectorElementType();
2764 unsigned NumElems = Mask.getNumOperands();
2765 SmallVector MaskVec;
2766 for (unsigned i = 0; i != NumElems; ++i) {
2767 SDValue Arg = Mask.getOperand(i);
2768 if (Arg.getOpcode() == ISD::UNDEF) {
2769 MaskVec.push_back(DAG.getUNDEF(EltVT));
2639 static void CommuteVectorShuffleMask(SmallVectorImpl &Mask, MVT VT) {
2640 int NumElems = VT.getVectorNumElements();
2641 for (int i = 0; i != NumElems; ++i) {
2642 int idx = Mask[i];
2643 if (idx < 0)
27702644 continue;
2771 }
2772 assert(isa(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2773 unsigned Val = cast(Arg)->getZExtValue();
2774 if (Val < NumElems)
2775 MaskVec.push_back(DAG.getConstant(Val + NumElems, EltVT));
2645 else if (idx < NumElems)
2646 Mask[i] = idx + NumElems;
27762647 else
2777 MaskVec.push_back(DAG.getConstant(Val - NumElems, EltVT));
2778 }
2779 return DAG.getNode(ISD::BUILD_VECTOR, dl, MaskVT, &MaskVec[0], NumElems);
2780 }
2781
2648 Mask[i] = idx - NumElems;
2649 }
2650 }
27822651
27832652 /// ShouldXformToMOVHLPS - Return true if the node should be transformed to
27842653 /// match movhlps. The lower half elements should come from upper half of
27852654 /// V1 (and in order), and the upper half elements should come from the upper
27862655 /// half of V2 (and in order).
2787 static bool ShouldXformToMOVHLPS(SDNode *Mask) {
2788 unsigned NumElems = Mask->getNumOperands();
2789 if (NumElems != 4)
2656 static bool ShouldXformToMOVHLPS(ShuffleVectorSDNode *Op) {
2657 if (Op->getValueType(0).getVectorNumElements() != 4)
27902658 return false;
27912659 for (unsigned i = 0, e = 2; i != e; ++i)
2792 if (!isUndefOrEqual(Mask->getOperand(i), i+2))
2660 if (!isUndefOrEqual(Op->getMaskElt(i), i+2))
27932661 return false;
27942662 for (unsigned i = 2; i != 4; ++i)
2795 if (!isUndefOrEqual(Mask->getOperand(i), i+4))
2663 if (!isUndefOrEqual(Op->getMaskElt(i), i+4))
27962664 return false;
27972665 return true;
27982666 }
28162684 /// V1 (and in order), and the upper half elements should come from the upper
28172685 /// half of V2 (and in order). And since V1 will become the source of the
28182686 /// MOVLP, it must be either a vector load or a scalar load to vector.
2819 static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2, SDNode *Mask) {
2687 static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2,
2688 ShuffleVectorSDNode *Op) {
28202689 if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1))
28212690 return false;
28222691 // Is V2 is a vector load, don't do this transformation. We will try to use
28242693 if (ISD::isNON_EXTLoad(V2))
28252694 return false;
28262695
2827 unsigned NumElems = Mask->getNumOperands();
2696 int NumElems = Op->getValueType(0).getVectorNumElements();
2697
28282698 if (NumElems != 2 && NumElems != 4)
28292699 return false;
2830 for (unsigned i = 0, e = NumElems/2; i != e; ++i)
2831 if (!isUndefOrEqual(Mask->getOperand(i), i))
2700 for (int i = 0, e = NumElems/2; i != e; ++i)
2701 if (!isUndefOrEqual(Op->getMaskElt(i), i))
28322702 return false;
2833 for (unsigned i = NumElems/2; i != NumElems; ++i)
2834 if (!isUndefOrEqual(Mask->getOperand(i), i+NumElems))
2703 for (int i = NumElems/2; i != NumElems; ++i)
2704 if (!isUndefOrEqual(Op->getMaskElt(i), i+NumElems))
28352705 return false;
28362706 return true;
28372707 }
28462716 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i)
28472717 if (N->getOperand(i) != SplatValue)
28482718 return false;
2849 return true;
2850 }
2851
2852 /// isUndefShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved
2853 /// to an undef.
2854 static bool isUndefShuffle(SDNode *N) {
2855 if (N->getOpcode() != ISD::VECTOR_SHUFFLE)
2856 return false;
2857
2858 SDValue V1 = N->getOperand(0);
2859 SDValue V2 = N->getOperand(1);
2860 SDValue Mask = N->getOperand(2);
2861 unsigned NumElems = Mask.getNumOperands();
2862 for (unsigned i = 0; i != NumElems; ++i) {
2863 SDValue Arg = Mask.getOperand(i);
2864 if (Arg.getOpcode() != ISD::UNDEF) {
2865 unsigned Val = cast(Arg)->getZExtValue();
2866 if (Val < NumElems && V1.getOpcode() != ISD::UNDEF)
2867 return false;
2868 else if (Val >= NumElems && V2.getOpcode() != ISD::UNDEF)
2869 return false;
2870 }
2871 }
28722719 return true;
28732720 }
28742721
28822729 }
28832730
28842731 /// isZeroShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved
2885 /// to an zero vector.
2886 static bool isZeroShuffle(SDNode *N) {
2887 if (N->getOpcode() != ISD::VECTOR_SHUFFLE)
2888 return false;
2889
2732 /// to an zero vector.
2733 /// FIXME: move to dag combiner?
2734 static bool isZeroShuffle(ShuffleVectorSDNode *N) {
28902735 SDValue V1 = N->getOperand(0);
28912736 SDValue V2 = N->getOperand(1);
2892 SDValue Mask = N->getOperand(2);
2893 unsigned NumElems = Mask.getNumOperands();
2894 for (unsigned i = 0; i != NumElems; ++i) {
2895 SDValue Arg = Mask.getOperand(i);
2896 if (Arg.getOpcode() == ISD::UNDEF)
2897 continue;
2898
2899 unsigned Idx = cast(Arg)->getZExtValue();
2900 if (Idx < NumElems) {
2901 unsigned Opc = V1.getNode()->getOpcode();
2737 int NumElems = N->getValueType(0).getVectorNumElements();
2738 for (int i = 0; i != NumElems; ++i) {
2739 int Idx = N->getMaskElt(i);
2740 if (Idx >= NumElems) {
2741 unsigned Opc = V2.getOpcode();
2742 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V2.getNode()))
2743 continue;
2744 if (Opc != ISD::BUILD_VECTOR || !isZeroNode(V2.getOperand(Idx-NumElems)))
2745 return false;
2746 } else if (Idx >= 0) {
2747 unsigned Opc = V1.getOpcode();
29022748 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V1.getNode()))
29032749 continue;
2904 if (Opc != ISD::BUILD_VECTOR ||
2905 !isZeroNode(V1.getNode()->getOperand(Idx)))
2906 return false;
2907 } else if (Idx >= NumElems) {
2908 unsigned Opc = V2.getNode()->getOpcode();
2909 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V2.getNode()))
2910 continue;
2911 if (Opc != ISD::BUILD_VECTOR ||
2912 !isZeroNode(V2.getNode()->getOperand(Idx - NumElems)))
2750 if (Opc != ISD::BUILD_VECTOR || !isZeroNode(V1.getOperand(Idx)))
29132751 return false;
29142752 }
29152753 }
29572795
29582796 /// NormalizeMask - V2 is a splat, modify the mask (if needed) so all elements