llvm.org GIT mirror llvm / 73a1b67
Revise scalar replacement to be more flexible about handle bitcasts and GEPs. While scanning through the uses of an alloca, keep track of the current offset relative to the start of the alloca, and check memory references to see if the offset & size correspond to a component within the alloca. This has the nice benefit of unifying much of the code from isSafeUseOfAllocation, isSafeElementUse, and isSafeUseOfBitCastedAllocation. The code to rewrite the uses of a promoted alloca, after it is determined to be safe, is reorganized in the same way. Also, when rewriting GEP instructions, mark them as "in-bounds" since all the indices are known to be safe. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@91184 91177308-0d34-0410-b5e6-96231b3b80d8 Bob Wilson 9 years ago
2 changed file(s) with 459 addition(s) and 415 deletion(s). Raw diff Collapse all Expand all
101101
102102 int isSafeAllocaToScalarRepl(AllocaInst *AI);
103103
104 void isSafeUseOfAllocation(Instruction *User, AllocaInst *AI,
105 AllocaInfo &Info);
106 void isSafeElementUse(Value *Ptr, bool isFirstElt, AllocaInst *AI,
107 AllocaInfo &Info);
108 void isSafeMemIntrinsicOnAllocation(MemIntrinsic *MI, AllocaInst *AI,
109 unsigned OpNo, AllocaInfo &Info);
110 void isSafeUseOfBitCastedAllocation(BitCastInst *User, AllocaInst *AI,
111 AllocaInfo &Info);
104 void isSafeForScalarRepl(Instruction *I, AllocaInst *AI, uint64_t Offset,
105 uint64_t ArrayOffset, AllocaInfo &Info);
106 void isSafeGEP(GetElementPtrInst *GEPI, AllocaInst *AI, uint64_t &Offset,
107 uint64_t &ArrayOffset, AllocaInfo &Info);
108 void isSafeMemAccess(AllocaInst *AI, uint64_t Offset, uint64_t ArrayOffset,
109 uint64_t MemSize, const Type *MemOpType, bool isStore,
110 AllocaInfo &Info);
111 bool TypeHasComponent(const Type *T, uint64_t Offset, uint64_t Size);
112 unsigned FindElementAndOffset(const Type *&T, uint64_t &Offset);
112113
113114 void DoScalarReplacement(AllocaInst *AI,
114115 std::vector &WorkList);
115116 void CleanupGEP(GetElementPtrInst *GEP);
116 void CleanupAllocaUsers(AllocaInst *AI);
117 void CleanupAllocaUsers(Value *V);
117118 AllocaInst *AddNewAlloca(Function &F, const Type *Ty, AllocaInst *Base);
118119
119 void RewriteBitCastUserOfAlloca(Instruction *BCInst, AllocaInst *AI,
120 SmallVector &NewElts);
121
122 void RewriteMemIntrinUserOfAlloca(MemIntrinsic *MI, Instruction *BCInst,
120 void RewriteForScalarRepl(Instruction *I, AllocaInst *AI, uint64_t Offset,
121 SmallVector &NewElts);
122 void RewriteGEP(GetElementPtrInst *GEPI, AllocaInst *AI, uint64_t Offset,
123 SmallVector &NewElts);
124 void RewriteMemIntrinUserOfAlloca(MemIntrinsic *MI, Instruction *Inst,
123125 AllocaInst *AI,
124126 SmallVector &NewElts);
125127 void RewriteStoreUserOfWholeAlloca(StoreInst *SI, AllocaInst *AI,
359361 }
360362 }
361363
362 // Now that we have created the alloca instructions that we want to use,
363 // expand the getelementptr instructions to use them.
364 while (!AI->use_empty()) {
365 Instruction *User = cast(AI->use_back());
366 if (BitCastInst *BCInst = dyn_cast(User)) {
367 RewriteBitCastUserOfAlloca(BCInst, AI, ElementAllocas);
368 BCInst->eraseFromParent();
369 continue;
370 }
371
372 // Replace:
373 // %res = load { i32, i32 }* %alloc
374 // with:
375 // %load.0 = load i32* %alloc.0
376 // %insert.0 insertvalue { i32, i32 } zeroinitializer, i32 %load.0, 0
377 // %load.1 = load i32* %alloc.1
378 // %insert = insertvalue { i32, i32 } %insert.0, i32 %load.1, 1
379 // (Also works for arrays instead of structs)
380 if (LoadInst *LI = dyn_cast(User)) {
381 Value *Insert = UndefValue::get(LI->getType());
382 for (unsigned i = 0, e = ElementAllocas.size(); i != e; ++i) {
383 Value *Load = new LoadInst(ElementAllocas[i], "load", LI);
384 Insert = InsertValueInst::Create(Insert, Load, i, "insert", LI);
385 }
386 LI->replaceAllUsesWith(Insert);
387 LI->eraseFromParent();
388 continue;
389 }
390
391 // Replace:
392 // store { i32, i32 } %val, { i32, i32 }* %alloc
393 // with:
394 // %val.0 = extractvalue { i32, i32 } %val, 0
395 // store i32 %val.0, i32* %alloc.0
396 // %val.1 = extractvalue { i32, i32 } %val, 1
397 // store i32 %val.1, i32* %alloc.1
398 // (Also works for arrays instead of structs)
399 if (StoreInst *SI = dyn_cast(User)) {
400 Value *Val = SI->getOperand(0);
401 for (unsigned i = 0, e = ElementAllocas.size(); i != e; ++i) {
402 Value *Extract = ExtractValueInst::Create(Val, i, Val->getName(), SI);
403 new StoreInst(Extract, ElementAllocas[i], SI);
404 }
405 SI->eraseFromParent();
406 continue;
407 }
408
409 GetElementPtrInst *GEPI = cast(User);
410 // We now know that the GEP is of the form: GEP , 0,
411 unsigned Idx =
412 (unsigned)cast(GEPI->getOperand(2))->getZExtValue();
413
414 assert(Idx < ElementAllocas.size() && "Index out of range?");
415 AllocaInst *AllocaToUse = ElementAllocas[Idx];
416
417 Value *RepValue;
418 if (GEPI->getNumOperands() == 3) {
419 // Do not insert a new getelementptr instruction with zero indices, only
420 // to have it optimized out later.
421 RepValue = AllocaToUse;
422 } else {
423 // We are indexing deeply into the structure, so we still need a
424 // getelement ptr instruction to finish the indexing. This may be
425 // expanded itself once the worklist is rerun.
426 //
427 SmallVector NewArgs;
428 NewArgs.push_back(Constant::getNullValue(
429 Type::getInt32Ty(AI->getContext())));
430 NewArgs.append(GEPI->op_begin()+3, GEPI->op_end());
431 RepValue = GetElementPtrInst::Create(AllocaToUse, NewArgs.begin(),
432 NewArgs.end(), "", GEPI);
433 RepValue->takeName(GEPI);
434 }
435
436 // If this GEP is to the start of the aggregate, check for memcpys.
437 if (Idx == 0 && GEPI->hasAllZeroIndices())
438 RewriteBitCastUserOfAlloca(GEPI, AI, ElementAllocas);
439
440 // Move all of the users over to the new GEP.
441 GEPI->replaceAllUsesWith(RepValue);
442 // Delete the old GEP
443 GEPI->eraseFromParent();
444 }
445
446 // Finally, delete the Alloca instruction
447 AI->eraseFromParent();
364 // Now that we have created the new alloca instructions, rewrite all the
365 // uses of the old alloca.
366 RewriteForScalarRepl(AI, AI, 0, ElementAllocas);
448367 NumReplaced++;
449368 }
450
451 /// isSafeElementUse - Check to see if this use is an allowed use for a
452 /// getelementptr instruction of an array aggregate allocation. isFirstElt
453 /// indicates whether Ptr is known to the start of the aggregate.
454 void SROA::isSafeElementUse(Value *Ptr, bool isFirstElt, AllocaInst *AI,
455 AllocaInfo &Info) {
456 for (Value::use_iterator I = Ptr->use_begin(), E = Ptr->use_end();
457 I != E; ++I) {
458 Instruction *User = cast(*I);
459 switch (User->getOpcode()) {
460 case Instruction::Load: break;
461 case Instruction::Store:
462 // Store is ok if storing INTO the pointer, not storing the pointer
463 if (User->getOperand(0) == Ptr) return MarkUnsafe(Info);
464 break;
465 case Instruction::GetElementPtr: {
466 GetElementPtrInst *GEP = cast(User);
467 bool AreAllZeroIndices = isFirstElt;
468 if (GEP->getNumOperands() > 1 &&
469 (!isa(GEP->getOperand(1)) ||
470 !cast(GEP->getOperand(1))->isZero()))
471 // Using pointer arithmetic to navigate the array.
472 return MarkUnsafe(Info);
473
474 // Verify that any array subscripts are in range.
475 for (gep_type_iterator GEPIt = gep_type_begin(GEP),
476 E = gep_type_end(GEP); GEPIt != E; ++GEPIt) {
477 // Ignore struct elements, no extra checking needed for these.
478 if (isa(*GEPIt))
479 continue;
480
481 // This GEP indexes an array. Verify that this is an in-range
482 // constant integer. Specifically, consider A[0][i]. We cannot know that
483 // the user isn't doing invalid things like allowing i to index an
484 // out-of-range subscript that accesses A[1]. Because of this, we have
485 // to reject SROA of any accesses into structs where any of the
486 // components are variables.
487 ConstantInt *IdxVal = dyn_cast(GEPIt.getOperand());
488 if (!IdxVal) return MarkUnsafe(Info);
489
490 // Are all indices still zero?
491 AreAllZeroIndices &= IdxVal->isZero();
492
493 if (const ArrayType *AT = dyn_cast(*GEPIt)) {
494 if (IdxVal->getZExtValue() >= AT->getNumElements())
495 return MarkUnsafe(Info);
496 } else if (const VectorType *VT = dyn_cast(*GEPIt)) {
497 if (IdxVal->getZExtValue() >= VT->getNumElements())
498 return MarkUnsafe(Info);
499 }
500 }
501
502 isSafeElementUse(GEP, AreAllZeroIndices, AI, Info);
503 if (Info.isUnsafe) return;
504 break;
505 }
506 case Instruction::BitCast:
507 if (isFirstElt) {
508 isSafeUseOfBitCastedAllocation(cast(User), AI, Info);
509 if (Info.isUnsafe) return;
510 break;
511 }
512 DEBUG(errs() << " Transformation preventing inst: " << *User << '\n');
513 return MarkUnsafe(Info);
514 case Instruction::Call:
515 if (MemIntrinsic *MI = dyn_cast(User)) {
516 if (isFirstElt) {
517 isSafeMemIntrinsicOnAllocation(MI, AI, I.getOperandNo(), Info);
518 if (Info.isUnsafe) return;
519 break;
520 }
521 }
522 DEBUG(errs() << " Transformation preventing inst: " << *User << '\n');
523 return MarkUnsafe(Info);
524 default:
525 DEBUG(errs() << " Transformation preventing inst: " << *User << '\n');
526 return MarkUnsafe(Info);
527 }
528 }
529 return; // All users look ok :)
530 }
531
369
532370 /// AllUsersAreLoads - Return true if all users of this value are loads.
533371 static bool AllUsersAreLoads(Value *Ptr) {
534372 for (Value::use_iterator I = Ptr->use_begin(), E = Ptr->use_end();
538376 return true;
539377 }
540378
541 /// isSafeUseOfAllocation - Check if this user is an allowed use for an
542 /// aggregate allocation.
543 void SROA::isSafeUseOfAllocation(Instruction *User, AllocaInst *AI,
544 AllocaInfo &Info) {
545 if (BitCastInst *C = dyn_cast(User))
546 return isSafeUseOfBitCastedAllocation(C, AI, Info);
547
548 if (LoadInst *LI = dyn_cast(User))
549 if (!LI->isVolatile())
550 return;// Loads (returning a first class aggregrate) are always rewritable
551
552 if (StoreInst *SI = dyn_cast(User))
553 if (!SI->isVolatile() && SI->getOperand(0) != AI)
554 return;// Store is ok if storing INTO the pointer, not storing the pointer
555
556 GetElementPtrInst *GEPI = dyn_cast(User);
557 if (GEPI == 0)
379 /// isSafeForScalarRepl - Check if instruction I is a safe use with regard to
380 /// performing scalar replacement of alloca AI. The results are flagged in
381 /// the Info parameter. Offset and ArrayOffset indicate the position within
382 /// AI that is referenced by this instruction.
383 void SROA::isSafeForScalarRepl(Instruction *I, AllocaInst *AI, uint64_t Offset,
384 uint64_t ArrayOffset, AllocaInfo &Info) {
385 for (Value::use_iterator UI = I->use_begin(), E = I->use_end(); UI!=E; ++UI) {
386 Instruction *User = cast(*UI);
387
388 if (BitCastInst *BC = dyn_cast(User)) {
389 isSafeForScalarRepl(BC, AI, Offset, ArrayOffset, Info);
390 } else if (GetElementPtrInst *GEPI = dyn_cast(User)) {
391 uint64_t GEPArrayOffset = ArrayOffset;
392 uint64_t GEPOffset = Offset;
393 isSafeGEP(GEPI, AI, GEPOffset, GEPArrayOffset, Info);
394 if (!Info.isUnsafe)
395 isSafeForScalarRepl(GEPI, AI, GEPOffset, GEPArrayOffset, Info);
396 } else if (MemIntrinsic *MI = dyn_cast(UI)) {
397 ConstantInt *Length = dyn_cast(MI->getLength());
398 if (Length)
399 isSafeMemAccess(AI, Offset, ArrayOffset, Length->getZExtValue(), 0,
400 UI.getOperandNo() == 1, Info);
401 else
402 MarkUnsafe(Info);
403 } else if (LoadInst *LI = dyn_cast(User)) {
404 if (!LI->isVolatile()) {
405 const Type *LIType = LI->getType();
406 isSafeMemAccess(AI, Offset, ArrayOffset, TD->getTypeAllocSize(LIType),
407 LIType, false, Info);
408 } else
409 MarkUnsafe(Info);
410 } else if (StoreInst *SI = dyn_cast(User)) {
411 // Store is ok if storing INTO the pointer, not storing the pointer
412 if (!SI->isVolatile() && SI->getOperand(0) != I) {
413 const Type *SIType = SI->getOperand(0)->getType();
414 isSafeMemAccess(AI, Offset, ArrayOffset, TD->getTypeAllocSize(SIType),
415 SIType, true, Info);
416 } else
417 MarkUnsafe(Info);
418 } else if (isa(UI)) {
419 // If one user is DbgInfoIntrinsic then check if all users are
420 // DbgInfoIntrinsics.
421 if (OnlyUsedByDbgInfoIntrinsics(I)) {
422 Info.needsCleanup = true;
423 return;
424 }
425 MarkUnsafe(Info);
426 } else {
427 DEBUG(errs() << " Transformation preventing inst: " << *User << '\n');
428 MarkUnsafe(Info);
429 }
430 if (Info.isUnsafe) return;
431 }
432 }
433
434 /// isSafeGEP - Check if a GEP instruction can be handled for scalar
435 /// replacement. It is safe when all the indices are constant, in-bounds
436 /// references, and when the resulting offset corresponds to an element within
437 /// the alloca type. The results are flagged in the Info parameter. Upon
438 /// return, Offset is adjusted as specified by the GEP indices. For the
439 /// special case of a variable index to a 2-element array, ArrayOffset is set
440 /// to the array element size.
441 void SROA::isSafeGEP(GetElementPtrInst *GEPI, AllocaInst *AI,
442 uint64_t &Offset, uint64_t &ArrayOffset,
443 AllocaInfo &Info) {
444 gep_type_iterator GEPIt = gep_type_begin(GEPI), E = gep_type_end(GEPI);
445 if (GEPIt == E)
446 return;
447
448 // The first GEP index must be zero.
449 if (!isa(GEPIt.getOperand()) ||
450 !cast(GEPIt.getOperand())->isZero())
558451 return MarkUnsafe(Info);
559
560 gep_type_iterator I = gep_type_begin(GEPI), E = gep_type_end(GEPI);
561
562 // The GEP is not safe to transform if not of the form "GEP , 0, ".
563 if (I == E ||
564 I.getOperand() != Constant::getNullValue(I.getOperand()->getType())) {
565 return MarkUnsafe(Info);
566 }
567
568 ++I;
569 if (I == E) return MarkUnsafe(Info); // ran out of GEP indices??
570
571 bool IsAllZeroIndices = true;
572
452 if (++GEPIt == E)
453 return;
454
573455 // If the first index is a non-constant index into an array, see if we can
574456 // handle it as a special case.
575 if (const ArrayType *AT = dyn_cast(*I)) {
576 if (!isa(I.getOperand())) {
577 IsAllZeroIndices = 0;
578 uint64_t NumElements = AT->getNumElements();
579
580 // If this is an array index and the index is not constant, we cannot
581 // promote... that is unless the array has exactly one or two elements in
582 // it, in which case we CAN promote it, but we have to canonicalize this
583 // out if this is the only problem.
584 if ((NumElements == 1 || NumElements == 2) &&
585 AllUsersAreLoads(GEPI)) {
457 const Type *ArrayEltTy = 0;
458 if (ArrayOffset == 0 && Offset == 0) {
459 if (const ArrayType *AT = dyn_cast(*GEPIt)) {
460 if (!isa(GEPIt.getOperand())) {
461 uint64_t NumElements = AT->getNumElements();
462
463 // If this is an array index and the index is not constant, we cannot
464 // promote... that is unless the array has exactly one or two elements
465 // in it, in which case we CAN promote it, but we have to canonicalize
466 // this out if this is the only problem.
467 if ((NumElements != 1 && NumElements != 2) || !AllUsersAreLoads(GEPI))
468 return MarkUnsafe(Info);
586469 Info.needsCleanup = true;
587 return; // Canonicalization required!
588 }
589 return MarkUnsafe(Info);
590 }
591 }
592
470 ArrayOffset = TD->getTypeAllocSizeInBits(AT->getElementType());
471 ArrayEltTy = AT->getElementType();
472 ++GEPIt;
473 }
474 }
475 }
476
593477 // Walk through the GEP type indices, checking the types that this indexes
594478 // into.
595 for (; I != E; ++I) {
479 for (; GEPIt != E; ++GEPIt) {
596480 // Ignore struct elements, no extra checking needed for these.
597 if (isa(*I))
481 if (isa(*GEPIt))
598482 continue;
599
600 ConstantInt *IdxVal = dyn_cast(I.getOperand());
601 if (!IdxVal) return MarkUnsafe(Info);
602
603 // Are all indices still zero?
604 IsAllZeroIndices &= IdxVal->isZero();
605
606 if (const ArrayType *AT = dyn_cast(*I)) {
483
484 ConstantInt *IdxVal = dyn_cast(GEPIt.getOperand());
485 if (!IdxVal)
486 return MarkUnsafe(Info);
487
488 if (const ArrayType *AT = dyn_cast(*GEPIt)) {
607489 // This GEP indexes an array. Verify that this is an in-range constant
608490 // integer. Specifically, consider A[0][i]. We cannot know that the user
609491 // isn't doing invalid things like allowing i to index an out-of-range
611493 // of any accesses into structs where any of the components are variables.
612494 if (IdxVal->getZExtValue() >= AT->getNumElements())
613495 return MarkUnsafe(Info);
614 } else if (const VectorType *VT = dyn_cast(*I)) {
496 } else {
497 const VectorType *VT = dyn_cast(*GEPIt);
498 assert(VT && "unexpected type in GEP type iterator");
615499 if (IdxVal->getZExtValue() >= VT->getNumElements())
616500 return MarkUnsafe(Info);
617501 }
618502 }
619
620 // If there are any non-simple uses of this getelementptr, make sure to reject
621 // them.
622 return isSafeElementUse(GEPI, IsAllZeroIndices, AI, Info);
623 }
624
625 /// isSafeMemIntrinsicOnAllocation - Check if the specified memory
626 /// intrinsic can be promoted by SROA. At this point, we know that the operand
627 /// of the memintrinsic is a pointer to the beginning of the allocation.
628 void SROA::isSafeMemIntrinsicOnAllocation(MemIntrinsic *MI, AllocaInst *AI,
629 unsigned OpNo, AllocaInfo &Info) {
630 // If not constant length, give up.
631 ConstantInt *Length = dyn_cast(MI->getLength());
632 if (!Length) return MarkUnsafe(Info);
633
634 // If not the whole aggregate, give up.
635 if (Length->getZExtValue() !=
636 TD->getTypeAllocSize(AI->getType()->getElementType()))
637 return MarkUnsafe(Info);
638
639 // We only know about memcpy/memset/memmove.
640 if (!isa(MI))
641 return MarkUnsafe(Info);
642
643 // Otherwise, we can transform it. Determine whether this is a memcpy/set
644 // into or out of the aggregate.
645 if (OpNo == 1)
646 Info.isMemCpyDst = true;
647 else {
648 assert(OpNo == 2);
649 Info.isMemCpySrc = true;
650 }
651 }
652
653 /// isSafeUseOfBitCastedAllocation - Check if all users of this bitcast
654 /// from an alloca are safe for SROA of that alloca.
655 void SROA::isSafeUseOfBitCastedAllocation(BitCastInst *BC, AllocaInst *AI,
656 AllocaInfo &Info) {
657 for (Value::use_iterator UI = BC->use_begin(), E = BC->use_end();
658 UI != E; ++UI) {
659 if (BitCastInst *BCU = dyn_cast(UI)) {
660 isSafeUseOfBitCastedAllocation(BCU, AI, Info);
661 } else if (MemIntrinsic *MI = dyn_cast(UI)) {
662 isSafeMemIntrinsicOnAllocation(MI, AI, UI.getOperandNo(), Info);
663 } else if (StoreInst *SI = dyn_cast(UI)) {
664 if (SI->isVolatile())
665 return MarkUnsafe(Info);
666
667 // If storing the entire alloca in one chunk through a bitcasted pointer
668 // to integer, we can transform it. This happens (for example) when you
669 // cast a {i32,i32}* to i64* and store through it. This is similar to the
670 // memcpy case and occurs in various "byval" cases and emulated memcpys.
671 if (isa(SI->getOperand(0)->getType()) &&
672 TD->getTypeAllocSize(SI->getOperand(0)->getType()) ==
673 TD->getTypeAllocSize(AI->getType()->getElementType())) {
674 Info.isMemCpyDst = true;
675 continue;
676 }
677 return MarkUnsafe(Info);
678 } else if (LoadInst *LI = dyn_cast(UI)) {
679 if (LI->isVolatile())
680 return MarkUnsafe(Info);
681
682 // If loading the entire alloca in one chunk through a bitcasted pointer
683 // to integer, we can transform it. This happens (for example) when you
684 // cast a {i32,i32}* to i64* and load through it. This is similar to the
685 // memcpy case and occurs in various "byval" cases and emulated memcpys.
686 if (isa(LI->getType()) &&
687 TD->getTypeAllocSize(LI->getType()) ==
688 TD->getTypeAllocSize(AI->getType()->getElementType())) {
689 Info.isMemCpySrc = true;
690 continue;
691 }
692 return MarkUnsafe(Info);
693 } else if (isa(UI)) {
694 // If one user is DbgInfoIntrinsic then check if all users are
695 // DbgInfoIntrinsics.
696 if (OnlyUsedByDbgInfoIntrinsics(BC)) {
697 Info.needsCleanup = true;
698 return;
699 }
700 else
701 MarkUnsafe(Info);
702 }
703 else {
704 return MarkUnsafe(Info);
705 }
706 if (Info.isUnsafe) return;
707 }
708 }
709
710 /// RewriteBitCastUserOfAlloca - BCInst (transitively) bitcasts AI, or indexes
711 /// to its first element. Transform users of the cast to use the new values
712 /// instead.
713 void SROA::RewriteBitCastUserOfAlloca(Instruction *BCInst, AllocaInst *AI,
714 SmallVector &NewElts) {
715 Value::use_iterator UI = BCInst->use_begin(), UE = BCInst->use_end();
716 while (UI != UE) {
503
504 // All the indices are safe. Now compute the offset due to this GEP and
505 // check if the alloca has a component element at that offset.
506 if (ArrayOffset == 0) {
507 SmallVector Indices(GEPI->op_begin() + 1, GEPI->op_end());
508 Offset += TD->getIndexedOffset(GEPI->getPointerOperandType(),
509 &Indices[0], Indices.size());
510 } else {
511 // Both array elements have the same type, so it suffices to check one of
512 // them. Copy the GEP indices starting from the array index, but replace
513 // that variable index with a constant zero.
514 SmallVector Indices(GEPI->op_begin() + 2, GEPI->op_end());
515 Indices[0] = Constant::getNullValue(Type::getInt32Ty(GEPI->getContext()));
516 const Type *ArrayEltPtr = PointerType::getUnqual(ArrayEltTy);
517 Offset += TD->getIndexedOffset(ArrayEltPtr, &Indices[0], Indices.size());
518 }
519 if (!TypeHasComponent(AI->getAllocatedType(), Offset, 0))
520 MarkUnsafe(Info);
521 }
522
523 /// isSafeMemAccess - Check if a load/store/memcpy operates on the entire AI
524 /// alloca or has an offset and size that corresponds to a component element
525 /// within it. The offset checked here may have been formed from a GEP with a
526 /// pointer bitcasted to a different type.
527 void SROA::isSafeMemAccess(AllocaInst *AI, uint64_t Offset,
528 uint64_t ArrayOffset, uint64_t MemSize,
529 const Type *MemOpType, bool isStore,
530 AllocaInfo &Info) {
531 // Check if this is a load/store of the entire alloca.
532 if (Offset == 0 && ArrayOffset == 0 &&
533 MemSize == TD->getTypeAllocSize(AI->getAllocatedType())) {
534 bool UsesAggregateType = (MemOpType == AI->getAllocatedType());
535 // This is safe for MemIntrinsics (where MemOpType is 0), integer types
536 // (which are essentially the same as the MemIntrinsics, especially with
537 // regard to copying padding between elements), or references using the
538 // aggregate type of the alloca.
539 if (!MemOpType || isa(MemOpType) || UsesAggregateType) {
540 if (!UsesAggregateType) {
541 if (isStore)
542 Info.isMemCpyDst = true;
543 else
544 Info.isMemCpySrc = true;
545 }
546 return;
547 }
548 }
549 // Check if the offset/size correspond to a component within the alloca type.
550 const Type *T = AI->getAllocatedType();
551 if (TypeHasComponent(T, Offset, MemSize) &&
552 (ArrayOffset == 0 || TypeHasComponent(T, Offset + ArrayOffset, MemSize)))
553 return;
554
555 return MarkUnsafe(Info);
556 }
557
558 /// TypeHasComponent - Return true if T has a component type with the
559 /// specified offset and size. If Size is zero, do not check the size.
560 bool SROA::TypeHasComponent(const Type *T, uint64_t Offset, uint64_t Size) {
561 const Type *EltTy;
562 uint64_t EltSize;
563 if (const StructType *ST = dyn_cast(T)) {
564 const StructLayout *Layout = TD->getStructLayout(ST);
565 unsigned EltIdx = Layout->getElementContainingOffset(Offset);
566 EltTy = ST->getContainedType(EltIdx);
567 EltSize = TD->getTypeAllocSize(EltTy);
568 Offset -= Layout->getElementOffset(EltIdx);
569 } else if (const ArrayType *AT = dyn_cast(T)) {
570 EltTy = AT->getElementType();
571 EltSize = TD->getTypeAllocSize(EltTy);
572 Offset %= EltSize;
573 } else {
574 return false;
575 }
576 if (Offset == 0 && (Size == 0 || EltSize == Size))
577 return true;
578 // Check if the component spans multiple elements.
579 if (Offset + Size > EltSize)
580 return false;
581 return TypeHasComponent(EltTy, Offset, Size);
582 }
583
584 /// RewriteForScalarRepl - Alloca AI is being split into NewElts, so rewrite
585 /// the instruction I, which references it, to use the separate elements.
586 /// Offset indicates the position within AI that is referenced by this
587 /// instruction.
588 void SROA::RewriteForScalarRepl(Instruction *I, AllocaInst *AI, uint64_t Offset,
589 SmallVector &NewElts) {
590 for (Value::use_iterator UI = I->use_begin(), E = I->use_end(); UI != E; ) {
717591 Instruction *User = cast(*UI++);
718 if (BitCastInst *BCU = dyn_cast(User)) {
719 RewriteBitCastUserOfAlloca(BCU, AI, NewElts);
720 if (BCU->use_empty()) BCU->eraseFromParent();
721 continue;
722 }
723
724 if (MemIntrinsic *MI = dyn_cast(User)) {
725 // This must be memcpy/memmove/memset of the entire aggregate.
726 // Split into one per element.
727 RewriteMemIntrinUserOfAlloca(MI, BCInst, AI, NewElts);
728 continue;
729 }
730
731 if (StoreInst *SI = dyn_cast(User)) {
732 // If this is a store of the entire alloca from an integer, rewrite it.
733 RewriteStoreUserOfWholeAlloca(SI, AI, NewElts);
734 continue;
735 }
736
737 if (LoadInst *LI = dyn_cast(User)) {
738 // If this is a load of the entire alloca to an integer, rewrite it.
739 RewriteLoadUserOfWholeAlloca(LI, AI, NewElts);
740 continue;
741 }
742
743 // Otherwise it must be some other user of a gep of the first pointer. Just
744 // leave these alone.
745 continue;
746 }
592
593 if (BitCastInst *BC = dyn_cast(User)) {
594 if (BC->getOperand(0) == AI)
595 BC->setOperand(0, NewElts[0]);
596 // If the bitcast type now matches the operand type, it will be removed
597 // after processing its uses.
598 RewriteForScalarRepl(BC, AI, Offset, NewElts);
599 } else if (GetElementPtrInst *GEPI = dyn_cast(User)) {
600 RewriteGEP(GEPI, AI, Offset, NewElts);
601 } else if (MemIntrinsic *MI = dyn_cast(User)) {
602 ConstantInt *Length = dyn_cast(MI->getLength());
603 uint64_t MemSize = Length->getZExtValue();
604 if (Offset == 0 &&
605 MemSize == TD->getTypeAllocSize(AI->getAllocatedType()))
606 RewriteMemIntrinUserOfAlloca(MI, I, AI, NewElts);
607 } else if (LoadInst *LI = dyn_cast(User)) {
608 const Type *LIType = LI->getType();
609 if (LIType == AI->getAllocatedType()) {
610 // Replace:
611 // %res = load { i32, i32 }* %alloc
612 // with:
613 // %load.0 = load i32* %alloc.0
614 // %insert.0 insertvalue { i32, i32 } zeroinitializer, i32 %load.0, 0
615 // %load.1 = load i32* %alloc.1
616 // %insert = insertvalue { i32, i32 } %insert.0, i32 %load.1, 1
617 // (Also works for arrays instead of structs)
618 Value *Insert = UndefValue::get(LIType);
619 for (unsigned i = 0, e = NewElts.size(); i != e; ++i) {
620 Value *Load = new LoadInst(NewElts[i], "load", LI);
621 Insert = InsertValueInst::Create(Insert, Load, i, "insert", LI);
622 }
623 LI->replaceAllUsesWith(Insert);
624 LI->eraseFromParent();
625 } else if (isa(LIType) &&
626 TD->getTypeAllocSize(LIType) ==
627 TD->getTypeAllocSize(AI->getAllocatedType())) {
628 // If this is a load of the entire alloca to an integer, rewrite it.
629 RewriteLoadUserOfWholeAlloca(LI, AI, NewElts);
630 }
631 } else if (StoreInst *SI = dyn_cast(User)) {
632 Value *Val = SI->getOperand(0);
633 const Type *SIType = Val->getType();
634 if (SIType == AI->getAllocatedType()) {
635 // Replace:
636 // store { i32, i32 } %val, { i32, i32 }* %alloc
637 // with:
638 // %val.0 = extractvalue { i32, i32 } %val, 0
639 // store i32 %val.0, i32* %alloc.0
640 // %val.1 = extractvalue { i32, i32 } %val, 1
641 // store i32 %val.1, i32* %alloc.1
642 // (Also works for arrays instead of structs)
643 for (unsigned i = 0, e = NewElts.size(); i != e; ++i) {
644 Value *Extract = ExtractValueInst::Create(Val, i, Val->getName(), SI);
645 new StoreInst(Extract, NewElts[i], SI);
646 }
647 SI->eraseFromParent();
648 } else if (isa(SIType) &&
649 TD->getTypeAllocSize(SIType) ==
650 TD->getTypeAllocSize(AI->getAllocatedType())) {
651 // If this is a store of the entire alloca from an integer, rewrite it.
652 RewriteStoreUserOfWholeAlloca(SI, AI, NewElts);
653 }
654 }
655 }
656 // Delete unused instructions and identity bitcasts.
657 if (I->use_empty())
658 I->eraseFromParent();
659 else if (BitCastInst *BC = dyn_cast(I)) {
660 if (BC->getDestTy() == BC->getSrcTy()) {
661 BC->replaceAllUsesWith(BC->getOperand(0));
662 BC->eraseFromParent();
663 }
664 }
665 }
666
667 /// FindElementAndOffset - Return the index of the element containing Offset
668 /// within the specified type, which must be either a struct or an array.
669 /// Sets T to the type of the element and Offset to the offset within that
670 /// element.
671 unsigned SROA::FindElementAndOffset(const Type *&T, uint64_t &Offset) {
672 unsigned Idx = 0;
673 if (const StructType *ST = dyn_cast(T)) {
674 const StructLayout *Layout = TD->getStructLayout(ST);
675 Idx = Layout->getElementContainingOffset(Offset);
676 T = ST->getContainedType(Idx);
677 Offset -= Layout->getElementOffset(Idx);
678 } else {
679 const ArrayType *AT = dyn_cast(T);
680 assert(AT && "unexpected type for scalar replacement");
681 T = AT->getElementType();
682 uint64_t EltSize = TD->getTypeAllocSize(T);
683 Idx = (unsigned)(Offset / EltSize);
684 Offset -= Idx * EltSize;
685 }
686 return Idx;
687 }
688
689 /// RewriteGEP - Check if this GEP instruction moves the pointer across
690 /// elements of the alloca that are being split apart, and if so, rewrite
691 /// the GEP to be relative to the new element.
692 void SROA::RewriteGEP(GetElementPtrInst *GEPI, AllocaInst *AI, uint64_t Offset,
693 SmallVector &NewElts) {
694 Instruction *Val = GEPI;
695
696 uint64_t OldOffset = Offset;
697 SmallVector Indices(GEPI->op_begin() + 1, GEPI->op_end());
698 Offset += TD->getIndexedOffset(GEPI->getPointerOperandType(),
699 &Indices[0], Indices.size());
700
701 const Type *T = AI->getAllocatedType();
702 unsigned OldIdx = FindElementAndOffset(T, OldOffset);
703 if (GEPI->getOperand(0) == AI)
704 OldIdx = ~0U; // Force the GEP to be rewritten.
705
706 T = AI->getAllocatedType();
707 uint64_t EltOffset = Offset;
708 unsigned Idx = FindElementAndOffset(T, EltOffset);
709
710 // If this GEP moves the pointer across elements of the alloca that are
711 // being split, then it needs to be rewritten.
712 if (Idx != OldIdx) {
713 const Type *i32Ty = Type::getInt32Ty(AI->getContext());
714 SmallVector NewArgs;
715 NewArgs.push_back(Constant::getNullValue(i32Ty));
716 while (EltOffset != 0) {
717 unsigned EltIdx = FindElementAndOffset(T, EltOffset);
718 NewArgs.push_back(ConstantInt::get(i32Ty, EltIdx));
719 }
720 if (NewArgs.size() > 1) {
721 Val = GetElementPtrInst::CreateInBounds(NewElts[Idx], NewArgs.begin(),
722 NewArgs.end(), "", GEPI);
723 Val->takeName(GEPI);
724 if (Val->getType() != GEPI->getType())
725 Val = new BitCastInst(Val, GEPI->getType(), Val->getNameStr(), GEPI);
726 } else {
727 Val = NewElts[Idx];
728 // Insert a new bitcast. If the types match, it will be removed after
729 // handling all of its uses.
730 Val = new BitCastInst(Val, GEPI->getType(), Val->getNameStr(), GEPI);
731 Val->takeName(GEPI);
732 }
733
734 GEPI->replaceAllUsesWith(Val);
735 GEPI->eraseFromParent();
736 }
737
738 RewriteForScalarRepl(Val, AI, Offset, NewElts);
747739 }
748740
749741 /// RewriteMemIntrinUserOfAlloca - MI is a memcpy/memset/memmove from or to AI.
750742 /// Rewrite it to copy or set the elements of the scalarized memory.
751 void SROA::RewriteMemIntrinUserOfAlloca(MemIntrinsic *MI, Instruction *BCInst,
743 void SROA::RewriteMemIntrinUserOfAlloca(MemIntrinsic *MI, Instruction *Inst,
752744 AllocaInst *AI,
753745 SmallVector &NewElts) {
754746
760752 LLVMContext &Context = MI->getContext();
761753 unsigned MemAlignment = MI->getAlignment();
762754 if (MemTransferInst *MTI = dyn_cast(MI)) { // memmove/memcopy
763 if (BCInst == MTI->getRawDest())
755 if (Inst == MTI->getRawDest())
764756 OtherPtr = MTI->getRawSource();
765757 else {
766 assert(BCInst == MTI->getRawSource());
758 assert(Inst == MTI->getRawSource());
767759 OtherPtr = MTI->getRawDest();
768760 }
769761 }
797789 // Process each element of the aggregate.
798790 Value *TheFn = MI->getOperand(0);
799791 const Type *BytePtrTy = MI->getRawDest()->getType();
800 bool SROADest = MI->getRawDest() == BCInst;
792 bool SROADest = MI->getRawDest() == Inst;
801793
802794 Constant *Zero = Constant::getNullValue(Type::getInt32Ty(MI->getContext()));
803795
809801 if (OtherPtr) {
810802 Value *Idx[2] = { Zero,
811803 ConstantInt::get(Type::getInt32Ty(MI->getContext()), i) };
812 OtherElt = GetElementPtrInst::Create(OtherPtr, Idx, Idx + 2,
804 OtherElt = GetElementPtrInst::CreateInBounds(OtherPtr, Idx, Idx + 2,
813805 OtherPtr->getNameStr()+"."+Twine(i),
814 MI);
806 MI);
815807 uint64_t EltOffset;
816808 const PointerType *OtherPtrTy = cast(OtherPtr->getType());
817809 if (const StructType *ST =
936928 // Extract each element out of the integer according to its structure offset
937929 // and store the element value to the individual alloca.
938930 Value *SrcVal = SI->getOperand(0);
939 const Type *AllocaEltTy = AI->getType()->getElementType();
931 const Type *AllocaEltTy = AI->getAllocatedType();
940932 uint64_t AllocaSizeBits = TD->getTypeAllocSizeInBits(AllocaEltTy);
941933
942 // If this isn't a store of an integer to the whole alloca, it may be a store
943 // to the first element. Just ignore the store in this case and normal SROA
944 // will handle it.
945 if (!isa(SrcVal->getType()) ||
946 TD->getTypeAllocSizeInBits(SrcVal->getType()) != AllocaSizeBits)
947 return;
948934 // Handle tail padding by extending the operand
949935 if (TD->getTypeSizeInBits(SrcVal->getType()) != AllocaSizeBits)
950936 SrcVal = new ZExtInst(SrcVal,
10581044 SmallVector &NewElts) {
10591045 // Extract each element out of the NewElts according to its structure offset
10601046 // and form the result value.
1061 const Type *AllocaEltTy = AI->getType()->getElementType();
1047 const Type *AllocaEltTy = AI->getAllocatedType();
10621048 uint64_t AllocaSizeBits = TD->getTypeAllocSizeInBits(AllocaEltTy);
1063
1064 // If this isn't a load of the whole alloca to an integer, it may be a load
1065 // of the first element. Just ignore the load in this case and normal SROA
1066 // will handle it.
1067 if (!isa(LI->getType()) ||
1068 TD->getTypeAllocSizeInBits(LI->getType()) != AllocaSizeBits)
1069 return;
10701049
10711050 DEBUG(errs() << "PROMOTING LOAD OF WHOLE ALLOCA: " << *AI << '\n' << *LI
10721051 << '\n');
11411120 LI->eraseFromParent();
11421121 }
11431122
1144
11451123 /// HasPadding - Return true if the specified type has any structure or
11461124 /// alignment padding, false otherwise.
11471125 static bool HasPadding(const Type *Ty, const TargetData &TD) {
11911169 // the users are safe to transform.
11921170 AllocaInfo Info;
11931171
1194 for (Value::use_iterator I = AI->use_begin(), E = AI->use_end();
1195 I != E; ++I) {
1196 isSafeUseOfAllocation(cast(*I), AI, Info);
1197 if (Info.isUnsafe) {
1198 DEBUG(errs() << "Cannot transform: " << *AI << "\n due to user: "
1199 << **I << '\n');
1200 return 0;
1201 }
1172 isSafeForScalarRepl(AI, AI, 0, 0, Info);
1173 if (Info.isUnsafe) {
1174 DEBUG(errs() << "Cannot transform: " << *AI << '\n');
1175 return 0;
12021176 }
12031177
12041178 // Okay, we know all the users are promotable. If the aggregate is a memcpy
12071181 // types, but may actually be used. In these cases, we refuse to promote the
12081182 // struct.
12091183 if (Info.isMemCpySrc && Info.isMemCpyDst &&
1210 HasPadding(AI->getType()->getElementType(), *TD))
1184 HasPadding(AI->getAllocatedType(), *TD))
12111185 return 0;
12121186
12131187 // If we require cleanup, return 1, otherwise return 3.
12441218 // Insert the new GEP instructions, which are properly indexed.
12451219 SmallVector Indices(GEPI->op_begin()+1, GEPI->op_end());
12461220 Indices[1] = Constant::getNullValue(Type::getInt32Ty(GEPI->getContext()));
1247 Value *ZeroIdx = GetElementPtrInst::Create(GEPI->getOperand(0),
1248 Indices.begin(),
1249 Indices.end(),
1250 GEPI->getName()+".0", GEPI);
1221 Value *ZeroIdx = GetElementPtrInst::CreateInBounds(GEPI->getOperand(0),
1222 Indices.begin(),
1223 Indices.end(),
1224 GEPI->getName()+".0",GEPI);
12511225 Indices[1] = ConstantInt::get(Type::getInt32Ty(GEPI->getContext()), 1);
1252 Value *OneIdx = GetElementPtrInst::Create(GEPI->getOperand(0),
1253 Indices.begin(),
1254 Indices.end(),
1255 GEPI->getName()+".1", GEPI);
1226 Value *OneIdx = GetElementPtrInst::CreateInBounds(GEPI->getOperand(0),
1227 Indices.begin(),
1228 Indices.end(),
1229 GEPI->getName()+".1", GEPI);
12561230 // Replace all loads of the variable index GEP with loads from both
12571231 // indexes and a select.
12581232 while (!GEPI->use_empty()) {
12631237 LI->replaceAllUsesWith(R);
12641238 LI->eraseFromParent();
12651239 }
1266 GEPI->eraseFromParent();
1267 }
1268
1240 }
12691241
12701242 /// CleanupAllocaUsers - If SROA reported that it can promote the specified
12711243 /// allocation, but only if cleaned up, perform the cleanups required.
1272 void SROA::CleanupAllocaUsers(AllocaInst *AI) {
1244 void SROA::CleanupAllocaUsers(Value *V) {
12731245 // At this point, we know that the end result will be SROA'd and promoted, so
12741246 // we can insert ugly code if required so long as sroa+mem2reg will clean it
12751247 // up.
1276 for (Value::use_iterator UI = AI->use_begin(), E = AI->use_end();
1248 for (Value::use_iterator UI = V->use_begin(), E = V->use_end();
12771249 UI != E; ) {
12781250 User *U = *UI++;
1279 if (GetElementPtrInst *GEPI = dyn_cast(U))
1251 if (isa(U)) {
1252 CleanupAllocaUsers(U);
1253 } else if (GetElementPtrInst *GEPI = dyn_cast(U)) {
12801254 CleanupGEP(GEPI);
1281 else {
1255 CleanupAllocaUsers(GEPI);
1256 if (GEPI->use_empty()) GEPI->eraseFromParent();
1257 } else {
12821258 Instruction *I = cast(U);
12831259 SmallVector DbgInUses;
12841260 if (!isa(I) && OnlyUsedByDbgInfoIntrinsics(I, &DbgInUses)) {
13941370
13951371 // Compute the offset that this GEP adds to the pointer.
13961372 SmallVector Indices(GEP->op_begin()+1, GEP->op_end());
1397 uint64_t GEPOffset = TD->getIndexedOffset(GEP->getOperand(0)->getType(),
1373 uint64_t GEPOffset = TD->getIndexedOffset(GEP->getPointerOperandType(),
13981374 &Indices[0], Indices.size());
13991375 // See if all uses can be converted.
14001376 if (!CanConvertToScalar(GEP, IsNotTrivial, VecTy, SawVec,Offset+GEPOffset,
14561432 if (GetElementPtrInst *GEP = dyn_cast(User)) {
14571433 // Compute the offset that this GEP adds to the pointer.
14581434 SmallVector Indices(GEP->op_begin()+1, GEP->op_end());
1459 uint64_t GEPOffset = TD->getIndexedOffset(GEP->getOperand(0)->getType(),
1435 uint64_t GEPOffset = TD->getIndexedOffset(GEP->getPointerOperandType(),
14601436 &Indices[0], Indices.size());
14611437 ConvertUsesToScalar(GEP, NewAI, Offset+GEPOffset*8);
14621438 GEP->eraseFromParent();
0 ; RUN: opt < %s -scalarrepl -S | FileCheck %s
1 ; Radar 7441282
2
3 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:64:64-v128:128:128-a0:0:32-n32"
4 target triple = "thumbv7-apple-darwin10"
5
6 %struct.__neon_int16x8x2_t = type { <8 x i16>, <8 x i16> }
7 %struct.int16x8_t = type { <8 x i16> }
8 %struct.int16x8x2_t = type { [2 x %struct.int16x8_t] }
9 %union..0anon = type { %struct.int16x8x2_t }
10
11 define arm_apcscc void @test(<8 x i16> %tmp.0, %struct.int16x8x2_t* %dst) nounwind {
12 ; CHECK: @test
13 ; CHECK-NOT: alloca
14 ; CHECK: "alloca point"
15 entry:
16 %tmp_addr = alloca %struct.int16x8_t ; <%struct.int16x8_t*> [#uses=3]
17 %dst_addr = alloca %struct.int16x8x2_t* ; <%struct.int16x8x2_t**> [#uses=2]
18 %__rv = alloca %union..0anon ; <%union..0anon*> [#uses=2]
19 %__bx = alloca %struct.int16x8_t ; <%struct.int16x8_t*> [#uses=2]
20 %__ax = alloca %struct.int16x8_t ; <%struct.int16x8_t*> [#uses=2]
21 %tmp2 = alloca %struct.int16x8x2_t ; <%struct.int16x8x2_t*> [#uses=2]
22 %0 = alloca %struct.int16x8x2_t ; <%struct.int16x8x2_t*> [#uses=2]
23 %"alloca point" = bitcast i32 0 to i32 ; [#uses=0]
24 %1 = getelementptr inbounds %struct.int16x8_t* %tmp_addr, i32 0, i32 0 ; <<8 x i16>*> [#uses=1]
25 store <8 x i16> %tmp.0, <8 x i16>* %1
26 store %struct.int16x8x2_t* %dst, %struct.int16x8x2_t** %dst_addr
27 %2 = getelementptr inbounds %struct.int16x8_t* %__ax, i32 0, i32 0 ; <<8 x i16>*> [#uses=1]
28 %3 = getelementptr inbounds %struct.int16x8_t* %tmp_addr, i32 0, i32 0 ; <<8 x i16>*> [#uses=1]
29 %4 = load <8 x i16>* %3, align 16 ; <<8 x i16>> [#uses=1]
30 store <8 x i16> %4, <8 x i16>* %2, align 16
31 %5 = getelementptr inbounds %struct.int16x8_t* %__bx, i32 0, i32 0 ; <<8 x i16>*> [#uses=1]
32 %6 = getelementptr inbounds %struct.int16x8_t* %tmp_addr, i32 0, i32 0 ; <<8 x i16>*> [#uses=1]
33 %7 = load <8 x i16>* %6, align 16 ; <<8 x i16>> [#uses=1]
34 store <8 x i16> %7, <8 x i16>* %5, align 16
35 %8 = getelementptr inbounds %struct.int16x8_t* %__ax, i32 0, i32 0 ; <<8 x i16>*> [#uses=1]
36 %9 = load <8 x i16>* %8, align 16 ; <<8 x i16>> [#uses=2]
37 %10 = getelementptr inbounds %struct.int16x8_t* %__bx, i32 0, i32 0 ; <<8 x i16>*> [#uses=1]
38 %11 = load <8 x i16>* %10, align 16 ; <<8 x i16>> [#uses=2]
39 %12 = getelementptr inbounds %union..0anon* %__rv, i32 0, i32 0 ; <%struct.int16x8x2_t*> [#uses=1]
40 %13 = bitcast %struct.int16x8x2_t* %12 to %struct.__neon_int16x8x2_t* ; <%struct.__neon_int16x8x2_t*> [#uses=2]
41 %14 = shufflevector <8 x i16> %9, <8 x i16> %11, <8 x i32> ; <<8 x i16>> [#uses=1]
42 %15 = getelementptr inbounds %struct.__neon_int16x8x2_t* %13, i32 0, i32 0 ; <<8 x i16>*> [#uses=1]
43 store <8 x i16> %14, <8 x i16>* %15
44 %16 = shufflevector <8 x i16> %9, <8 x i16> %11, <8 x i32> ; <<8 x i16>> [#uses=1]
45 %17 = getelementptr inbounds %struct.__neon_int16x8x2_t* %13, i32 0, i32 1 ; <<8 x i16>*> [#uses=1]
46 store <8 x i16> %16, <8 x i16>* %17
47 %18 = getelementptr inbounds %union..0anon* %__rv, i32 0, i32 0 ; <%struct.int16x8x2_t*> [#uses=1]
48 %19 = bitcast %struct.int16x8x2_t* %0 to i8* ; [#uses=1]
49 %20 = bitcast %struct.int16x8x2_t* %18 to i8* ; [#uses=1]
50 call void @llvm.memcpy.i32(i8* %19, i8* %20, i32 32, i32 16)
51 %tmp21 = bitcast %struct.int16x8x2_t* %tmp2 to i8* ; [#uses=1]
52 %21 = bitcast %struct.int16x8x2_t* %0 to i8* ; [#uses=1]
53 call void @llvm.memcpy.i32(i8* %tmp21, i8* %21, i32 32, i32 16)
54 %22 = load %struct.int16x8x2_t** %dst_addr, align 4 ; <%struct.int16x8x2_t*> [#uses=1]
55 %23 = bitcast %struct.int16x8x2_t* %22 to i8* ; [#uses=1]
56 %tmp22 = bitcast %struct.int16x8x2_t* %tmp2 to i8* ; [#uses=1]
57 call void @llvm.memcpy.i32(i8* %23, i8* %tmp22, i32 32, i32 16)
58 br label %return
59
60 ; CHECK: store <8 x i16>
61 ; CHECK: store <8 x i16>
62
63 return: ; preds = %entry
64 ret void
65 }
66
67 declare void @llvm.memcpy.i32(i8* nocapture, i8* nocapture, i32, i32) nounwind