llvm.org GIT mirror llvm / 05783c1
Modernize repmovsb implementation of x86 memcpy and allow runtime sizes. Summary: This is a prerequisite to RFC http://lists.llvm.org/pipermail/llvm-dev/2019-April/131973.html Reviewers: courbet Subscribers: hiraditya, llvm-commits Tags: #llvm Differential Revision: https://reviews.llvm.org/D61593 Fix typo. Turn this patch into an NFC. Addressing comments git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@360050 91177308-0d34-0410-b5e6-96231b3b80d8 Guillaume Chatelet 1 year, 5 months ago
1 changed file(s) with 124 addition(s) and 103 deletion(s). Raw diff Collapse all Expand all
4141 return true;
4242 return false;
4343 }
44
45 namespace {
46
47 // Represents a cover of a buffer of Size bytes with Count() blocks of type AVT
48 // (of size UBytes() bytes), as well as how many bytes remain (BytesLeft() is
49 // always smaller than the block size).
50 struct RepMovsRepeats {
51 RepMovsRepeats(uint64_t Size) : Size(Size) {}
52
53 uint64_t Count() const { return Size / UBytes(); }
54 uint64_t BytesLeft() const { return Size % UBytes(); }
55 uint64_t UBytes() const { return AVT.getSizeInBits() / 8; }
56
57 const uint64_t Size;
58 MVT AVT = MVT::i8;
59 };
60
61 } // namespace
6244
6345 SDValue X86SelectionDAGInfo::EmitTargetCodeForMemset(
6446 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Val,
199181 return Chain;
200182 }
201183
184 /// Emit a single REP MOVS{B,W,D,Q} instruction.
185 static SDValue emitRepmovs(const X86Subtarget &Subtarget, SelectionDAG &DAG,
186 const SDLoc &dl, SDValue Chain, SDValue Dst,
187 SDValue Src, SDValue Size, MVT AVT) {
188 const bool Use64BitRegs = Subtarget.isTarget64BitLP64();
189 const unsigned CX = Use64BitRegs ? X86::RCX : X86::ECX;
190 const unsigned DI = Use64BitRegs ? X86::RDI : X86::EDI;
191 const unsigned SI = Use64BitRegs ? X86::RSI : X86::ESI;
192
193 SDValue InFlag;
194 Chain = DAG.getCopyToReg(Chain, dl, CX, Size, InFlag);
195 InFlag = Chain.getValue(1);
196 Chain = DAG.getCopyToReg(Chain, dl, DI, Dst, InFlag);
197 InFlag = Chain.getValue(1);
198 Chain = DAG.getCopyToReg(Chain, dl, SI, Src, InFlag);
199 InFlag = Chain.getValue(1);
200
201 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
202 SDValue Ops[] = {Chain, DAG.getValueType(AVT), InFlag};
203 return DAG.getNode(X86ISD::REP_MOVS, dl, Tys, Ops);
204 }
205
206 /// Emit a single REP MOVSB instruction for a particular constant size.
207 static SDValue emitRepmovsB(const X86Subtarget &Subtarget, SelectionDAG &DAG,
208 const SDLoc &dl, SDValue Chain, SDValue Dst,
209 SDValue Src, uint64_t Size) {
210 return emitRepmovs(Subtarget, DAG, dl, Chain, Dst, Src,
211 DAG.getIntPtrConstant(Size, dl), MVT::i8);
212 }
213
214 /// Returns the best type to use with repmovs depending on alignment.
215 static MVT getOptimalRepmovsType(const X86Subtarget &Subtarget,
216 uint64_t Align) {
217 assert((Align != 0) && "Align is normalized");
218 assert(isPowerOf2_64(Align) && "Align is a power of 2");
219 switch (Align) {
220 case 1:
221 return MVT::i8;
222 case 2:
223 return MVT::i16;
224 case 4:
225 return MVT::i32;
226 default:
227 return Subtarget.is64Bit() ? MVT::i64 : MVT::i32;
228 }
229 }
230
231 /// Returns a REP MOVS instruction, possibly with a few load/stores to implement
232 /// a constant size memory copy. In some cases where we know REP MOVS is
233 /// inefficient we return an empty SDValue so the calling code can either
234 /// generate a load/store sequence or call the runtime memcpy function.
235 static SDValue emitConstantSizeRepmov(
236 SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &dl,
237 SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, EVT SizeVT,
238 unsigned Align, bool isVolatile, bool AlwaysInline,
239 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) {
240
241 /// TODO: Revisit next line: big copy with ERMSB on march >= haswell are very
242 /// efficient.
243 if (!AlwaysInline && Size > Subtarget.getMaxInlineSizeThreshold())
244 return SDValue();
245
246 /// If we have enhanced repmovs we use it.
247 if (Subtarget.hasERMSB())
248 return emitRepmovsB(Subtarget, DAG, dl, Chain, Dst, Src, Size);
249
250 assert(!Subtarget.hasERMSB() && "No efficient RepMovs");
251 /// We assume runtime memcpy will do a better job for unaligned copies when
252 /// ERMS is not present.
253 if (!AlwaysInline && (Align & 3) != 0)
254 return SDValue();
255
256 const MVT BlockType = getOptimalRepmovsType(Subtarget, Align);
257 const uint64_t BlockBytes = BlockType.getSizeInBits() / 8;
258 const uint64_t BlockCount = Size / BlockBytes;
259 const uint64_t BytesLeft = Size % BlockBytes;
260 SDValue RepMovs =
261 emitRepmovs(Subtarget, DAG, dl, Chain, Dst, Src,
262 DAG.getIntPtrConstant(BlockCount, dl), BlockType);
263
264 /// RepMov can process the whole length.
265 if (BytesLeft == 0)
266 return RepMovs;
267
268 assert(BytesLeft && "We have leftover at this point");
269
270 /// In case we optimize for size we use repmovsb even if it's less efficient
271 /// so we can save the loads/stores of the leftover.
272 if (DAG.getMachineFunction().getFunction().hasMinSize())
273 return emitRepmovsB(Subtarget, DAG, dl, Chain, Dst, Src, Size);
274
275 // Handle the last 1 - 7 bytes.
276 SmallVector Results;
277 Results.push_back(RepMovs);
278 unsigned Offset = Size - BytesLeft;
279 EVT DstVT = Dst.getValueType();
280 EVT SrcVT = Src.getValueType();
281 Results.push_back(DAG.getMemcpy(
282 Chain, dl,
283 DAG.getNode(ISD::ADD, dl, DstVT, Dst, DAG.getConstant(Offset, dl, DstVT)),
284 DAG.getNode(ISD::ADD, dl, SrcVT, Src, DAG.getConstant(Offset, dl, SrcVT)),
285 DAG.getConstant(BytesLeft, dl, SizeVT), Align, isVolatile,
286 /*AlwaysInline*/ true, /*isTailCall*/ false,
287 DstPtrInfo.getWithOffset(Offset), SrcPtrInfo.getWithOffset(Offset)));
288 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Results);
289 }
290
202291 SDValue X86SelectionDAGInfo::EmitTargetCodeForMemcpy(
203292 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
204293 SDValue Size, unsigned Align, bool isVolatile, bool AlwaysInline,
205294 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
206 // This requires the copy size to be a constant, preferably
207 // within a subtarget-specific limit.
208 ConstantSDNode *ConstantSize = dyn_cast(Size);
209 const X86Subtarget &Subtarget =
210 DAG.getMachineFunction().getSubtarget();
211 if (!ConstantSize)
212 return SDValue();
213 RepMovsRepeats Repeats(ConstantSize->getZExtValue());
214 if (!AlwaysInline && Repeats.Size > Subtarget.getMaxInlineSizeThreshold())
215 return SDValue();
216
217 /// If not DWORD aligned, it is more efficient to call the library. However
218 /// if calling the library is not allowed (AlwaysInline), then soldier on as
219 /// the code generated here is better than the long load-store sequence we
220 /// would otherwise get.
221 if (!AlwaysInline && (Align & 3) != 0)
222 return SDValue();
223
224295 // If to a segment-relative address space, use the default lowering.
225 if (DstPtrInfo.getAddrSpace() >= 256 ||
226 SrcPtrInfo.getAddrSpace() >= 256)
227 return SDValue();
228
229 // If the base register might conflict with our physical registers, bail out.
296 if (DstPtrInfo.getAddrSpace() >= 256 || SrcPtrInfo.getAddrSpace() >= 256)
297 return SDValue();
298
299 // If the base registers conflict with our physical registers, use the default
300 // lowering.
230301 const MCPhysReg ClobberSet[] = {X86::RCX, X86::RSI, X86::RDI,
231302 X86::ECX, X86::ESI, X86::EDI};
232303 if (isBaseRegConflictPossible(DAG, ClobberSet))
233304 return SDValue();
234305
235 // If the target has enhanced REPMOVSB, then it's at least as fast to use
236 // REP MOVSB instead of REP MOVS{W,D,Q}, and it avoids having to handle
237 // BytesLeft.
238 if (!Subtarget.hasERMSB() && !(Align & 1)) {
239 if (Align & 2)
240 // WORD aligned
241 Repeats.AVT = MVT::i16;
242 else if (Align & 4)
243 // DWORD aligned
244 Repeats.AVT = MVT::i32;
245 else
246 // QWORD aligned
247 Repeats.AVT = Subtarget.is64Bit() ? MVT::i64 : MVT::i32;
248
249 if (Repeats.BytesLeft() > 0 &&
250 DAG.getMachineFunction().getFunction().hasMinSize()) {
251 // When aggressively optimizing for size, avoid generating the code to
252 // handle BytesLeft.
253 Repeats.AVT = MVT::i8;
254 }
255 }
256
257 bool Use64BitRegs = Subtarget.isTarget64BitLP64();
258 SDValue InFlag;
259 Chain = DAG.getCopyToReg(Chain, dl, Use64BitRegs ? X86::RCX : X86::ECX,
260 DAG.getIntPtrConstant(Repeats.Count(), dl), InFlag);
261 InFlag = Chain.getValue(1);
262 Chain = DAG.getCopyToReg(Chain, dl, Use64BitRegs ? X86::RDI : X86::EDI,
263 Dst, InFlag);
264 InFlag = Chain.getValue(1);
265 Chain = DAG.getCopyToReg(Chain, dl, Use64BitRegs ? X86::RSI : X86::ESI,
266 Src, InFlag);
267 InFlag = Chain.getValue(1);
268
269 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
270 SDValue Ops[] = { Chain, DAG.getValueType(Repeats.AVT), InFlag };
271 SDValue RepMovs = DAG.getNode(X86ISD::REP_MOVS, dl, Tys, Ops);
272
273 SmallVector Results;
274 Results.push_back(RepMovs);
275 if (Repeats.BytesLeft()) {
276 // Handle the last 1 - 7 bytes.
277 unsigned Offset = Repeats.Size - Repeats.BytesLeft();
278 EVT DstVT = Dst.getValueType();
279 EVT SrcVT = Src.getValueType();
280 EVT SizeVT = Size.getValueType();
281 Results.push_back(DAG.getMemcpy(Chain, dl,
282 DAG.getNode(ISD::ADD, dl, DstVT, Dst,
283 DAG.getConstant(Offset, dl,
284 DstVT)),
285 DAG.getNode(ISD::ADD, dl, SrcVT, Src,
286 DAG.getConstant(Offset, dl,
287 SrcVT)),
288 DAG.getConstant(Repeats.BytesLeft(), dl,
289 SizeVT),
290 Align, isVolatile, AlwaysInline, false,
291 DstPtrInfo.getWithOffset(Offset),
292 SrcPtrInfo.getWithOffset(Offset)));
293 }
294
295 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Results);
296 }
306 const X86Subtarget &Subtarget =
307 DAG.getMachineFunction().getSubtarget();
308
309 /// Handle constant sizes,
310 if (ConstantSDNode *ConstantSize = dyn_cast(Size))
311 return emitConstantSizeRepmov(DAG, Subtarget, dl, Chain, Dst, Src,
312 ConstantSize->getZExtValue(),
313 Size.getValueType(), Align, isVolatile,
314 AlwaysInline, DstPtrInfo, SrcPtrInfo);
315
316 return SDValue();
317 }