llvm.org GIT mirror llvm / 1b6a51a
Support: Rewrite Windows implementation of sys::fs::rename to be more POSIXy. The current implementation of rename uses ReplaceFile if the destination file already exists. According to the documentation for ReplaceFile, the source file is opened without a sharing mode. This means that there is a short interval of time between when ReplaceFile renames the file and when it closes the file during which the destination file cannot be opened. This behaviour is not POSIX compliant because rename is supposed to be atomic. It was also causing intermittent link failures when linking with a ThinLTO cache; the ThinLTO cache implementation expects all cache files to be openable. This patch addresses that problem by re-implementing rename using CreateFile and SetFileInformationByHandle. It is roughly a reimplementation of ReplaceFile with a better sharing policy as well as support for renaming in the case where the destination file does not exist. This implementation is still not fully POSIX. Specifically in the case where the destination file is open at the point when rename is called, there will be a short interval of time during which the destination file will not exist. It isn't clear whether it is possible to avoid this using the Windows API. Differential Revision: https://reviews.llvm.org/D38570 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@315079 91177308-0d34-0410-b5e6-96231b3b80d8 Peter Collingbourne 1 year, 11 months ago
3 changed file(s) with 194 addition(s) and 69 deletion(s). Raw diff Collapse all Expand all
342342 /// platform-specific error code.
343343 std::error_code remove_directories(const Twine &path, bool IgnoreErrors = true);
344344
345 /// @brief Rename \a from to \a to. Files are renamed as if by POSIX rename().
345 /// @brief Rename \a from to \a to.
346 ///
347 /// Files are renamed as if by POSIX rename(), except that on Windows there may
348 /// be a short interval of time during which the destination file does not
349 /// exist.
346350 ///
347351 /// @param from The path to rename from.
348352 /// @param to The path to rename to. This is created.
358358 return is_local_internal(FinalPath, Result);
359359 }
360360
361 std::error_code rename(const Twine &from, const Twine &to) {
361 static std::error_code rename_internal(HANDLE FromHandle, const Twine &To,
362 bool ReplaceIfExists) {
363 SmallVector ToWide;
364 if (auto EC = widenPath(To, ToWide))
365 return EC;
366
367 std::vector RenameInfoBuf(sizeof(FILE_RENAME_INFO) - sizeof(wchar_t) +
368 (ToWide.size() * sizeof(wchar_t)));
369 FILE_RENAME_INFO &RenameInfo =
370 *reinterpret_cast(RenameInfoBuf.data());
371 RenameInfo.ReplaceIfExists = ReplaceIfExists;
372 RenameInfo.RootDirectory = 0;
373 RenameInfo.FileNameLength = ToWide.size();
374 std::copy(ToWide.begin(), ToWide.end(), RenameInfo.FileName);
375
376 if (!SetFileInformationByHandle(FromHandle, FileRenameInfo, &RenameInfo,
377 RenameInfoBuf.size()))
378 return mapWindowsError(GetLastError());
379
380 return std::error_code();
381 }
382
383 std::error_code rename(const Twine &From, const Twine &To) {
362384 // Convert to utf-16.
363 SmallVector wide_from;
364 SmallVector wide_to;
365 if (std::error_code ec = widenPath(from, wide_from))
366 return ec;
367 if (std::error_code ec = widenPath(to, wide_to))
368 return ec;
369
370 std::error_code ec = std::error_code();
371
372 // Retry while we see recoverable errors.
373 // System scanners (eg. indexer) might open the source file when it is written
374 // and closed.
375
376 bool TryReplace = true;
377
378 for (int i = 0; i < 2000; i++) {
379 if (i > 0)
380 ::Sleep(1);
381
382 if (TryReplace) {
383 // Try ReplaceFile first, as it is able to associate a new data stream
384 // with the destination even if the destination file is currently open.
385 if (::ReplaceFileW(wide_to.data(), wide_from.data(), NULL, 0, NULL, NULL))
386 return std::error_code();
387
388 DWORD ReplaceError = ::GetLastError();
389 ec = mapWindowsError(ReplaceError);
390
391 // If ReplaceFileW returned ERROR_UNABLE_TO_MOVE_REPLACEMENT or
392 // ERROR_UNABLE_TO_MOVE_REPLACEMENT_2, retry but only use MoveFileExW().
393 if (ReplaceError == ERROR_UNABLE_TO_MOVE_REPLACEMENT ||
394 ReplaceError == ERROR_UNABLE_TO_MOVE_REPLACEMENT_2) {
395 TryReplace = false;
385 SmallVector WideFrom;
386 SmallVector WideTo;
387 if (std::error_code EC = widenPath(From, WideFrom))
388 return EC;
389 if (std::error_code EC = widenPath(To, WideTo))
390 return EC;
391
392 ScopedFileHandle FromHandle;
393 // Retry this a few times to defeat badly behaved file system scanners.
394 for (unsigned Retry = 0; Retry != 200; ++Retry) {
395 if (Retry != 0)
396 ::Sleep(10);
397 FromHandle =
398 ::CreateFileW(WideFrom.begin(), GENERIC_READ | DELETE,
399 FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
400 NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
401 if (FromHandle)
402 break;
403 }
404 if (!FromHandle)
405 return mapWindowsError(GetLastError());
406
407 // We normally expect this loop to succeed after a few iterations. If it
408 // requires more than 200 tries, it's more likely that the failures are due to
409 // a true error, so stop trying.
410 for (unsigned Retry = 0; Retry != 200; ++Retry) {
411 auto EC = rename_internal(FromHandle, To, true);
412 if (!EC || EC != errc::permission_denied)
413 return EC;
414
415 // The destination file probably exists and is currently open in another
416 // process, either because the file was opened without FILE_SHARE_DELETE or
417 // it is mapped into memory (e.g. using MemoryBuffer). Rename it in order to
418 // move it out of the way of the source file. Use FILE_FLAG_DELETE_ON_CLOSE
419 // to arrange for the destination file to be deleted when the other process
420 // closes it.
421 ScopedFileHandle ToHandle(
422 ::CreateFileW(WideTo.begin(), GENERIC_READ | DELETE,
423 FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
424 NULL, OPEN_EXISTING,
425 FILE_ATTRIBUTE_NORMAL | FILE_FLAG_DELETE_ON_CLOSE, NULL));
426 if (!ToHandle) {
427 auto EC = mapWindowsError(GetLastError());
428 // Another process might have raced with us and moved the existing file
429 // out of the way before we had a chance to open it. If that happens, try
430 // to rename the source file again.
431 if (EC == errc::no_such_file_or_directory)
396432 continue;
433 return EC;
434 }
435
436 BY_HANDLE_FILE_INFORMATION FI;
437 if (!GetFileInformationByHandle(ToHandle, &FI))
438 return mapWindowsError(GetLastError());
439
440 // Try to find a unique new name for the destination file.
441 for (unsigned UniqueId = 0; UniqueId != 200; ++UniqueId) {
442 std::string TmpFilename = (To + ".tmp" + utostr(UniqueId)).str();
443 if (auto EC = rename_internal(ToHandle, TmpFilename, false)) {
444 if (EC == errc::file_exists || EC == errc::permission_denied) {
445 // Again, another process might have raced with us and moved the file
446 // before we could move it. Check whether this is the case, as it
447 // might have caused the permission denied error. If that was the
448 // case, we don't need to move it ourselves.
449 ScopedFileHandle ToHandle2(::CreateFileW(
450 WideTo.begin(), 0,
451 FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL,
452 OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL));
453 if (!ToHandle2) {
454 auto EC = mapWindowsError(GetLastError());
455 if (EC == errc::no_such_file_or_directory)
456 break;
457 return EC;
458 }
459 BY_HANDLE_FILE_INFORMATION FI2;
460 if (!GetFileInformationByHandle(ToHandle2, &FI2))
461 return mapWindowsError(GetLastError());
462 if (FI.nFileIndexHigh != FI2.nFileIndexHigh ||
463 FI.nFileIndexLow != FI2.nFileIndexLow ||
464 FI.dwVolumeSerialNumber != FI2.dwVolumeSerialNumber)
465 break;
466 continue;
467 }
468 return EC;
397469 }
398 // If ReplaceFileW returned ERROR_UNABLE_TO_REMOVE_REPLACED, retry
399 // using ReplaceFileW().
400 if (ReplaceError == ERROR_UNABLE_TO_REMOVE_REPLACED)
401 continue;
402 // We get ERROR_FILE_NOT_FOUND if the destination file is missing.
403 // MoveFileEx can handle this case.
404 if (ReplaceError != ERROR_ACCESS_DENIED &&
405 ReplaceError != ERROR_FILE_NOT_FOUND &&
406 ReplaceError != ERROR_SHARING_VIOLATION)
407 break;
470 break;
408471 }
409472
410 if (::MoveFileExW(wide_from.begin(), wide_to.begin(),
411 MOVEFILE_COPY_ALLOWED | MOVEFILE_REPLACE_EXISTING))
412 return std::error_code();
413
414 DWORD MoveError = ::GetLastError();
415 ec = mapWindowsError(MoveError);
416 if (MoveError != ERROR_ACCESS_DENIED) break;
417 }
418
419 return ec;
473 // Okay, the old destination file has probably been moved out of the way at
474 // this point, so try to rename the source file again. Still, another
475 // process might have raced with us to create and open the destination
476 // file, so we need to keep doing this until we succeed.
477 }
478
479 // The most likely root cause.
480 return errc::permission_denied;
420481 }
421482
422483 std::error_code resize_file(int FD, uint64_t Size) {
5151 ~ScopedFD() { Process::SafelyCloseFileDescriptor(FD); }
5252 };
5353
54 bool FDHasContent(int FD, StringRef Content) {
55 auto Buffer = MemoryBuffer::getOpenFile(FD, "", -1);
56 assert(Buffer);
57 return Buffer.get()->getBuffer() == Content;
58 }
59
60 bool FileHasContent(StringRef File, StringRef Content) {
61 int FD = 0;
62 auto EC = fs::openFileForRead(File, FD);
63 (void)EC;
64 assert(!EC);
65 ScopedFD EventuallyCloseIt(FD);
66 return FDHasContent(FD, Content);
67 }
68
5469 TEST(rename, FileOpenedForReadingCanBeReplaced) {
5570 // Create unique temporary directory for this test.
5671 SmallString<128> TestDirectory;
7893
7994 // We should still be able to read the old data through the existing
8095 // descriptor.
81 auto Buffer = MemoryBuffer::getOpenFile(ReadFD, TargetFileName, -1);
82 ASSERT_TRUE(static_cast(Buffer));
83 EXPECT_EQ(Buffer.get()->getBuffer(), "!!target!!");
96 EXPECT_TRUE(FDHasContent(ReadFD, "!!target!!"));
8497
8598 // The source file should no longer exist
8699 EXPECT_FALSE(fs::exists(SourceFileName));
87100 }
88101
89 {
90 // If we obtain a new descriptor for the target file, we should find that it
91 // contains the content that was in the source file.
92 int ReadFD = 0;
93 ASSERT_NO_ERROR(fs::openFileForRead(TargetFileName, ReadFD));
94 ScopedFD EventuallyCloseIt(ReadFD);
95 auto Buffer = MemoryBuffer::getOpenFile(ReadFD, TargetFileName, -1);
96 ASSERT_TRUE(static_cast(Buffer));
97
98 EXPECT_EQ(Buffer.get()->getBuffer(), "!!source!!");
99 }
102 // If we obtain a new descriptor for the target file, we should find that it
103 // contains the content that was in the source file.
104 EXPECT_TRUE(FileHasContent(TargetFileName, "!!source!!"));
100105
101106 // Rename the target file back to the source file name to confirm that rename
102107 // still works if the destination does not already exist.
109114 ASSERT_NO_ERROR(fs::remove(TestDirectory.str()));
110115 }
111116
117 TEST(rename, ExistingTemp) {
118 // Test that existing .tmpN files don't get deleted by the Windows
119 // sys::fs::rename implementation.
120 SmallString<128> TestDirectory;
121 ASSERT_NO_ERROR(
122 fs::createUniqueDirectory("ExistingTemp-test", TestDirectory));
123
124 SmallString<128> SourceFileName(TestDirectory);
125 path::append(SourceFileName, "source");
126
127 SmallString<128> TargetFileName(TestDirectory);
128 path::append(TargetFileName, "target");
129
130 SmallString<128> TargetTmp0FileName(TestDirectory);
131 path::append(TargetTmp0FileName, "target.tmp0");
132
133 SmallString<128> TargetTmp1FileName(TestDirectory);
134 path::append(TargetTmp1FileName, "target.tmp1");
135
136 ASSERT_NO_ERROR(CreateFileWithContent(SourceFileName, "!!source!!"));
137 ASSERT_NO_ERROR(CreateFileWithContent(TargetFileName, "!!target!!"));
138 ASSERT_NO_ERROR(CreateFileWithContent(TargetTmp0FileName, "!!target.tmp0!!"));
139
140 {
141 // Use mapped_file_region to make sure that the destination file is mmap'ed.
142 // This will cause SetInformationByHandle to fail when renaming to the
143 // destination, and we will follow the code path that tries to give target
144 // a temporary name.
145 int TargetFD;
146 std::error_code EC;
147 ASSERT_NO_ERROR(fs::openFileForRead(TargetFileName, TargetFD));
148 ScopedFD X(TargetFD);
149 sys::fs::mapped_file_region MFR(
150 TargetFD, sys::fs::mapped_file_region::readonly, 10, 0, EC);
151 ASSERT_FALSE(EC);
152
153 ASSERT_NO_ERROR(fs::rename(SourceFileName, TargetFileName));
154
155 #ifdef _WIN32
156 // Make sure that target was temporarily renamed to target.tmp1 on Windows.
157 // This is signified by a permission denied error as opposed to no such file
158 // or directory when trying to open it.
159 int Tmp1FD;
160 EXPECT_EQ(errc::permission_denied,
161 fs::openFileForRead(TargetTmp1FileName, Tmp1FD));
162 #endif
163 }
164
165 EXPECT_TRUE(FileHasContent(TargetTmp0FileName, "!!target.tmp0!!"));
166
167 ASSERT_NO_ERROR(fs::remove(TargetFileName));
168 ASSERT_NO_ERROR(fs::remove(TargetTmp0FileName));
169 ASSERT_NO_ERROR(fs::remove(TestDirectory.str()));
170 }
171
112172 } // anonymous namespace