llvm.org GIT mirror llvm / 3ffae19
[Support] Improve readNativeFile(Slice) interface Summary: There was a subtle, but pretty important difference between the Slice and regular versions of this function. The Slice function was zero-initializing the rest of the buffer when the read syscall returned less bytes than expected, while the regular function did not. This patch removes the inconsistency by making both functions *not* zero-initialize the buffer. The zeroing code is moved to the MemoryBuffer class, which is currently the only user of this code. This makes the API more consistent, and the code shorter. While in there, I also refactor the functions to return the number of bytes through the regular return value (via Expected<size_t>) instead of a separate by-ref argument. Reviewers: aganea, rnk Subscribers: kristina, Bigcheese, llvm-commits Tags: #llvm Differential Revision: https://reviews.llvm.org/D66471 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@369627 91177308-0d34-0410-b5e6-96231b3b80d8 Pavel Labath 24 days ago
6 changed file(s) with 158 addition(s) and 125 deletion(s). Raw diff Collapse all Expand all
990990 /// Returns kInvalidFile when the stream is closed.
991991 file_t getStderrHandle();
992992
993 /// Reads \p Buf.size() bytes from \p FileHandle into \p Buf. The number of
994 /// bytes actually read is returned in \p BytesRead. On Unix, this is equivalent
995 /// to `*BytesRead = ::read(FD, Buf.data(), Buf.size())`, with error reporting.
996 /// BytesRead will contain zero when reaching EOF.
993 /// Reads \p Buf.size() bytes from \p FileHandle into \p Buf. Returns the number
994 /// of bytes actually read. On Unix, this is equivalent to `return ::read(FD,
995 /// Buf.data(), Buf.size())`, with error reporting. Returns 0 when reaching EOF.
997996 ///
998997 /// @param FileHandle File to read from.
999998 /// @param Buf Buffer to read into.
1000 /// @param BytesRead Output parameter of the number of bytes read.
1001 /// @returns The error, if any, or errc::success.
1002 std::error_code readNativeFile(file_t FileHandle, MutableArrayRef Buf,
1003 size_t *BytesRead);
999 /// @returns The number of bytes read, or error.
1000 Expected readNativeFile(file_t FileHandle, MutableArrayRef Buf);
10041001
10051002 /// Reads \p Buf.size() bytes from \p FileHandle at offset \p Offset into \p
10061003 /// Buf. If 'pread' is available, this will use that, otherwise it will use
1007 /// 'lseek'. Bytes requested beyond the end of the file will be zero
1008 /// initialized.
1004 /// 'lseek'. Returns the number of bytes actually read. Returns 0 when reaching
1005 /// EOF.
10091006 ///
10101007 /// @param FileHandle File to read from.
10111008 /// @param Buf Buffer to read into.
10121009 /// @param Offset Offset into the file at which the read should occur.
1013 /// @returns The error, if any, or errc::success.
1014 std::error_code readNativeFileSlice(file_t FileHandle,
1015 MutableArrayRef Buf, size_t Offset);
1010 /// @returns The number of bytes read, or error.
1011 Expected readNativeFileSlice(file_t FileHandle,
1012 MutableArrayRef Buf,
1013 uint64_t Offset);
10161014
10171015 /// @brief Opens the file with the given name in a write-only or read-write
10181016 /// mode, returning its open file descriptor. If the file does not exist, it
210210 getMemoryBufferForStream(sys::fs::file_t FD, const Twine &BufferName) {
211211 const ssize_t ChunkSize = 4096*4;
212212 SmallString Buffer;
213 size_t ReadBytes;
214213 // Read into Buffer until we hit EOF.
215 do {
214 for (;;) {
216215 Buffer.reserve(Buffer.size() + ChunkSize);
217 if (auto EC = sys::fs::readNativeFile(
218 FD, makeMutableArrayRef(Buffer.end(), ChunkSize), &ReadBytes))
219 return EC;
220 Buffer.set_size(Buffer.size() + ReadBytes);
221 } while (ReadBytes != 0);
216 Expected ReadBytes = sys::fs::readNativeFile(
217 FD, makeMutableArrayRef(Buffer.end(), ChunkSize));
218 if (!ReadBytes)
219 return errorToErrorCode(ReadBytes.takeError());
220 if (*ReadBytes == 0)
221 break;
222 Buffer.set_size(Buffer.size() + *ReadBytes);
223 }
222224
223225 return getMemBufferCopyImpl(Buffer, BufferName);
224226 }
457459 return make_error_code(errc::not_enough_memory);
458460 }
459461
460 if (std::error_code EC =
461 sys::fs::readNativeFileSlice(FD, Buf->getBuffer(), Offset))
462 return EC;
462 // Read until EOF, zero-initialize the rest.
463 MutableArrayRef ToRead = Buf->getBuffer();
464 while (!ToRead.empty()) {
465 Expected ReadBytes =
466 sys::fs::readNativeFileSlice(FD, ToRead, Offset);
467 if (!ReadBytes)
468 return errorToErrorCode(ReadBytes.takeError());
469 if (*ReadBytes == 0) {
470 std::memset(ToRead.data(), 0, ToRead.size());
471 break;
472 }
473 ToRead = ToRead.drop_front(*ReadBytes);
474 Offset += *ReadBytes;
475 }
463476
464477 return std::move(Buf);
465478 }
998998 file_t getStdoutHandle() { return 1; }
999999 file_t getStderrHandle() { return 2; }
10001000
1001 std::error_code readNativeFile(file_t FD, MutableArrayRef Buf,
1002 size_t *BytesRead) {
1003 *BytesRead = sys::RetryAfterSignal(-1, ::read, FD, Buf.data(), Buf.size());
1004 if (ssize_t(*BytesRead) == -1)
1005 return std::error_code(errno, std::generic_category());
1006 return std::error_code();
1007 }
1008
1009 std::error_code readNativeFileSlice(file_t FD, MutableArrayRef Buf,
1010 size_t Offset) {
1011 char *BufPtr = Buf.data();
1012 size_t BytesLeft = Buf.size();
1013
1014 #ifndef HAVE_PREAD
1015 // If we don't have pread, seek to Offset.
1001 Expected readNativeFile(file_t FD, MutableArrayRef Buf) {
1002 ssize_t NumRead =
1003 sys::RetryAfterSignal(-1, ::read, FD, Buf.data(), Buf.size());
1004 if (ssize_t(NumRead) == -1)
1005 return errorCodeToError(std::error_code(errno, std::generic_category()));
1006 return NumRead;
1007 }
1008
1009 Expected readNativeFileSlice(file_t FD, MutableArrayRef Buf,
1010 uint64_t Offset) {
1011 #ifdef HAVE_PREAD
1012 ssize_t NumRead =
1013 sys::RetryAfterSignal(-1, ::pread, FD, Buf.data(), Buf.size(), Offset);
1014 #else
10161015 if (lseek(FD, Offset, SEEK_SET) == -1)
1017 return std::error_code(errno, std::generic_category());
1018 #endif
1019
1020 while (BytesLeft) {
1021 #ifdef HAVE_PREAD
1022 ssize_t NumRead = sys::RetryAfterSignal(-1, ::pread, FD, BufPtr, BytesLeft,
1023 Buf.size() - BytesLeft + Offset);
1024 #else
1025 ssize_t NumRead = sys::RetryAfterSignal(-1, ::read, FD, BufPtr, BytesLeft);
1026 #endif
1027 if (NumRead == -1) {
1028 // Error while reading.
1029 return std::error_code(errno, std::generic_category());
1030 }
1031 if (NumRead == 0) {
1032 memset(BufPtr, 0, BytesLeft); // zero-initialize rest of the buffer.
1033 break;
1034 }
1035 BytesLeft -= NumRead;
1036 BufPtr += NumRead;
1037 }
1038 return std::error_code();
1016 return errorCodeToError(std::error_code(errno, std::generic_category()));
1017 ssize_t NumRead =
1018 sys::RetryAfterSignal(-1, ::read, FD, Buf.data(), Buf.size());
1019 #endif
1020 if (NumRead == -1)
1021 return errorCodeToError(std::error_code(errno, std::generic_category()));
1022 return NumRead;
10391023 }
10401024
10411025 std::error_code closeFile(file_t &F) {
12161216 file_t getStdoutHandle() { return ::GetStdHandle(STD_OUTPUT_HANDLE); }
12171217 file_t getStderrHandle() { return ::GetStdHandle(STD_ERROR_HANDLE); }
12181218
1219 std::error_code readNativeFileImpl(file_t FileHandle, char *BufPtr, size_t BytesToRead,
1220 size_t *BytesRead, OVERLAPPED *Overlap) {
1219 Expected readNativeFileImpl(file_t FileHandle,
1220 MutableArrayRef Buf,
1221 OVERLAPPED *Overlap) {
12211222 // ReadFile can only read 2GB at a time. The caller should check the number of
12221223 // bytes and read in a loop until termination.
1223 DWORD BytesToRead32 =
1224 std::min(size_t(std::numeric_limits::max()), BytesToRead);
1225 DWORD BytesRead32 = 0;
1226 bool Success =
1227 ::ReadFile(FileHandle, BufPtr, BytesToRead32, &BytesRead32, Overlap);
1228 *BytesRead = BytesRead32;
1229 if (!Success) {
1230 DWORD Err = ::GetLastError();
1231 // EOF is not an error.
1232 if (Err == ERROR_BROKEN_PIPE || Err == ERROR_HANDLE_EOF)
1233 return std::error_code();
1234 return mapWindowsError(Err);
1235 }
1236 return std::error_code();
1237 }
1238
1239 std::error_code readNativeFile(file_t FileHandle, MutableArrayRef Buf,
1240 size_t *BytesRead) {
1241 return readNativeFileImpl(FileHandle, Buf.data(), Buf.size(), BytesRead,
1242 /*Overlap=*/nullptr);
1243 }
1244
1245 std::error_code readNativeFileSlice(file_t FileHandle,
1246 MutableArrayRef Buf, size_t Offset) {
1247 char *BufPtr = Buf.data();
1248 size_t BytesLeft = Buf.size();
1249
1250 while (BytesLeft) {
1251 uint64_t CurOff = Buf.size() - BytesLeft + Offset;
1252 OVERLAPPED Overlapped = {};
1253 Overlapped.Offset = uint32_t(CurOff);
1254 Overlapped.OffsetHigh = uint32_t(uint64_t(CurOff) >> 32);
1255
1256 size_t BytesRead = 0;
1257 if (auto EC = readNativeFileImpl(FileHandle, BufPtr, BytesLeft, &BytesRead,
1258 &Overlapped))
1259 return EC;
1260
1261 // Once we reach EOF, zero the remaining bytes in the buffer.
1262 if (BytesRead == 0) {
1263 memset(BufPtr, 0, BytesLeft);
1264 break;
1265 }
1266 BytesLeft -= BytesRead;
1267 BufPtr += BytesRead;
1268 }
1269 return std::error_code();
1224 DWORD BytesToRead =
1225 std::min(size_t(std::numeric_limits::max()), Buf.size());
1226 DWORD BytesRead = 0;
1227 if (::ReadFile(FileHandle, Buf.data(), BytesToRead, &BytesRead, Overlap))
1228 return BytesRead;
1229 DWORD Err = ::GetLastError();
1230 // EOF is not an error.
1231 if (Err == ERROR_BROKEN_PIPE || Err == ERROR_HANDLE_EOF)
1232 return BytesRead;
1233 return errorCodeToError(mapWindowsError(Err));
1234 }
1235
1236 Expected readNativeFile(file_t FileHandle, MutableArrayRef Buf) {
1237 return readNativeFileImpl(FileHandle, Buf, /*Overlap=*/nullptr);
1238 }
1239
1240 Expected readNativeFileSlice(file_t FileHandle,
1241 MutableArrayRef Buf,
1242 uint64_t Offset) {
1243 OVERLAPPED Overlapped = {};
1244 Overlapped.Offset = uint32_t(Offset);
1245 Overlapped.OffsetHigh = uint32_t(Offset >> 32);
1246 return readNativeFileImpl(FileHandle, Buf, &Overlapped);
12701247 }
12711248
12721249 std::error_code closeFile(file_t &F) {
1616 #include "llvm/Support/raw_ostream.h"
1717 #include "llvm/Testing/Support/Error.h"
1818 #include "gtest/gtest.h"
19 #if LLVM_ENABLE_THREADS
20 #include
21 #endif
22 #if LLVM_ON_UNIX
23 #include
24 #endif
25 #if _WIN32
26 #include
27 #endif
1928
2029 using namespace llvm;
2130
150159 // verify the two copies do not point to the same place
151160 EXPECT_NE(MBC1->getBufferStart(), MBC2->getBufferStart());
152161 }
162
163 #if LLVM_ENABLE_THREADS
164 TEST_F(MemoryBufferTest, createFromPipe) {
165 sys::fs::file_t pipes[2];
166 #if LLVM_ON_UNIX
167 ASSERT_EQ(::pipe(pipes), 0) << strerror(errno);
168 #else
169 ASSERT_TRUE(::CreatePipe(&pipes[0], &pipes[1], nullptr, 0))
170 << ::GetLastError();
171 #endif
172 auto ReadCloser = make_scope_exit([&] { sys::fs::closeFile(pipes[0]); });
173 std::thread Writer([&] {
174 auto WriteCloser = make_scope_exit([&] { sys::fs::closeFile(pipes[1]); });
175 for (unsigned i = 0; i < 5; ++i) {
176 std::this_thread::sleep_for(std::chrono::milliseconds(10));
177 #if LLVM_ON_UNIX
178 ASSERT_EQ(::write(pipes[1], "foo", 3), 3) << strerror(errno);
179 #else
180 DWORD Written;
181 ASSERT_TRUE(::WriteFile(pipes[1], "foo", 3, &Written, nullptr))
182 << ::GetLastError();
183 ASSERT_EQ(Written, 3u);
184 #endif
185 }
186 });
187 ErrorOr MB =
188 MemoryBuffer::getOpenFile(pipes[0], "pipe", /*FileSize*/ -1);
189 Writer.join();
190 ASSERT_NO_ERROR(MB.getError());
191 EXPECT_EQ(MB.get()->getBuffer(), "foofoofoofoofoo");
192 }
193 #endif
153194
154195 TEST_F(MemoryBufferTest, make_new) {
155196 // 0-sized buffer
77
88 #include "llvm/Support/Path.h"
99 #include "llvm/ADT/STLExtras.h"
10 #include "llvm/ADT/ScopeExit.h"
1011 #include "llvm/ADT/SmallVector.h"
1112 #include "llvm/ADT/Triple.h"
1213 #include "llvm/BinaryFormat/Magic.h"
15131514 verifyWrite(FD, "Buzz", true);
15141515 }
15151516
1517 TEST_F(FileSystemTest, readNativeFile) {
1518 createFileWithData(NonExistantFile, false, fs::CD_CreateNew, "01234");
1519 FileRemover Cleanup(NonExistantFile);
1520 const auto &Read = [&](size_t ToRead) -> Expected {
1521 std::string Buf(ToRead, '?');
1522 Expected FD = fs::openNativeFileForRead(NonExistantFile);
1523 if (!FD)
1524 return FD.takeError();
1525 auto Close = make_scope_exit([&] { fs::closeFile(*FD); });
1526 if (Expected BytesRead = fs::readNativeFile(
1527 *FD, makeMutableArrayRef(&*Buf.begin(), Buf.size())))
1528 return Buf.substr(0, *BytesRead);
1529 else
1530 return BytesRead.takeError();
1531 };
1532 EXPECT_THAT_EXPECTED(Read(5), HasValue("01234"));
1533 EXPECT_THAT_EXPECTED(Read(3), HasValue("012"));
1534 EXPECT_THAT_EXPECTED(Read(6), HasValue("01234"));
1535 }
1536
15161537 TEST_F(FileSystemTest, readNativeFileSlice) {
1517 char Data[10] = {'0', '1', '2', '3', '4', 0, 0, 0, 0, 0};
1518 createFileWithData(NonExistantFile, false, fs::CD_CreateNew,
1519 StringRef(Data, 5));
1538 createFileWithData(NonExistantFile, false, fs::CD_CreateNew, "01234");
15201539 FileRemover Cleanup(NonExistantFile);
15211540 Expected FD = fs::openNativeFileForRead(NonExistantFile);
15221541 ASSERT_THAT_EXPECTED(FD, Succeeded());
1523 char Buf[10];
1542 auto Close = make_scope_exit([&] { fs::closeFile(*FD); });
15241543 const auto &Read = [&](size_t Offset,
1525 size_t ToRead) -> Expected> {
1526 std::memset(Buf, 0x47, sizeof(Buf));
1527 if (std::error_code EC = fs::readNativeFileSlice(
1528 *FD, makeMutableArrayRef(Buf, ToRead), Offset))
1529 return errorCodeToError(EC);
1530 return makeArrayRef(Buf, ToRead);
1544 size_t ToRead) -> Expected {
1545 std::string Buf(ToRead, '?');
1546 if (Expected BytesRead = fs::readNativeFileSlice(
1547 *FD, makeMutableArrayRef(&*Buf.begin(), Buf.size()), Offset))
1548 return Buf.substr(0, *BytesRead);
1549 else
1550 return BytesRead.takeError();
15311551 };
1532 EXPECT_THAT_EXPECTED(Read(0, 5), HasValue(makeArrayRef(Data + 0, 5)));
1533 EXPECT_THAT_EXPECTED(Read(0, 3), HasValue(makeArrayRef(Data + 0, 3)));
1534 EXPECT_THAT_EXPECTED(Read(2, 3), HasValue(makeArrayRef(Data + 2, 3)));
1535 EXPECT_THAT_EXPECTED(Read(0, 6), HasValue(makeArrayRef(Data + 0, 6)));
1536 EXPECT_THAT_EXPECTED(Read(2, 6), HasValue(makeArrayRef(Data + 2, 6)));
1537 EXPECT_THAT_EXPECTED(Read(5, 5), HasValue(makeArrayRef(Data + 5, 5)));
1552 EXPECT_THAT_EXPECTED(Read(0, 5), HasValue("01234"));
1553 EXPECT_THAT_EXPECTED(Read(0, 3), HasValue("012"));
1554 EXPECT_THAT_EXPECTED(Read(2, 3), HasValue("234"));
1555 EXPECT_THAT_EXPECTED(Read(0, 6), HasValue("01234"));
1556 EXPECT_THAT_EXPECTED(Read(2, 6), HasValue("234"));
1557 EXPECT_THAT_EXPECTED(Read(5, 5), HasValue(""));
15381558 }
15391559
15401560 TEST_F(FileSystemTest, is_local) {