llvm.org GIT mirror llvm / d051856
Merge StreamableMemoryObject into MemoryObject. Every MemoryObject is a StreamableMemoryObject since the removal of StringRefMemoryObject, so just merge the two. I will clean up the MemoryObject interface in the upcoming commits. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@221766 91177308-0d34-0410-b5e6-96231b3b80d8 Rafael Espindola 5 years ago
8 changed file(s) with 260 addition(s) and 272 deletion(s). Raw diff Collapse all Expand all
1616
1717 #include "llvm/Bitcode/BitCodes.h"
1818 #include "llvm/Support/Endian.h"
19 #include "llvm/Support/StreamableMemoryObject.h"
19 #include "llvm/Support/StreamingMemoryObject.h"
2020 #include
2121 #include
2222 #include
4141 std::vector > RecordNames;
4242 };
4343 private:
44 std::unique_ptr<StreamableMemoryObject> BitcodeBytes;
44 std::unique_ptr<MemoryObject> BitcodeBytes;
4545
4646 std::vector BlockInfoRecords;
4747
6060 init(Start, End);
6161 }
6262
63 BitstreamReader(StreamableMemoryObject *bytes) : IgnoreBlockInfoNames(true) {
63 BitstreamReader(MemoryObject *bytes) : IgnoreBlockInfoNames(true) {
6464 BitcodeBytes.reset(bytes);
6565 }
6666
8181 BitcodeBytes.reset(getNonStreamedMemoryObject(Start, End));
8282 }
8383
84 StreamableMemoryObject &getBitcodeBytes() { return *BitcodeBytes; }
84 MemoryObject &getBitcodeBytes() { return *BitcodeBytes; }
8585
8686 /// This is called by clients that want block/record name information.
8787 void CollectBlockInfoNames() { IgnoreBlockInfoNames = false; }
1313
1414 namespace llvm {
1515
16 /// Abstract base class for contiguous addressable memory. Necessary for cases
17 /// in which the memory is in another process, in a file, or on a remote
18 /// machine. All size and offset parameters are uint64_ts, to allow 32-bit
19 /// processes access to 64-bit address spaces.
16 /// Interface to data which might be streamed. Streamability has 2 important
17 /// implications/restrictions. First, the data might not yet exist in memory
18 /// when the request is made. This just means that readByte/readBytes might have
19 /// to block or do some work to get it. More significantly, the exact size of
20 /// the object might not be known until it has all been fetched. This means that
21 /// to return the right result, getExtent must also wait for all the data to
22 /// arrive; therefore it should not be called on objects which are actually
23 /// streamed (this would defeat the purpose of streaming). Instead,
24 /// isValidAddress and isObjectEnd can be used to test addresses without knowing
25 /// the exact size of the stream. Finally, getPointer can be used instead of
26 /// readBytes to avoid extra copying.
2027 class MemoryObject {
2128 public:
2229 virtual ~MemoryObject();
4047 /// bounds violation or an implementation-specific error.
4148 virtual int readBytes(uint64_t address, uint64_t size,
4249 uint8_t *buf) const = 0;
50
51 /// Ensures that the requested data is in memory, and returns a pointer to it.
52 /// More efficient than using readBytes if the data is already in memory. May
53 /// block until (address - base + size) bytes have been read
54 /// @param address - address of the byte, in the same space as getBase()
55 /// @param size - amount of data that must be available on return
56 /// @result - valid pointer to the requested data
57 virtual const uint8_t *getPointer(uint64_t address, uint64_t size) const = 0;
58
59 /// Returns true if the address is within the object (i.e. between base and
60 /// base + extent - 1 inclusive). May block until (address - base) bytes have
61 /// been read
62 /// @param address - address of the byte, in the same space as getBase()
63 /// @result - true if the address may be read with readByte()
64 virtual bool isValidAddress(uint64_t address) const = 0;
65
66 /// Returns true if the address is one past the end of the object (i.e. if it
67 /// is equal to base + extent). May block until (address - base) bytes have
68 /// been read
69 /// @param address - address of the byte, in the same space as getBase()
70 /// @result - true if the address is equal to base + extent
71 virtual bool isObjectEnd(uint64_t address) const = 0;
4372 };
4473
4574 }
+0
-135
include/llvm/Support/StreamableMemoryObject.h less more
None //===- StreamableMemoryObject.h - Streamable data interface -----*- C++ -*-===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8
9
10 #ifndef LLVM_SUPPORT_STREAMABLEMEMORYOBJECT_H
11 #define LLVM_SUPPORT_STREAMABLEMEMORYOBJECT_H
12
13 #include "llvm/Support/Compiler.h"
14 #include "llvm/Support/DataStream.h"
15 #include "llvm/Support/ErrorHandling.h"
16 #include "llvm/Support/MemoryObject.h"
17 #include
18 #include
19 #include
20
21 namespace llvm {
22
23 /// Interface to data which might be streamed. Streamability has 2 important
24 /// implications/restrictions. First, the data might not yet exist in memory
25 /// when the request is made. This just means that readByte/readBytes might have
26 /// to block or do some work to get it. More significantly, the exact size of
27 /// the object might not be known until it has all been fetched. This means that
28 /// to return the right result, getExtent must also wait for all the data to
29 /// arrive; therefore it should not be called on objects which are actually
30 /// streamed (this would defeat the purpose of streaming). Instead,
31 /// isValidAddress and isObjectEnd can be used to test addresses without knowing
32 /// the exact size of the stream. Finally, getPointer can be used instead of
33 /// readBytes to avoid extra copying.
34 class StreamableMemoryObject : public MemoryObject {
35 public:
36 virtual ~StreamableMemoryObject();
37
38 /// Ensures that the requested data is in memory, and returns a pointer to it.
39 /// More efficient than using readBytes if the data is already in memory. May
40 /// block until (address - base + size) bytes have been read
41 /// @param address - address of the byte, in the same space as getBase()
42 /// @param size - amount of data that must be available on return
43 /// @result - valid pointer to the requested data
44 virtual const uint8_t *getPointer(uint64_t address, uint64_t size) const = 0;
45
46 /// Returns true if the address is within the object (i.e. between base and
47 /// base + extent - 1 inclusive). May block until (address - base) bytes have
48 /// been read
49 /// @param address - address of the byte, in the same space as getBase()
50 /// @result - true if the address may be read with readByte()
51 virtual bool isValidAddress(uint64_t address) const = 0;
52
53 /// Returns true if the address is one past the end of the object (i.e. if it
54 /// is equal to base + extent). May block until (address - base) bytes have
55 /// been read
56 /// @param address - address of the byte, in the same space as getBase()
57 /// @result - true if the address is equal to base + extent
58 virtual bool isObjectEnd(uint64_t address) const = 0;
59 };
60
61 /// Interface to data which is actually streamed from a DataStreamer. In
62 /// addition to inherited members, it has the dropLeadingBytes and
63 /// setKnownObjectSize methods which are not applicable to non-streamed objects.
64 class StreamingMemoryObject : public StreamableMemoryObject {
65 public:
66 StreamingMemoryObject(DataStreamer *streamer);
67 uint64_t getExtent() const override;
68 int readBytes(uint64_t address, uint64_t size,
69 uint8_t *buf) const override;
70 const uint8_t *getPointer(uint64_t address, uint64_t size) const override {
71 // This could be fixed by ensuring the bytes are fetched and making a copy,
72 // requiring that the bitcode size be known, or otherwise ensuring that
73 // the memory doesn't go away/get reallocated, but it's
74 // not currently necessary. Users that need the pointer don't stream.
75 llvm_unreachable("getPointer in streaming memory objects not allowed");
76 return nullptr;
77 }
78 bool isValidAddress(uint64_t address) const override;
79 bool isObjectEnd(uint64_t address) const override;
80
81 /// Drop s bytes from the front of the stream, pushing the positions of the
82 /// remaining bytes down by s. This is used to skip past the bitcode header,
83 /// since we don't know a priori if it's present, and we can't put bytes
84 /// back into the stream once we've read them.
85 bool dropLeadingBytes(size_t s);
86
87 /// If the data object size is known in advance, many of the operations can
88 /// be made more efficient, so this method should be called before reading
89 /// starts (although it can be called anytime).
90 void setKnownObjectSize(size_t size);
91
92 private:
93 const static uint32_t kChunkSize = 4096 * 4;
94 mutable std::vector Bytes;
95 std::unique_ptr Streamer;
96 mutable size_t BytesRead; // Bytes read from stream
97 size_t BytesSkipped;// Bytes skipped at start of stream (e.g. wrapper/header)
98 mutable size_t ObjectSize; // 0 if unknown, set if wrapper seen or EOF reached
99 mutable bool EOFReached;
100
101 // Fetch enough bytes such that Pos can be read or EOF is reached
102 // (i.e. BytesRead > Pos). Return true if Pos can be read.
103 // Unlike most of the functions in BitcodeReader, returns true on success.
104 // Most of the requests will be small, but we fetch at kChunkSize bytes
105 // at a time to avoid making too many potentially expensive GetBytes calls
106 bool fetchToPos(size_t Pos) const {
107 if (EOFReached) return Pos < ObjectSize;
108 while (Pos >= BytesRead) {
109 Bytes.resize(BytesRead + BytesSkipped + kChunkSize);
110 size_t bytes = Streamer->GetBytes(&Bytes[BytesRead + BytesSkipped],
111 kChunkSize);
112 BytesRead += bytes;
113 if (bytes < kChunkSize) {
114 assert((!ObjectSize || BytesRead >= Pos) &&
115 "Unexpected short read fetching bitcode");
116 if (BytesRead <= Pos) { // reached EOF/ran out of bytes
117 ObjectSize = BytesRead;
118 EOFReached = true;
119 return false;
120 }
121 }
122 }
123 return true;
124 }
125
126 StreamingMemoryObject(const StreamingMemoryObject&) LLVM_DELETED_FUNCTION;
127 void operator=(const StreamingMemoryObject&) LLVM_DELETED_FUNCTION;
128 };
129
130 StreamableMemoryObject *getNonStreamedMemoryObject(
131 const unsigned char *Start, const unsigned char *End);
132
133 }
134 #endif // STREAMABLEMEMORYOBJECT_H_
0 //===- StreamingMemoryObject.h - Streamable data interface -----*- C++ -*-===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8
9 #ifndef LLVM_SUPPORT_STREAMINGMEMORYOBJECT_H
10 #define LLVM_SUPPORT_STREAMINGMEMORYOBJECT_H
11
12 #include "llvm/Support/Compiler.h"
13 #include "llvm/Support/DataStream.h"
14 #include "llvm/Support/ErrorHandling.h"
15 #include "llvm/Support/MemoryObject.h"
16 #include
17 #include
18 #include
19
20 namespace llvm {
21
22 /// Interface to data which is actually streamed from a DataStreamer. In
23 /// addition to inherited members, it has the dropLeadingBytes and
24 /// setKnownObjectSize methods which are not applicable to non-streamed objects.
25 class StreamingMemoryObject : public MemoryObject {
26 public:
27 StreamingMemoryObject(DataStreamer *streamer);
28 uint64_t getExtent() const override;
29 int readBytes(uint64_t address, uint64_t size,
30 uint8_t *buf) const override;
31 const uint8_t *getPointer(uint64_t address, uint64_t size) const override {
32 // This could be fixed by ensuring the bytes are fetched and making a copy,
33 // requiring that the bitcode size be known, or otherwise ensuring that
34 // the memory doesn't go away/get reallocated, but it's
35 // not currently necessary. Users that need the pointer don't stream.
36 llvm_unreachable("getPointer in streaming memory objects not allowed");
37 return nullptr;
38 }
39 bool isValidAddress(uint64_t address) const override;
40 bool isObjectEnd(uint64_t address) const override;
41
42 /// Drop s bytes from the front of the stream, pushing the positions of the
43 /// remaining bytes down by s. This is used to skip past the bitcode header,
44 /// since we don't know a priori if it's present, and we can't put bytes
45 /// back into the stream once we've read them.
46 bool dropLeadingBytes(size_t s);
47
48 /// If the data object size is known in advance, many of the operations can
49 /// be made more efficient, so this method should be called before reading
50 /// starts (although it can be called anytime).
51 void setKnownObjectSize(size_t size);
52
53 private:
54 const static uint32_t kChunkSize = 4096 * 4;
55 mutable std::vector Bytes;
56 std::unique_ptr Streamer;
57 mutable size_t BytesRead; // Bytes read from stream
58 size_t BytesSkipped;// Bytes skipped at start of stream (e.g. wrapper/header)
59 mutable size_t ObjectSize; // 0 if unknown, set if wrapper seen or EOF reached
60 mutable bool EOFReached;
61
62 // Fetch enough bytes such that Pos can be read or EOF is reached
63 // (i.e. BytesRead > Pos). Return true if Pos can be read.
64 // Unlike most of the functions in BitcodeReader, returns true on success.
65 // Most of the requests will be small, but we fetch at kChunkSize bytes
66 // at a time to avoid making too many potentially expensive GetBytes calls
67 bool fetchToPos(size_t Pos) const {
68 if (EOFReached) return Pos < ObjectSize;
69 while (Pos >= BytesRead) {
70 Bytes.resize(BytesRead + BytesSkipped + kChunkSize);
71 size_t bytes = Streamer->GetBytes(&Bytes[BytesRead + BytesSkipped],
72 kChunkSize);
73 BytesRead += bytes;
74 if (bytes < kChunkSize) {
75 assert((!ObjectSize || BytesRead >= Pos) &&
76 "Unexpected short read fetching bitcode");
77 if (BytesRead <= Pos) { // reached EOF/ran out of bytes
78 ObjectSize = BytesRead;
79 EOFReached = true;
80 return false;
81 }
82 }
83 }
84 return true;
85 }
86
87 StreamingMemoryObject(const StreamingMemoryObject&) LLVM_DELETED_FUNCTION;
88 void operator=(const StreamingMemoryObject&) LLVM_DELETED_FUNCTION;
89 };
90
91 MemoryObject *getNonStreamedMemoryObject(
92 const unsigned char *Start, const unsigned char *End);
93
94 }
95 #endif // STREAMINGMEMORYOBJECT_H_
7979 SourceMgr.cpp
8080 SpecialCaseList.cpp
8181 Statistic.cpp
82 StreamableMemoryObject.cpp
82 StreamingMemoryObject.cpp
8383 StringExtras.cpp
8484 StringMap.cpp
8585 StringPool.cpp
3131 #define DEBUG_TYPE "Data-stream"
3232
3333 // Interface goals:
34 // * StreamableMemoryObject doesn't care about complexities like using
34 // * StreamingMemoryObject doesn't care about complexities like using
3535 // threads/async callbacks to actually overlap download+compile
3636 // * Don't want to duplicate Data in memory
3737 // * Don't need to know total Data len in advance
3838 // Non-goals:
39 // StreamableMemoryObject already has random access so this interface only does
39 // StreamingMemoryObject already has random access so this interface only does
4040 // in-order streaming (no arbitrary seeking, else we'd have to buffer all the
4141 // Data here in addition to MemoryObject). This also means that if we want
4242 // to be able to to free Data, BitstreamBytes/BitcodeReader will implement it
+0
-126
lib/Support/StreamableMemoryObject.cpp less more
None //===- StreamableMemoryObject.cpp - Streamable data interface -------------===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/Support/StreamableMemoryObject.h"
10 #include "llvm/Support/Compiler.h"
11 #include
12 #include
13 #include
14
15
16 using namespace llvm;
17
18 namespace {
19
20 class RawMemoryObject : public StreamableMemoryObject {
21 public:
22 RawMemoryObject(const unsigned char *Start, const unsigned char *End) :
23 FirstChar(Start), LastChar(End) {
24 assert(LastChar >= FirstChar && "Invalid start/end range");
25 }
26
27 uint64_t getExtent() const override {
28 return LastChar - FirstChar;
29 }
30 int readBytes(uint64_t address, uint64_t size,
31 uint8_t *buf) const override;
32 const uint8_t *getPointer(uint64_t address, uint64_t size) const override;
33 bool isValidAddress(uint64_t address) const override {
34 return validAddress(address);
35 }
36 bool isObjectEnd(uint64_t address) const override {
37 return objectEnd(address);
38 }
39
40 private:
41 const uint8_t* const FirstChar;
42 const uint8_t* const LastChar;
43
44 // These are implemented as inline functions here to avoid multiple virtual
45 // calls per public function
46 bool validAddress(uint64_t address) const {
47 return static_cast(address) < LastChar - FirstChar;
48 }
49 bool objectEnd(uint64_t address) const {
50 return static_cast(address) == LastChar - FirstChar;
51 }
52
53 RawMemoryObject(const RawMemoryObject&) LLVM_DELETED_FUNCTION;
54 void operator=(const RawMemoryObject&) LLVM_DELETED_FUNCTION;
55 };
56
57 int RawMemoryObject::readBytes(uint64_t address,
58 uint64_t size,
59 uint8_t *buf) const {
60 if (!validAddress(address) || !validAddress(address + size - 1)) return -1;
61 memcpy(buf, (uint8_t *)(uintptr_t)(address + FirstChar), size);
62 return size;
63 }
64
65 const uint8_t *RawMemoryObject::getPointer(uint64_t address,
66 uint64_t size) const {
67 return FirstChar + address;
68 }
69 } // anonymous namespace
70
71 namespace llvm {
72 // If the bitcode has a header, then its size is known, and we don't have to
73 // block until we actually want to read it.
74 bool StreamingMemoryObject::isValidAddress(uint64_t address) const {
75 if (ObjectSize && address < ObjectSize) return true;
76 return fetchToPos(address);
77 }
78
79 bool StreamingMemoryObject::isObjectEnd(uint64_t address) const {
80 if (ObjectSize) return address == ObjectSize;
81 fetchToPos(address);
82 return address == ObjectSize && address != 0;
83 }
84
85 uint64_t StreamingMemoryObject::getExtent() const {
86 if (ObjectSize) return ObjectSize;
87 size_t pos = BytesRead + kChunkSize;
88 // keep fetching until we run out of bytes
89 while (fetchToPos(pos)) pos += kChunkSize;
90 return ObjectSize;
91 }
92
93 int StreamingMemoryObject::readBytes(uint64_t address,
94 uint64_t size,
95 uint8_t *buf) const {
96 if (!fetchToPos(address + size - 1)) return -1;
97 memcpy(buf, &Bytes[address + BytesSkipped], size);
98 return 0;
99 }
100
101 bool StreamingMemoryObject::dropLeadingBytes(size_t s) {
102 if (BytesRead < s) return true;
103 BytesSkipped = s;
104 BytesRead -= s;
105 return false;
106 }
107
108 void StreamingMemoryObject::setKnownObjectSize(size_t size) {
109 ObjectSize = size;
110 Bytes.reserve(size);
111 }
112
113 StreamableMemoryObject *getNonStreamedMemoryObject(
114 const unsigned char *Start, const unsigned char *End) {
115 return new RawMemoryObject(Start, End);
116 }
117
118 StreamableMemoryObject::~StreamableMemoryObject() { }
119
120 StreamingMemoryObject::StreamingMemoryObject(DataStreamer *streamer) :
121 Bytes(kChunkSize), Streamer(streamer), BytesRead(0), BytesSkipped(0),
122 ObjectSize(0), EOFReached(false) {
123 BytesRead = streamer->GetBytes(&Bytes[0], kChunkSize);
124 }
125 }
0 //===- StreamingMemoryObject.cpp - Streamable data interface -------------===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/Support/StreamingMemoryObject.h"
10 #include "llvm/Support/Compiler.h"
11 #include
12 #include
13 #include
14
15
16 using namespace llvm;
17
18 namespace {
19
20 class RawMemoryObject : public MemoryObject {
21 public:
22 RawMemoryObject(const unsigned char *Start, const unsigned char *End) :
23 FirstChar(Start), LastChar(End) {
24 assert(LastChar >= FirstChar && "Invalid start/end range");
25 }
26
27 uint64_t getExtent() const override {
28 return LastChar - FirstChar;
29 }
30 int readBytes(uint64_t address, uint64_t size,
31 uint8_t *buf) const override;
32 const uint8_t *getPointer(uint64_t address, uint64_t size) const override;
33 bool isValidAddress(uint64_t address) const override {
34 return validAddress(address);
35 }
36 bool isObjectEnd(uint64_t address) const override {
37 return objectEnd(address);
38 }
39
40 private:
41 const uint8_t* const FirstChar;
42 const uint8_t* const LastChar;
43
44 // These are implemented as inline functions here to avoid multiple virtual
45 // calls per public function
46 bool validAddress(uint64_t address) const {
47 return static_cast(address) < LastChar - FirstChar;
48 }
49 bool objectEnd(uint64_t address) const {
50 return static_cast(address) == LastChar - FirstChar;
51 }
52
53 RawMemoryObject(const RawMemoryObject&) LLVM_DELETED_FUNCTION;
54 void operator=(const RawMemoryObject&) LLVM_DELETED_FUNCTION;
55 };
56
57 int RawMemoryObject::readBytes(uint64_t address,
58 uint64_t size,
59 uint8_t *buf) const {
60 if (!validAddress(address) || !validAddress(address + size - 1)) return -1;
61 memcpy(buf, (uint8_t *)(uintptr_t)(address + FirstChar), size);
62 return size;
63 }
64
65 const uint8_t *RawMemoryObject::getPointer(uint64_t address,
66 uint64_t size) const {
67 return FirstChar + address;
68 }
69 } // anonymous namespace
70
71 namespace llvm {
72 // If the bitcode has a header, then its size is known, and we don't have to
73 // block until we actually want to read it.
74 bool StreamingMemoryObject::isValidAddress(uint64_t address) const {
75 if (ObjectSize && address < ObjectSize) return true;
76 return fetchToPos(address);
77 }
78
79 bool StreamingMemoryObject::isObjectEnd(uint64_t address) const {
80 if (ObjectSize) return address == ObjectSize;
81 fetchToPos(address);
82 return address == ObjectSize && address != 0;
83 }
84
85 uint64_t StreamingMemoryObject::getExtent() const {
86 if (ObjectSize) return ObjectSize;
87 size_t pos = BytesRead + kChunkSize;
88 // keep fetching until we run out of bytes
89 while (fetchToPos(pos)) pos += kChunkSize;
90 return ObjectSize;
91 }
92
93 int StreamingMemoryObject::readBytes(uint64_t address,
94 uint64_t size,
95 uint8_t *buf) const {
96 if (!fetchToPos(address + size - 1)) return -1;
97 memcpy(buf, &Bytes[address + BytesSkipped], size);
98 return 0;
99 }
100
101 bool StreamingMemoryObject::dropLeadingBytes(size_t s) {
102 if (BytesRead < s) return true;
103 BytesSkipped = s;
104 BytesRead -= s;
105 return false;
106 }
107
108 void StreamingMemoryObject::setKnownObjectSize(size_t size) {
109 ObjectSize = size;
110 Bytes.reserve(size);
111 }
112
113 MemoryObject *getNonStreamedMemoryObject(const unsigned char *Start,
114 const unsigned char *End) {
115 return new RawMemoryObject(Start, End);
116 }
117
118 StreamingMemoryObject::StreamingMemoryObject(DataStreamer *streamer) :
119 Bytes(kChunkSize), Streamer(streamer), BytesRead(0), BytesSkipped(0),
120 ObjectSize(0), EOFReached(false) {
121 BytesRead = streamer->GetBytes(&Bytes[0], kChunkSize);
122 }
123 }