llvm.org GIT mirror llvm / 81a153c
[BinaryStream] Support growable streams. The existing library assumed that a stream's length would never change. This makes some things simpler, but it's not flexible enough for what we need, especially for writable streams where what you really want is for each call to write to actually append. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@319070 91177308-0d34-0410-b5e6-96231b3b80d8 Zachary Turner 2 years ago
10 changed file(s) with 261 addition(s) and 40 deletion(s). Raw diff Collapse all Expand all
4444 /// Construct a string ref from an array ref of unsigned chars.
4545 inline StringRef toStringRef(ArrayRef Input) {
4646 return StringRef(reinterpret_cast(Input.begin()), Input.size());
47 }
48
49 /// Construct a string ref from an array ref of unsigned chars.
50 inline ArrayRef arrayRefFromStringRef(StringRef Input) {
51 return {Input.bytes_begin(), Input.bytes_end()};
4752 }
4853
4954 /// Interpret the given character \p C as a hexadecimal digit and return its
4040
4141 Error readBytes(uint32_t Offset, uint32_t Size,
4242 ArrayRef &Buffer) override {
43 if (auto EC = checkOffset(Offset, Size))
43 if (auto EC = checkOffsetForRead(Offset, Size))
4444 return EC;
4545 Buffer = Data.slice(Offset, Size);
4646 return Error::success();
4848
4949 Error readLongestContiguousChunk(uint32_t Offset,
5050 ArrayRef &Buffer) override {
51 if (auto EC = checkOffset(Offset, 1))
51 if (auto EC = checkOffsetForRead(Offset, 1))
5252 return EC;
5353 Buffer = Data.slice(Offset);
5454 return Error::success();
113113 if (Buffer.empty())
114114 return Error::success();
115115
116 if (auto EC = checkOffset(Offset, Buffer.size()))
116 if (auto EC = checkOffsetForWrite(Offset, Buffer.size()))
117117 return EC;
118118
119119 uint8_t *DataPtr = const_cast(Data.data());
128128 private:
129129 MutableArrayRef Data;
130130 BinaryByteStream ImmutableStream;
131 };
132
133 /// \brief An implementation of WritableBinaryStream which can write at its end
134 /// causing the underlying data to grow. This class owns the underlying data.
135 class AppendingBinaryByteStream : public WritableBinaryStream {
136 std::vector Data;
137 llvm::support::endianness Endian;
138
139 public:
140 AppendingBinaryByteStream() = default;
141 AppendingBinaryByteStream(llvm::support::endianness Endian)
142 : Endian(Endian) {}
143
144 void clear() { Data.clear(); }
145
146 llvm::support::endianness getEndian() const override { return Endian; }
147
148 Error readBytes(uint32_t Offset, uint32_t Size,
149 ArrayRef &Buffer) override {
150 if (auto EC = checkOffsetForWrite(Offset, Buffer.size()))
151 return EC;
152
153 Buffer = makeArrayRef(Data).slice(Offset, Size);
154 return Error::success();
155 }
156
157 Error readLongestContiguousChunk(uint32_t Offset,
158 ArrayRef &Buffer) override {
159 if (auto EC = checkOffsetForWrite(Offset, 1))
160 return EC;
161
162 Buffer = makeArrayRef(Data).slice(Offset);
163 return Error::success();
164 }
165
166 uint32_t getLength() override { return Data.size(); }
167
168 Error writeBytes(uint32_t Offset, ArrayRef Buffer) override {
169 if (Buffer.empty())
170 return Error::success();
171
172 // This is well-defined for any case except where offset is strictly
173 // greater than the current length. If offset is equal to the current
174 // length, we can still grow. If offset is beyond the current length, we
175 // would have to decide how to deal with the intermediate uninitialized
176 // bytes. So we punt on that case for simplicity and just say it's an
177 // error.
178 if (Offset > getLength())
179 return make_error(stream_error_code::invalid_offset);
180
181 uint32_t RequiredSize = Offset + Buffer.size();
182 if (RequiredSize > Data.size())
183 Data.resize(RequiredSize);
184
185 ::memcpy(Data.data() + Offset, Buffer.data(), Buffer.size());
186 return Error::success();
187 }
188
189 Error commit() override { return Error::success(); }
190
191 /// \brief Return the properties of this stream.
192 virtual BinaryStreamFlags getFlags() const override {
193 return BSF_Write | BSF_Append;
194 }
195
196 MutableArrayRef data() { return Data; }
131197 };
132198
133199 /// \brief An implementation of WritableBinaryStream backed by an llvm
4444 if (!ExpectedIndex)
4545 return ExpectedIndex.takeError();
4646 const auto &Item = Items[*ExpectedIndex];
47 if (auto EC = checkOffset(Offset, Size))
47 if (auto EC = checkOffsetForRead(Offset, Size))
4848 return EC;
4949 if (Size > Traits::length(Item))
5050 return make_error(stream_error_code::stream_too_short);
1010 #define LLVM_SUPPORT_BINARYSTREAM_H
1111
1212 #include "llvm/ADT/ArrayRef.h"
13 #include "llvm/ADT/BitmaskEnum.h"
1314 #include "llvm/Support/BinaryStreamError.h"
1415 #include "llvm/Support/Endian.h"
1516 #include "llvm/Support/Error.h"
1617 #include
1718
1819 namespace llvm {
20
21 enum BinaryStreamFlags {
22 BSF_None = 0,
23 BSF_Write = 1, // Stream supports writing.
24 BSF_Append = 2, // Writing can occur at offset == length.
25 LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue = */ BSF_Append)
26 };
1927
2028 /// \brief An interface for accessing data in a stream-like format, but which
2129 /// discourages copying. Instead of specifying a buffer in which to copy
4452 /// \brief Return the number of bytes of data in this stream.
4553 virtual uint32_t getLength() = 0;
4654
55 /// \brief Return the properties of this stream.
56 virtual BinaryStreamFlags getFlags() const { return BSF_None; }
57
4758 protected:
48 Error checkOffset(uint32_t Offset, uint32_t DataSize) {
59 Error checkOffsetForRead(uint32_t Offset, uint32_t DataSize) {
4960 if (Offset > getLength())
5061 return make_error(stream_error_code::invalid_offset);
5162 if (getLength() < DataSize + Offset)
7081
7182 /// \brief For buffered streams, commits changes to the backing store.
7283 virtual Error commit() = 0;
84
85 /// \brief Return the properties of this stream.
86 BinaryStreamFlags getFlags() const override { return BSF_Write; }
87
88 protected:
89 Error checkOffsetForWrite(uint32_t Offset, uint32_t DataSize) {
90 if (!(getFlags() & BSF_Append))
91 return checkOffsetForRead(Offset, DataSize);
92
93 if (Offset > getLength())
94 return make_error(stream_error_code::invalid_offset);
95 return Error::success();
96 }
7397 };
7498
7599 } // end namespace llvm
1010 #define LLVM_SUPPORT_BINARYSTREAMREF_H
1111
1212 #include "llvm/ADT/ArrayRef.h"
13 #include "llvm/ADT/Optional.h"
1314 #include "llvm/Support/BinaryStream.h"
1415 #include "llvm/Support/BinaryStreamError.h"
1516 #include "llvm/Support/Error.h"
2324 template class BinaryStreamRefBase {
2425 protected:
2526 BinaryStreamRefBase() = default;
27 explicit BinaryStreamRefBase(StreamType &BorrowedImpl)
28 : BorrowedImpl(&BorrowedImpl), ViewOffset(0) {
29 if (!(BorrowedImpl.getFlags() & BSF_Append))
30 Length = BorrowedImpl.getLength();
31 }
32
2633 BinaryStreamRefBase(std::shared_ptr SharedImpl, uint32_t Offset,
27 uint32_t Length)
34 Optional Length)
2835 : SharedImpl(SharedImpl), BorrowedImpl(SharedImpl.get()),
2936 ViewOffset(Offset), Length(Length) {}
3037 BinaryStreamRefBase(StreamType &BorrowedImpl, uint32_t Offset,
31 uint32_t Length)
38 Optional Length)
3239 : BorrowedImpl(&BorrowedImpl), ViewOffset(Offset), Length(Length) {}
3340 BinaryStreamRefBase(const BinaryStreamRefBase &Other) = default;
3441 BinaryStreamRefBase &operator=(const BinaryStreamRefBase &Other) = default;
4148 return BorrowedImpl->getEndian();
4249 }
4350
44 uint32_t getLength() const { return Length; }
45
46 /// Return a new BinaryStreamRef with the first \p N elements removed.
51 uint32_t getLength() const {
52 if (Length.hasValue())
53 return *Length;
54
55 return BorrowedImpl ? (BorrowedImpl->getLength() - ViewOffset) : 0;
56 }
57
58 /// Return a new BinaryStreamRef with the first \p N elements removed. If
59 /// this BinaryStreamRef is length-tracking, then the resulting one will be
60 /// too.
4761 RefType drop_front(uint32_t N) const {
4862 if (!BorrowedImpl)
4963 return RefType();
5064
51 N = std::min(N, Length);
65 N = std::min(N, getLength());
5266 RefType Result(static_cast(*this));
67 if (N == 0)
68 return Result;
69
5370 Result.ViewOffset += N;
54 Result.Length -= N;
71 if (Result.Length.hasValue())
72 *Result.Length -= N;
5573 return Result;
5674 }
5775
58 /// Return a new BinaryStreamRef with the first \p N elements removed.
76 /// Return a new BinaryStreamRef with the last \p N elements removed. If
77 /// this BinaryStreamRef is length-tracking and \p N is greater than 0, then
78 /// this BinaryStreamRef will no longer length-track.
5979 RefType drop_back(uint32_t N) const {
6080 if (!BorrowedImpl)
6181 return RefType();
6282
63 N = std::min(N, Length);
6483 RefType Result(static_cast(*this));
65 Result.Length -= N;
84 N = std::min(N, getLength());
85
86 if (N == 0)
87 return Result;
88
89 // Since we're dropping non-zero bytes from the end, stop length-tracking
90 // by setting the length of the resulting StreamRef to an explicit value.
91 if (!Result.Length.hasValue())
92 Result.Length = getLength();
93
94 *Result.Length -= N;
6695 return Result;
6796 }
6897
103132 }
104133
105134 protected:
106 Error checkOffset(uint32_t Offset, uint32_t DataSize) const {
135 Error checkOffsetForRead(uint32_t Offset, uint32_t DataSize) const {
107136 if (Offset > getLength())
108137 return make_error(stream_error_code::invalid_offset);
109138 if (getLength() < DataSize + Offset)
114143 std::shared_ptr SharedImpl;
115144 StreamType *BorrowedImpl = nullptr;
116145 uint32_t ViewOffset = 0;
117 uint32_t Length = 0;
146 Optional Length;
118147 };
119148
120149 /// \brief BinaryStreamRef is to BinaryStream what ArrayRef is to an Array. It
129158 friend BinaryStreamRefBase;
130159 friend class WritableBinaryStreamRef;
131160 BinaryStreamRef(std::shared_ptr Impl, uint32_t ViewOffset,
132 uint32_t Length)
161 Optional Length)
133162 : BinaryStreamRefBase(Impl, ViewOffset, Length) {}
134163
135164 public:
136165 BinaryStreamRef() = default;
137166 BinaryStreamRef(BinaryStream &Stream);
138 BinaryStreamRef(BinaryStream &Stream, uint32_t Offset, uint32_t Length);
167 BinaryStreamRef(BinaryStream &Stream, uint32_t Offset,
168 Optional Length);
139169 explicit BinaryStreamRef(ArrayRef Data,
140170 llvm::support::endianness Endian);
141171 explicit BinaryStreamRef(StringRef Data, llvm::support::endianness Endian);
194224 WritableBinaryStream> {
195225 friend BinaryStreamRefBase;
196226 WritableBinaryStreamRef(std::shared_ptr Impl,
197 uint32_t ViewOffset, uint32_t Length)
227 uint32_t ViewOffset, Optional Length)
198228 : BinaryStreamRefBase(Impl, ViewOffset, Length) {}
229
230 Error checkOffsetForWrite(uint32_t Offset, uint32_t DataSize) const {
231 if (!(BorrowedImpl->getFlags() & BSF_Append))
232 return checkOffsetForRead(Offset, DataSize);
233
234 if (Offset > getLength())
235 return make_error(stream_error_code::invalid_offset);
236 return Error::success();
237 }
199238
200239 public:
201240 WritableBinaryStreamRef() = default;
202241 WritableBinaryStreamRef(WritableBinaryStream &Stream);
203242 WritableBinaryStreamRef(WritableBinaryStream &Stream, uint32_t Offset,
204 uint32_t Length);
243 Optional Length);
205244 explicit WritableBinaryStreamRef(MutableArrayRef Data,
206245 llvm::support::endianness Endian);
207246 WritableBinaryStreamRef(const WritableBinaryStreamRef &Other) = default;
8888 Error MappedBlockStream::readBytes(uint32_t Offset, uint32_t Size,
8989 ArrayRef &Buffer) {
9090 // Make sure we aren't trying to read beyond the end of the stream.
91 if (auto EC = checkOffset(Offset, Size))
91 if (auto EC = checkOffsetForRead(Offset, Size))
9292 return EC;
9393
9494 if (tryReadContiguously(Offset, Size, Buffer))
166166 Error MappedBlockStream::readLongestContiguousChunk(uint32_t Offset,
167167 ArrayRef &Buffer) {
168168 // Make sure we aren't trying to read beyond the end of the stream.
169 if (auto EC = checkOffset(Offset, 1))
169 if (auto EC = checkOffsetForRead(Offset, 1))
170170 return EC;
171171
172172 uint32_t First = Offset / BlockSize;
242242 uint32_t OffsetInBlock = Offset % BlockSize;
243243
244244 // Make sure we aren't trying to read beyond the end of the stream.
245 if (auto EC = checkOffset(Offset, Buffer.size()))
245 if (auto EC = checkOffsetForRead(Offset, Buffer.size()))
246246 return EC;
247247
248248 uint32_t BytesLeft = Buffer.size();
387387 Error WritableMappedBlockStream::writeBytes(uint32_t Offset,
388388 ArrayRef Buffer) {
389389 // Make sure we aren't trying to write beyond the end of the stream.
390 if (auto EC = checkOffset(Offset, Buffer.size()))
390 if (auto EC = checkOffsetForWrite(Offset, Buffer.size()))
391391 return EC;
392392
393393 uint32_t BlockNum = Offset / getBlockSize();
6565 }
6666
6767 BinaryStreamRef::BinaryStreamRef(BinaryStream &Stream)
68 : BinaryStreamRef(Stream, 0, Stream.getLength()) {}
68 : BinaryStreamRefBase(Stream) {}
6969 BinaryStreamRef::BinaryStreamRef(BinaryStream &Stream, uint32_t Offset,
70 uint32_t Length)
70 Optional Length)
7171 : BinaryStreamRefBase(Stream, Offset, Length) {}
7272 BinaryStreamRef::BinaryStreamRef(ArrayRef Data, endianness Endian)
7373 : BinaryStreamRefBase(std::make_shared(Data, Endian), 0,
7878
7979 Error BinaryStreamRef::readBytes(uint32_t Offset, uint32_t Size,
8080 ArrayRef &Buffer) const {
81 if (auto EC = checkOffset(Offset, Size))
81 if (auto EC = checkOffsetForRead(Offset, Size))
8282 return EC;
8383 return BorrowedImpl->readBytes(ViewOffset + Offset, Size, Buffer);
8484 }
8585
8686 Error BinaryStreamRef::readLongestContiguousChunk(
8787 uint32_t Offset, ArrayRef &Buffer) const {
88 if (auto EC = checkOffset(Offset, 1))
88 if (auto EC = checkOffsetForRead(Offset, 1))
8989 return EC;
9090
9191 if (auto EC =
9494 // This StreamRef might refer to a smaller window over a larger stream. In
9595 // that case we will have read out more bytes than we should return, because
9696 // we should not read past the end of the current view.
97 uint32_t MaxLength = Length - Offset;
97 uint32_t MaxLength = getLength() - Offset;
9898 if (Buffer.size() > MaxLength)
9999 Buffer = Buffer.slice(0, MaxLength);
100100 return Error::success();
101101 }
102102
103103 WritableBinaryStreamRef::WritableBinaryStreamRef(WritableBinaryStream &Stream)
104 : WritableBinaryStreamRef(Stream, 0, Stream.getLength()) {}
104 : BinaryStreamRefBase(Stream) {}
105105
106106 WritableBinaryStreamRef::WritableBinaryStreamRef(WritableBinaryStream &Stream,
107107 uint32_t Offset,
108 uint32_t Length)
108 Optional Length)
109109 : BinaryStreamRefBase(Stream, Offset, Length) {}
110110
111111 WritableBinaryStreamRef::WritableBinaryStreamRef(MutableArrayRef Data,
116116
117117 Error WritableBinaryStreamRef::writeBytes(uint32_t Offset,
118118 ArrayRef Data) const {
119 if (auto EC = checkOffset(Offset, Data.size()))
119 if (auto EC = checkOffsetForWrite(Offset, Data.size()))
120120 return EC;
121121
122122 return BorrowedImpl->writeBytes(ViewOffset + Offset, Data);
4141 }
4242
4343 Error BinaryStreamWriter::writeFixedString(StringRef Str) {
44 return writeBytes(ArrayRef(Str.bytes_begin(), Str.bytes_end()));
44
45 return writeBytes(arrayRefFromStringRef(Str));
4546 }
4647
4748 Error BinaryStreamWriter::writeStreamRef(BinaryStreamRef Ref) {
4141
4242 Error readBytes(uint32_t Offset, uint32_t Size,
4343 ArrayRef &Buffer) override {
44 if (auto EC = checkOffset(Offset, Size))
44 if (auto EC = checkOffsetForRead(Offset, Size))
4545 return EC;
4646 Buffer = Data.slice(Offset, Size);
4747 return Error::success();
4949
5050 Error readLongestContiguousChunk(uint32_t Offset,
5151 ArrayRef &Buffer) override {
52 if (auto EC = checkOffset(Offset, 1))
52 if (auto EC = checkOffsetForRead(Offset, 1))
5353 return EC;
5454 Buffer = Data.drop_front(Offset);
5555 return Error::success();
5858 uint32_t getLength() override { return Data.size(); }
5959
6060 Error writeBytes(uint32_t Offset, ArrayRef SrcData) override {
61 if (auto EC = checkOffset(Offset, SrcData.size()))
61 if (auto EC = checkOffsetForWrite(Offset, SrcData.size()))
6262 return EC;
6363 ::memcpy(&Data[Offset], SrcData.data(), SrcData.size());
6464 return Error::success();
3535
3636 Error readBytes(uint32_t Offset, uint32_t Size,
3737 ArrayRef &Buffer) override {
38 if (auto EC = checkOffset(Offset, Size))
38 if (auto EC = checkOffsetForRead(Offset, Size))
3939 return EC;
4040 uint32_t S = startIndex(Offset);
4141 auto Ref = Data.drop_front(S);
5454
5555 Error readLongestContiguousChunk(uint32_t Offset,
5656 ArrayRef &Buffer) override {
57 if (auto EC = checkOffset(Offset, 1))
57 if (auto EC = checkOffsetForRead(Offset, 1))
5858 return EC;
5959 uint32_t S = startIndex(Offset);
6060 Buffer = Data.drop_front(S);
6464 uint32_t getLength() override { return Data.size(); }
6565
6666 Error writeBytes(uint32_t Offset, ArrayRef SrcData) override {
67 if (auto EC = checkOffset(Offset, SrcData.size()))
67 if (auto EC = checkOffsetForWrite(Offset, SrcData.size()))
6868 return EC;
6969 if (SrcData.empty())
7070 return Error::success();
266266 }
267267 }
268268
269 TEST_F(BinaryStreamTest, StreamRefDynamicSize) {
270 StringRef Strings[] = {"1", "2", "3", "4"};
271 AppendingBinaryByteStream Stream(support::little);
272
273 BinaryStreamWriter Writer(Stream);
274 BinaryStreamReader Reader(Stream);
275 const uint8_t *Byte;
276 StringRef Str;
277
278 // When the stream is empty, it should report a 0 length and we should get an
279 // error trying to read even 1 byte from it.
280 BinaryStreamRef ConstRef(Stream);
281 EXPECT_EQ(0, ConstRef.getLength());
282 EXPECT_THAT_ERROR(Reader.readObject(Byte), Failed());
283
284 // But if we write to it, its size should increase and we should be able to
285 // read not just a byte, but the string that was written.
286 EXPECT_THAT_ERROR(Writer.writeCString(Strings[0]), Succeeded());
287 EXPECT_EQ(2, ConstRef.getLength());
288 EXPECT_THAT_ERROR(Reader.readObject(Byte), Succeeded());
289
290 Reader.setOffset(0);
291 EXPECT_THAT_ERROR(Reader.readCString(Str), Succeeded());
292 EXPECT_EQ(Str, Strings[0]);
293
294 // If we drop some bytes from the front, we should still track the length as
295 // the
296 // underlying stream grows.
297 BinaryStreamRef Dropped = ConstRef.drop_front(1);
298 EXPECT_EQ(1, Dropped.getLength());
299
300 EXPECT_THAT_ERROR(Writer.writeCString(Strings[1]), Succeeded());
301 EXPECT_EQ(4, ConstRef.getLength());
302 EXPECT_EQ(3, Dropped.getLength());
303
304 // If we drop zero bytes from the back, we should continue tracking the
305 // length.
306 Dropped = Dropped.drop_back(0);
307 EXPECT_THAT_ERROR(Writer.writeCString(Strings[2]), Succeeded());
308 EXPECT_EQ(6, ConstRef.getLength());
309 EXPECT_EQ(5, Dropped.getLength());
310
311 // If we drop non-zero bytes from the back, we should stop tracking the
312 // length.
313 Dropped = Dropped.drop_back(1);
314 EXPECT_THAT_ERROR(Writer.writeCString(Strings[3]), Succeeded());
315 EXPECT_EQ(8, ConstRef.getLength());
316 EXPECT_EQ(4, Dropped.getLength());
317 }
318
269319 TEST_F(BinaryStreamTest, DropOperations) {
270320 std::vector InputData = {1, 2, 3, 4, 5, 4, 3, 2, 1};
271321 auto RefData = makeArrayRef(InputData);
342392 // 2. If the write is too big, it should fail.
343393 EXPECT_THAT_ERROR(Stream.Output->writeBytes(3, BigData), Failed());
344394 }
395 }
396
397 TEST_F(BinaryStreamTest, AppendingStream) {
398 AppendingBinaryByteStream Stream(llvm::support::little);
399 EXPECT_EQ(0, Stream.getLength());
400
401 std::vector InputData = {'T', 'e', 's', 't', 'T', 'e', 's', 't'};
402 auto Test = makeArrayRef(InputData).take_front(4);
403 // Writing past the end of the stream is an error.
404 EXPECT_THAT_ERROR(Stream.writeBytes(4, Test), Failed());
405
406 // Writing exactly at the end of the stream is ok.
407 EXPECT_THAT_ERROR(Stream.writeBytes(0, Test), Succeeded());
408 EXPECT_EQ(Test, Stream.data());
409
410 // And now that the end of the stream is where we couldn't write before, now
411 // we can write.
412 EXPECT_THAT_ERROR(Stream.writeBytes(4, Test), Succeeded());
413 EXPECT_EQ(MutableArrayRef(InputData), Stream.data());
345414 }
346415
347416 // Test that FixedStreamArray works correctly.
692761 EXPECT_EQ(makeArrayRef(Strings), makeArrayRef(InStrings));
693762 }
694763 }
764
765 TEST_F(BinaryStreamTest, StreamWriterAppend) {
766 StringRef Strings[] = {"First", "Second", "Third", "Fourth"};
767 AppendingBinaryByteStream Stream(support::little);
768 BinaryStreamWriter Writer(Stream);
769
770 for (auto &Str : Strings) {
771 EXPECT_THAT_ERROR(Writer.writeCString(Str), Succeeded());
772 }
773
774 BinaryStreamReader Reader(Stream);
775 for (auto &Str : Strings) {
776 StringRef S;
777 EXPECT_THAT_ERROR(Reader.readCString(S), Succeeded());
778 EXPECT_EQ(Str, S);
779 }
780 }
695781 }
696782
697783 namespace {