llvm.org GIT mirror llvm / 50b0dc9
[pdb/lld] Write a valid FPM. The PDB reserves certain blocks for the FPM that describe which blocks in the file are allocated and which are free. We weren't filling that out at all, and in some cases we were even stomping it with incorrect data. This patch writes a correct FPM. Differential Revision: https://reviews.llvm.org/D36235 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@309896 91177308-0d34-0410-b5e6-96231b3b80d8 Zachary Turner 2 years ago
14 changed file(s) with 336 addition(s) and 20 deletion(s). Raw diff Collapse all Expand all
7373 /// \brief Determine the layout of the FPM stream, given the MSF layout. An FPM
7474 /// stream spans 1 or more blocks, each at equally spaced intervals throughout
7575 /// the file.
76 MSFStreamLayout getFpmStreamLayout(const MSFLayout &Msf);
76 MSFStreamLayout getFpmStreamLayout(const MSFLayout &Msf,
77 bool IncludeUnusedFpmData = false,
78 bool AltFpm = false);
7779
7880 inline bool isValidBlockSize(uint32_t Size) {
7981 switch (Size) {
9496 inline uint32_t getFirstUnreservedBlock() { return 3; }
9597
9698 inline uint64_t bytesToBlocks(uint64_t NumBytes, uint64_t BlockSize) {
97 return alignTo(NumBytes, BlockSize) / BlockSize;
99 return divideCeil(NumBytes, BlockSize);
98100 }
99101
100102 inline uint64_t blockToOffset(uint64_t BlockNumber, uint64_t BlockSize) {
105107 return L.SB->BlockSize;
106108 }
107109
108 inline uint32_t getNumFpmIntervals(const MSFLayout &L) {
109 uint32_t Length = getFpmIntervalLength(L);
110 return alignTo(L.SB->NumBlocks, Length) / Length;
111 }
110 inline uint32_t getNumFpmIntervals(const MSFLayout &L,
111 bool IncludeUnusedFpmData = false) {
112 if (IncludeUnusedFpmData)
113 return divideCeil(L.SB->NumBlocks, L.SB->BlockSize);
112114
113 inline uint32_t getFullFpmByteSize(const MSFLayout &L) {
114 return alignTo(L.SB->NumBlocks, 8) / 8;
115 // We want the minimum number of intervals required, where each interval can
116 // represent BlockSize * 8 blocks.
117 return divideCeil(L.SB->NumBlocks, 8 * L.SB->BlockSize);
115118 }
116119
117120 Error validateSuperBlock(const SuperBlock &SB);
121121
122122 static std::unique_ptr
123123 createFpmStream(const MSFLayout &Layout, WritableBinaryStreamRef MsfData,
124 BumpPtrAllocator &Allocator);
124 BumpPtrAllocator &Allocator, bool AltFpm = false);
125125
126126 support::endianness getEndian() const override {
127127 return support::little;
6060 private:
6161 Expected finalizeMsfLayout();
6262
63 void commitFpm(WritableBinaryStream &MsfBuffer, const msf::MSFLayout &Layout);
64
6365 BumpPtrAllocator &Allocator;
6466
6567 std::unique_ptr Msf;
686686 return (Value + Align - 1) / Align * Align;
687687 }
688688
689 /// Returns the integer ceil(Numerator / Denominator).
690 inline uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator) {
691 return alignTo(Numerator, Denominator) / Denominator;
692 }
693
689694 /// \c alignTo for contexts where a constant expression is required.
690695 /// \sa alignTo
691696 ///
106106 return make_error(msf_error_code::insufficient_buffer,
107107 "There are no free Blocks in the file");
108108 uint32_t AllocBlocks = NumBlocks - NumFreeBlocks;
109 FreeBlocks.resize(AllocBlocks + FreeBlocks.size(), true);
109 uint32_t OldBlockCount = FreeBlocks.size();
110 uint32_t NewBlockCount = AllocBlocks + OldBlockCount;
111 uint32_t NextFpmBlock = alignTo(OldBlockCount, BlockSize) + 1;
112 FreeBlocks.resize(NewBlockCount, true);
113 // If we crossed over an fpm page, we actually need to allocate 2 extra
114 // blocks for each FPM group crossed and mark both blocks from the group as
115 // used. We may not actually use them since there are many more FPM blocks
116 // present than are required to represent all blocks in a given PDB, but we
117 // need to make sure they aren't allocated to a stream or something else.
118 // At the end when committing the PDB, we'll go through and mark the
119 // extraneous ones unused.
120 while (NextFpmBlock < NewBlockCount) {
121 NewBlockCount += 2;
122 FreeBlocks.resize(NewBlockCount, true);
123 FreeBlocks.reset(NextFpmBlock, NextFpmBlock + 2);
124 NextFpmBlock += BlockSize;
125 }
110126 }
111127
112128 int I = 0;
228244 return Size;
229245 }
230246
247 static void finalizeFpmBlockStatus(uint32_t B, ArrayRef &FpmBlocks,
248 BitVector &Fpm) {
249 if (FpmBlocks.empty() || FpmBlocks.front() != B) {
250 Fpm.set(B);
251 return;
252 }
253
254 // If the next block in the actual layout is this block, it should *not* be
255 // free.
256 assert(!Fpm.test(B));
257 FpmBlocks = FpmBlocks.drop_front();
258 }
259
231260 Expected MSFBuilder::build() {
232261 SuperBlock *SB = Allocator.Allocate();
233262 MSFLayout L;
286315 }
287316 }
288317
318 // FPM blocks occur in pairs at every `BlockLength` interval. While blocks of
319 // this form are reserved for FPM blocks, not all blocks of this form will
320 // actually be needed for FPM data because there are more blocks of this form
321 // than are required to represent a PDB file with a given number of blocks.
322 // So we need to find out which blocks are *actually* going to be real FPM
323 // blocks, then mark the reset of the reserved blocks as unallocated.
324 MSFStreamLayout FpmLayout = msf::getFpmStreamLayout(L, true);
325 auto FpmBlocks = makeArrayRef(FpmLayout.Blocks);
326 for (uint32_t B = kFreePageMap0Block; B < SB->NumBlocks;
327 B += msf::getFpmIntervalLength(L)) {
328 finalizeFpmBlockStatus(B, FpmBlocks, FreeBlocks);
329 finalizeFpmBlockStatus(B + 1, FpmBlocks, FreeBlocks);
330 }
331 L.FreePageMap = FreeBlocks;
332
289333 return L;
290334 }
5959 return Error::success();
6060 }
6161
62 MSFStreamLayout llvm::msf::getFpmStreamLayout(const MSFLayout &Msf) {
62 MSFStreamLayout llvm::msf::getFpmStreamLayout(const MSFLayout &Msf,
63 bool IncludeUnusedFpmData,
64 bool AltFpm) {
6365 MSFStreamLayout FL;
64 uint32_t NumFpmIntervals = getNumFpmIntervals(Msf);
66 uint32_t NumFpmIntervals = getNumFpmIntervals(Msf, IncludeUnusedFpmData);
6567 support::ulittle32_t FpmBlock = Msf.SB->FreeBlockMapBlock;
6668 assert(FpmBlock == 1 || FpmBlock == 2);
67 while (NumFpmIntervals > 0) {
69 if (AltFpm) {
70 // If they requested the alternate FPM, then 2 becomes 1 and 1 becomes 2.
71 FpmBlock = 3U - FpmBlock;
72 }
73 for (uint32_t I = 0; I < NumFpmIntervals; ++I) {
6874 FL.Blocks.push_back(FpmBlock);
6975 FpmBlock += msf::getFpmIntervalLength(Msf);
70 --NumFpmIntervals;
7176 }
72 FL.Length = getFullFpmByteSize(Msf);
77
78 if (IncludeUnusedFpmData)
79 FL.Length = NumFpmIntervals * Msf.SB->BlockSize;
80 else
81 FL.Length = divideCeil(Msf.SB->NumBlocks, 8);
7382
7483 return FL;
7584 }
1010 #include "llvm/ADT/ArrayRef.h"
1111 #include "llvm/ADT/STLExtras.h"
1212 #include "llvm/DebugInfo/MSF/MSFCommon.h"
13 #include "llvm/Support/BinaryStreamWriter.h"
1314 #include "llvm/Support/Endian.h"
1415 #include "llvm/Support/Error.h"
1516 #include "llvm/Support/MathExtras.h"
346347 std::unique_ptr
347348 WritableMappedBlockStream::createFpmStream(const MSFLayout &Layout,
348349 WritableBinaryStreamRef MsfData,
349 BumpPtrAllocator &Allocator) {
350 MSFStreamLayout SL(getFpmStreamLayout(Layout));
351 return createStream(Layout.SB->BlockSize, SL, MsfData, Allocator);
350 BumpPtrAllocator &Allocator,
351 bool AltFpm) {
352 // We only want to give the user a stream containing the bytes of the FPM that
353 // are actually valid, but we want to initialize all of the bytes, even those
354 // that come from reserved FPM blocks where the entire block is unused. To do
355 // this, we first create the full layout, which gives us a stream with all
356 // bytes and all blocks, and initialize everything to 0xFF (all blocks in the
357 // file are unused). Then we create the minimal layout (which contains only a
358 // subset of the bytes previously initialized), and return that to the user.
359 MSFStreamLayout MinLayout(getFpmStreamLayout(Layout, false, AltFpm));
360
361 MSFStreamLayout FullLayout(getFpmStreamLayout(Layout, true, AltFpm));
362 auto Result =
363 createStream(Layout.SB->BlockSize, FullLayout, MsfData, Allocator);
364 if (!Result)
365 return Result;
366 std::vector InitData(Layout.SB->BlockSize, 0xFF);
367 BinaryStreamWriter Initializer(*Result);
368 while (Initializer.bytesRemaining() > 0)
369 cantFail(Initializer.writeBytes(InitData));
370 return createStream(Layout.SB->BlockSize, MinLayout, MsfData, Allocator);
352371 }
353372
354373 Error WritableMappedBlockStream::readBytes(uint32_t Offset, uint32_t Size,
149149 MappedBlockStream::createFpmStream(ContainerLayout, *Buffer, Allocator);
150150 BinaryStreamReader FpmReader(*FpmStream);
151151 ArrayRef FpmBytes;
152 if (auto EC = FpmReader.readBytes(FpmBytes,
153 msf::getFullFpmByteSize(ContainerLayout)))
152 if (auto EC = FpmReader.readBytes(FpmBytes, FpmReader.bytesRemaining()))
154153 return EC;
155154 uint32_t BlocksRemaining = getBlockCount();
156155 uint32_t BI = 0;
154154 return SN;
155155 }
156156
157 void PDBFileBuilder::commitFpm(WritableBinaryStream &MsfBuffer,
158 const MSFLayout &Layout) {
159 auto FpmStream =
160 WritableMappedBlockStream::createFpmStream(Layout, MsfBuffer, Allocator);
161
162 // We only need to create the alt fpm stream so that it gets initialized.
163 WritableMappedBlockStream::createFpmStream(Layout, MsfBuffer, Allocator,
164 true);
165
166 uint32_t BI = 0;
167 BinaryStreamWriter FpmWriter(*FpmStream);
168 while (BI < Layout.SB->NumBlocks) {
169 uint8_t ThisByte = 0;
170 for (uint32_t I = 0; I < 8; ++I) {
171 bool IsFree =
172 (BI < Layout.SB->NumBlocks) ? Layout.FreePageMap.test(BI) : true;
173 uint8_t Mask = uint8_t(IsFree) << I;
174 ThisByte |= Mask;
175 ++BI;
176 }
177 cantFail(FpmWriter.writeObject(ThisByte));
178 }
179 assert(FpmWriter.bytesRemaining() == 0);
180 }
181
157182 Error PDBFileBuilder::commit(StringRef Filename) {
158183 assert(!Filename.empty());
159184 auto ExpectedLayout = finalizeMsfLayout();
172197
173198 if (auto EC = Writer.writeObject(*Layout.SB))
174199 return EC;
200
201 commitFpm(Buffer, Layout);
202
175203 uint32_t BlockMapOffset =
176204 msf::blockToOffset(Layout.SB->BlockMapAddr, Layout.SB->BlockSize);
177205 Writer.setOffset(BlockMapOffset);
0 ; RUN: llvm-pdbutil yaml2pdb -pdb=%t.pdb %p/Inputs/one-symbol.yaml
1 ; RUN: llvm-pdbutil bytes -fpm %t.pdb | FileCheck %s
2
3
4 CHECK: Free Page Map
5 CHECK-NEXT: ============================================================
6 CHECK-NEXT: Block 1 (
7 CHECK-NEXT: 1000: 04F8FFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF |................................|
8 CHECK-NEXT: 1020: FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF |................................|
9 CHECK: 1FE0: FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF |................................|
10 CHECK: )
44 set(DebugInfoMSFSources
55 MappedBlockStreamTest.cpp
66 MSFBuilderTest.cpp
7 MSFCommonTest.cpp
78 )
89
910 add_llvm_unittest(DebugInfoMSFTests
1010 #include "llvm/DebugInfo/MSF/MSFCommon.h"
1111 #include "llvm/Testing/Support/Error.h"
1212
13 #include "gmock/gmock-matchers.h"
14 #include "gmock/gmock.h"
1315 #include "gtest/gtest.h"
1416
1517 using namespace llvm;
1618 using namespace llvm::msf;
19 using namespace testing;
1720
1821 namespace {
1922 class MSFBuilderTest : public testing::Test {
358361 EXPECT_EQ(1U, L.DirectoryBlocks.size());
359362 EXPECT_EQ(B + 1, L.DirectoryBlocks[0]);
360363 }
364
365 TEST_F(MSFBuilderTest, StreamDoesntUseFpmBlocks) {
366 Expected ExpectedMsf = MSFBuilder::create(Allocator, 4096);
367 ASSERT_THAT_EXPECTED(ExpectedMsf, Succeeded());
368 auto &Msf = *ExpectedMsf;
369
370 // A block is 4096 bytes, and every 4096 blocks we have 2 reserved FPM blocks.
371 // By creating add a stream that spans 4096*4096*3 bytes, we ensure that we
372 // cross over a couple of reserved FPM blocks, and that none of them are
373 // allocated to the stream.
374 constexpr uint32_t StreamSize = 4096 * 4096 * 3;
375 Expected SN = Msf.addStream(StreamSize);
376 ASSERT_THAT_EXPECTED(SN, Succeeded());
377
378 auto ExpectedLayout = Msf.build();
379 ASSERT_THAT_EXPECTED(ExpectedLayout, Succeeded());
380 MSFLayout &L = *ExpectedLayout;
381 auto BlocksRef = L.StreamMap[*SN];
382 std::vector Blocks(BlocksRef.begin(), BlocksRef.end());
383 EXPECT_EQ(StreamSize, L.StreamSizes[*SN]);
384
385 for (uint32_t I = 0; I <= 3; ++I) {
386 // Pages from the regular FPM are allocated, while pages from the alt fpm
387 // are free.
388 EXPECT_FALSE(L.FreePageMap.test(1 + I * 4096));
389 EXPECT_TRUE(L.FreePageMap.test(2 + I * 4096));
390 }
391
392 for (uint32_t I = 1; I <= 3; ++I) {
393 EXPECT_THAT(Blocks, Not(Contains(1 + I * 4096)));
394 EXPECT_THAT(Blocks, Not(Contains(2 + I * 4096)));
395 }
396 }
0 //===- MSFBuilderTest.cpp Tests manipulation of MSF stream metadata ------===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/DebugInfo/MSF/MSFCommon.h"
10 #include "llvm/Testing/Support/Error.h"
11
12 #include "gtest/gtest.h"
13
14 using namespace llvm;
15 using namespace llvm::msf;
16
17 TEST(MSFCommonTest, BytesToBlocks) {
18 EXPECT_EQ(0ULL, bytesToBlocks(0, 4096));
19 EXPECT_EQ(1ULL, bytesToBlocks(12, 4096));
20 EXPECT_EQ(1ULL, bytesToBlocks(4096, 4096));
21 EXPECT_EQ(2ULL, bytesToBlocks(4097, 4096));
22 EXPECT_EQ(2ULL, bytesToBlocks(600, 512));
23 }
24
25 TEST(MSFCommonTest, FpmIntervals) {
26 SuperBlock SB;
27 SB.FreeBlockMapBlock = 1;
28 SB.BlockSize = 4096;
29
30 MSFLayout L;
31 L.SB = &SB;
32
33 SB.NumBlocks = 12;
34 EXPECT_EQ(1u, getNumFpmIntervals(L, false));
35 SB.NumBlocks = SB.BlockSize;
36 EXPECT_EQ(1u, getNumFpmIntervals(L, false));
37 SB.NumBlocks = SB.BlockSize + 1;
38 EXPECT_EQ(1u, getNumFpmIntervals(L, false));
39 SB.NumBlocks = SB.BlockSize * 8;
40 EXPECT_EQ(1u, getNumFpmIntervals(L, false));
41 SB.NumBlocks = SB.BlockSize * 8 + 1;
42 EXPECT_EQ(2u, getNumFpmIntervals(L, false));
43
44 SB.NumBlocks = 12;
45 EXPECT_EQ(1u, getNumFpmIntervals(L, true));
46 SB.NumBlocks = SB.BlockSize;
47 EXPECT_EQ(1u, getNumFpmIntervals(L, true));
48 SB.NumBlocks = SB.BlockSize + 1;
49 EXPECT_EQ(2u, getNumFpmIntervals(L, true));
50 SB.NumBlocks = SB.BlockSize * 8;
51 EXPECT_EQ(8u, getNumFpmIntervals(L, true));
52 SB.NumBlocks = SB.BlockSize * 8 + 1;
53 EXPECT_EQ(9u, getNumFpmIntervals(L, true));
54 }
55
56 TEST(MSFCommonTest, FpmStreamLayout) {
57 SuperBlock SB;
58 MSFLayout L;
59 L.SB = &SB;
60 SB.FreeBlockMapBlock = 1;
61
62 // Each FPM block has 4096 bytes for a maximum of 4096*8 allocation states.
63 SB.BlockSize = 4096;
64
65 // 1. When we're not including unused FPM data, the length of the FPM stream
66 // should be only long enough to contain 1 bit for each block.
67
68 // 1a. When the PDB has <= 4096*8 blocks, there should only be one FPM block.
69 SB.NumBlocks = 8000;
70 MSFStreamLayout SL = getFpmStreamLayout(L, false, false);
71 EXPECT_EQ(1000u, SL.Length);
72 EXPECT_EQ(1u, SL.Blocks.size());
73 EXPECT_EQ(SB.FreeBlockMapBlock, SL.Blocks.front());
74
75 SL = getFpmStreamLayout(L, false, true);
76 EXPECT_EQ(1000u, SL.Length);
77 EXPECT_EQ(1u, SL.Blocks.size());
78 EXPECT_EQ(3u - SB.FreeBlockMapBlock, SL.Blocks.front());
79
80 // 1b. When the PDB has > 4096*8 blocks, there should be multiple FPM blocks.
81 SB.NumBlocks = SB.BlockSize * 8 + 1;
82 SL = getFpmStreamLayout(L, false, false);
83 EXPECT_EQ(SB.BlockSize + 1, SL.Length);
84 EXPECT_EQ(2u, SL.Blocks.size());
85 EXPECT_EQ(SB.FreeBlockMapBlock, SL.Blocks[0]);
86 EXPECT_EQ(SB.FreeBlockMapBlock + SB.BlockSize, SL.Blocks[1]);
87
88 SL = getFpmStreamLayout(L, false, true);
89 EXPECT_EQ(SB.BlockSize + 1, SL.Length);
90 EXPECT_EQ(2u, SL.Blocks.size());
91 EXPECT_EQ(3u - SB.FreeBlockMapBlock, SL.Blocks[0]);
92 EXPECT_EQ(3u - SB.FreeBlockMapBlock + SB.BlockSize, SL.Blocks[1]);
93
94 // 2. When we are including unused FPM data, there should be one FPM block
95 // at every BlockSize interval in the file, even if entire FPM blocks are
96 // unused.
97 SB.NumBlocks = SB.BlockSize * 8 + 1;
98 SL = getFpmStreamLayout(L, true, false);
99 EXPECT_EQ(SB.BlockSize * 9, SL.Length);
100 EXPECT_EQ(9u, SL.Blocks.size());
101 for (int I = 0; I < 9; ++I)
102 EXPECT_EQ(I * SB.BlockSize + SB.FreeBlockMapBlock, SL.Blocks[I]);
103 }
1515 #include "llvm/Support/BinaryStreamWriter.h"
1616 #include "llvm/Testing/Support/Error.h"
1717
18 #include "gmock/gmock.h"
1819 #include "gtest/gtest.h"
1920
2021 #include
493494
494495 EXPECT_EQ(Str[0], Str[1]);
495496 }
497 } // namespace
498
499 MATCHER_P3(BlockIsFilledWith, Layout, BlockIndex, Byte, "succeeded") {
500 uint64_t Offset = msf::blockToOffset(BlockIndex, Layout.SB->BlockSize);
501 ArrayRef BufferRef = makeArrayRef(arg);
502 BufferRef = BufferRef.slice(Offset, Layout.SB->BlockSize);
503 return llvm::all_of(BufferRef, [this](uint8_t B) { return B == Byte; });
504 }
505
506 namespace {
507 TEST(MappedBlockStreamTest, CreateFpmStream) {
508 BumpPtrAllocator Allocator;
509 SuperBlock SB;
510 MSFLayout L;
511 L.SB = &SB;
512
513 SB.FreeBlockMapBlock = 1;
514 SB.BlockSize = 4096;
515
516 constexpr uint32_t NumFileBlocks = 4096 * 4;
517
518 std::vector MsfBuffer(NumFileBlocks * SB.BlockSize);
519 MutableBinaryByteStream MsfStream(MsfBuffer, llvm::support::little);
520
521 SB.NumBlocks = NumFileBlocks;
522 auto FpmStream =
523 WritableMappedBlockStream::createFpmStream(L, MsfStream, Allocator);
524 // 4096 * 4 / 8 = 2048 bytes of FPM data is needed to describe 4096 * 4
525 // blocks. This translates to 1 FPM block.
526 EXPECT_EQ(2048u, FpmStream->getLength());
527 EXPECT_EQ(1u, FpmStream->getStreamLayout().Blocks.size());
528 EXPECT_EQ(1u, FpmStream->getStreamLayout().Blocks[0]);
529 // All blocks from FPM1 should be 1 initialized, and all blocks from FPM2
530 // should be 0 initialized (since we requested the main FPM, not the alt FPM)
531 for (int I = 0; I < 4; ++I) {
532 EXPECT_THAT(MsfBuffer, BlockIsFilledWith(L, 1 + I * SB.BlockSize, 0xFF));
533 EXPECT_THAT(MsfBuffer, BlockIsFilledWith(L, 2 + I * SB.BlockSize, 0));
534 }
535
536 ::memset(MsfBuffer.data(), 0, MsfBuffer.size());
537 FpmStream =
538 WritableMappedBlockStream::createFpmStream(L, MsfStream, Allocator, true);
539 // 4096 * 4 / 8 = 2048 bytes of FPM data is needed to describe 4096 * 4
540 // blocks. This translates to 1 FPM block.
541 EXPECT_EQ(2048u, FpmStream->getLength());
542 EXPECT_EQ(1u, FpmStream->getStreamLayout().Blocks.size());
543 EXPECT_EQ(2u, FpmStream->getStreamLayout().Blocks[0]);
544 // All blocks from FPM2 should be 1 initialized, and all blocks from FPM1
545 // should be 0 initialized (since we requested the alt FPM, not the main FPM)
546 for (int I = 0; I < 4; ++I) {
547 EXPECT_THAT(MsfBuffer, BlockIsFilledWith(L, 1 + I * SB.BlockSize, 0));
548 EXPECT_THAT(MsfBuffer, BlockIsFilledWith(L, 2 + I * SB.BlockSize, 0xFF));
549 }
550 }
496551
497552 } // end anonymous namespace