diff options
Diffstat (limited to 'contrib/llvm/lib/ExecutionEngine/JIT/JITMemoryManager.cpp')
| -rw-r--r-- | contrib/llvm/lib/ExecutionEngine/JIT/JITMemoryManager.cpp | 727 |
1 files changed, 727 insertions, 0 deletions
diff --git a/contrib/llvm/lib/ExecutionEngine/JIT/JITMemoryManager.cpp b/contrib/llvm/lib/ExecutionEngine/JIT/JITMemoryManager.cpp new file mode 100644 index 0000000..653e6f1 --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/JIT/JITMemoryManager.cpp @@ -0,0 +1,727 @@ +//===-- JITMemoryManager.cpp - Memory Allocator for JIT'd code ------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the DefaultJITMemoryManager class. +// +//===----------------------------------------------------------------------===// + +#define DEBUG_TYPE "jit" +#include "llvm/ExecutionEngine/JITMemoryManager.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/Twine.h" +#include "llvm/GlobalValue.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/System/Memory.h" +#include <vector> +#include <cassert> +#include <climits> +#include <cstring> +using namespace llvm; + +STATISTIC(NumSlabs, "Number of slabs of memory allocated by the JIT"); + +JITMemoryManager::~JITMemoryManager() {} + +//===----------------------------------------------------------------------===// +// Memory Block Implementation. +//===----------------------------------------------------------------------===// + +namespace { + /// MemoryRangeHeader - For a range of memory, this is the header that we put + /// on the block of memory. It is carefully crafted to be one word of memory. + /// Allocated blocks have just this header, free'd blocks have FreeRangeHeader + /// which starts with this. + struct FreeRangeHeader; + struct MemoryRangeHeader { + /// ThisAllocated - This is true if this block is currently allocated. If + /// not, this can be converted to a FreeRangeHeader. + unsigned ThisAllocated : 1; + + /// PrevAllocated - Keep track of whether the block immediately before us is + /// allocated. If not, the word immediately before this header is the size + /// of the previous block. + unsigned PrevAllocated : 1; + + /// BlockSize - This is the size in bytes of this memory block, + /// including this header. + uintptr_t BlockSize : (sizeof(intptr_t)*CHAR_BIT - 2); + + + /// getBlockAfter - Return the memory block immediately after this one. + /// + MemoryRangeHeader &getBlockAfter() const { + return *(MemoryRangeHeader*)((char*)this+BlockSize); + } + + /// getFreeBlockBefore - If the block before this one is free, return it, + /// otherwise return null. + FreeRangeHeader *getFreeBlockBefore() const { + if (PrevAllocated) return 0; + intptr_t PrevSize = ((intptr_t *)this)[-1]; + return (FreeRangeHeader*)((char*)this-PrevSize); + } + + /// FreeBlock - Turn an allocated block into a free block, adjusting + /// bits in the object headers, and adding an end of region memory block. + FreeRangeHeader *FreeBlock(FreeRangeHeader *FreeList); + + /// TrimAllocationToSize - If this allocated block is significantly larger + /// than NewSize, split it into two pieces (where the former is NewSize + /// bytes, including the header), and add the new block to the free list. + FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList, + uint64_t NewSize); + }; + + /// FreeRangeHeader - For a memory block that isn't already allocated, this + /// keeps track of the current block and has a pointer to the next free block. + /// Free blocks are kept on a circularly linked list. + struct FreeRangeHeader : public MemoryRangeHeader { + FreeRangeHeader *Prev; + FreeRangeHeader *Next; + + /// getMinBlockSize - Get the minimum size for a memory block. Blocks + /// smaller than this size cannot be created. + static unsigned getMinBlockSize() { + return sizeof(FreeRangeHeader)+sizeof(intptr_t); + } + + /// SetEndOfBlockSizeMarker - The word at the end of every free block is + /// known to be the size of the free block. Set it for this block. + void SetEndOfBlockSizeMarker() { + void *EndOfBlock = (char*)this + BlockSize; + ((intptr_t *)EndOfBlock)[-1] = BlockSize; + } + + FreeRangeHeader *RemoveFromFreeList() { + assert(Next->Prev == this && Prev->Next == this && "Freelist broken!"); + Next->Prev = Prev; + return Prev->Next = Next; + } + + void AddToFreeList(FreeRangeHeader *FreeList) { + Next = FreeList; + Prev = FreeList->Prev; + Prev->Next = this; + Next->Prev = this; + } + + /// GrowBlock - The block after this block just got deallocated. Merge it + /// into the current block. + void GrowBlock(uintptr_t NewSize); + + /// AllocateBlock - Mark this entire block allocated, updating freelists + /// etc. This returns a pointer to the circular free-list. + FreeRangeHeader *AllocateBlock(); + }; +} + + +/// AllocateBlock - Mark this entire block allocated, updating freelists +/// etc. This returns a pointer to the circular free-list. +FreeRangeHeader *FreeRangeHeader::AllocateBlock() { + assert(!ThisAllocated && !getBlockAfter().PrevAllocated && + "Cannot allocate an allocated block!"); + // Mark this block allocated. + ThisAllocated = 1; + getBlockAfter().PrevAllocated = 1; + + // Remove it from the free list. + return RemoveFromFreeList(); +} + +/// FreeBlock - Turn an allocated block into a free block, adjusting +/// bits in the object headers, and adding an end of region memory block. +/// If possible, coalesce this block with neighboring blocks. Return the +/// FreeRangeHeader to allocate from. +FreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) { + MemoryRangeHeader *FollowingBlock = &getBlockAfter(); + assert(ThisAllocated && "This block is already free!"); + assert(FollowingBlock->PrevAllocated && "Flags out of sync!"); + + FreeRangeHeader *FreeListToReturn = FreeList; + + // If the block after this one is free, merge it into this block. + if (!FollowingBlock->ThisAllocated) { + FreeRangeHeader &FollowingFreeBlock = *(FreeRangeHeader *)FollowingBlock; + // "FreeList" always needs to be a valid free block. If we're about to + // coalesce with it, update our notion of what the free list is. + if (&FollowingFreeBlock == FreeList) { + FreeList = FollowingFreeBlock.Next; + FreeListToReturn = 0; + assert(&FollowingFreeBlock != FreeList && "No tombstone block?"); + } + FollowingFreeBlock.RemoveFromFreeList(); + + // Include the following block into this one. + BlockSize += FollowingFreeBlock.BlockSize; + FollowingBlock = &FollowingFreeBlock.getBlockAfter(); + + // Tell the block after the block we are coalescing that this block is + // allocated. + FollowingBlock->PrevAllocated = 1; + } + + assert(FollowingBlock->ThisAllocated && "Missed coalescing?"); + + if (FreeRangeHeader *PrevFreeBlock = getFreeBlockBefore()) { + PrevFreeBlock->GrowBlock(PrevFreeBlock->BlockSize + BlockSize); + return FreeListToReturn ? FreeListToReturn : PrevFreeBlock; + } + + // Otherwise, mark this block free. + FreeRangeHeader &FreeBlock = *(FreeRangeHeader*)this; + FollowingBlock->PrevAllocated = 0; + FreeBlock.ThisAllocated = 0; + + // Link this into the linked list of free blocks. + FreeBlock.AddToFreeList(FreeList); + + // Add a marker at the end of the block, indicating the size of this free + // block. + FreeBlock.SetEndOfBlockSizeMarker(); + return FreeListToReturn ? FreeListToReturn : &FreeBlock; +} + +/// GrowBlock - The block after this block just got deallocated. Merge it +/// into the current block. +void FreeRangeHeader::GrowBlock(uintptr_t NewSize) { + assert(NewSize > BlockSize && "Not growing block?"); + BlockSize = NewSize; + SetEndOfBlockSizeMarker(); + getBlockAfter().PrevAllocated = 0; +} + +/// TrimAllocationToSize - If this allocated block is significantly larger +/// than NewSize, split it into two pieces (where the former is NewSize +/// bytes, including the header), and add the new block to the free list. +FreeRangeHeader *MemoryRangeHeader:: +TrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize) { + assert(ThisAllocated && getBlockAfter().PrevAllocated && + "Cannot deallocate part of an allocated block!"); + + // Don't allow blocks to be trimmed below minimum required size + NewSize = std::max<uint64_t>(FreeRangeHeader::getMinBlockSize(), NewSize); + + // Round up size for alignment of header. + unsigned HeaderAlign = __alignof(FreeRangeHeader); + NewSize = (NewSize+ (HeaderAlign-1)) & ~(HeaderAlign-1); + + // Size is now the size of the block we will remove from the start of the + // current block. + assert(NewSize <= BlockSize && + "Allocating more space from this block than exists!"); + + // If splitting this block will cause the remainder to be too small, do not + // split the block. + if (BlockSize <= NewSize+FreeRangeHeader::getMinBlockSize()) + return FreeList; + + // Otherwise, we splice the required number of bytes out of this block, form + // a new block immediately after it, then mark this block allocated. + MemoryRangeHeader &FormerNextBlock = getBlockAfter(); + + // Change the size of this block. + BlockSize = NewSize; + + // Get the new block we just sliced out and turn it into a free block. + FreeRangeHeader &NewNextBlock = (FreeRangeHeader &)getBlockAfter(); + NewNextBlock.BlockSize = (char*)&FormerNextBlock - (char*)&NewNextBlock; + NewNextBlock.ThisAllocated = 0; + NewNextBlock.PrevAllocated = 1; + NewNextBlock.SetEndOfBlockSizeMarker(); + FormerNextBlock.PrevAllocated = 0; + NewNextBlock.AddToFreeList(FreeList); + return &NewNextBlock; +} + +//===----------------------------------------------------------------------===// +// Memory Block Implementation. +//===----------------------------------------------------------------------===// + +namespace { + + class DefaultJITMemoryManager; + + class JITSlabAllocator : public SlabAllocator { + DefaultJITMemoryManager &JMM; + public: + JITSlabAllocator(DefaultJITMemoryManager &jmm) : JMM(jmm) { } + virtual ~JITSlabAllocator() { } + virtual MemSlab *Allocate(size_t Size); + virtual void Deallocate(MemSlab *Slab); + }; + + /// DefaultJITMemoryManager - Manage memory for the JIT code generation. + /// This splits a large block of MAP_NORESERVE'd memory into two + /// sections, one for function stubs, one for the functions themselves. We + /// have to do this because we may need to emit a function stub while in the + /// middle of emitting a function, and we don't know how large the function we + /// are emitting is. + class DefaultJITMemoryManager : public JITMemoryManager { + + // Whether to poison freed memory. + bool PoisonMemory; + + /// LastSlab - This points to the last slab allocated and is used as the + /// NearBlock parameter to AllocateRWX so that we can attempt to lay out all + /// stubs, data, and code contiguously in memory. In general, however, this + /// is not possible because the NearBlock parameter is ignored on Windows + /// platforms and even on Unix it works on a best-effort pasis. + sys::MemoryBlock LastSlab; + + // Memory slabs allocated by the JIT. We refer to them as slabs so we don't + // confuse them with the blocks of memory described above. + std::vector<sys::MemoryBlock> CodeSlabs; + JITSlabAllocator BumpSlabAllocator; + BumpPtrAllocator StubAllocator; + BumpPtrAllocator DataAllocator; + + // Circular list of free blocks. + FreeRangeHeader *FreeMemoryList; + + // When emitting code into a memory block, this is the block. + MemoryRangeHeader *CurBlock; + + uint8_t *GOTBase; // Target Specific reserved memory + public: + DefaultJITMemoryManager(); + ~DefaultJITMemoryManager(); + + /// allocateNewSlab - Allocates a new MemoryBlock and remembers it as the + /// last slab it allocated, so that subsequent allocations follow it. + sys::MemoryBlock allocateNewSlab(size_t size); + + /// DefaultCodeSlabSize - When we have to go map more memory, we allocate at + /// least this much unless more is requested. + static const size_t DefaultCodeSlabSize; + + /// DefaultSlabSize - Allocate data into slabs of this size unless we get + /// an allocation above SizeThreshold. + static const size_t DefaultSlabSize; + + /// DefaultSizeThreshold - For any allocation larger than this threshold, we + /// should allocate a separate slab. + static const size_t DefaultSizeThreshold; + + void AllocateGOT(); + + // Testing methods. + virtual bool CheckInvariants(std::string &ErrorStr); + size_t GetDefaultCodeSlabSize() { return DefaultCodeSlabSize; } + size_t GetDefaultDataSlabSize() { return DefaultSlabSize; } + size_t GetDefaultStubSlabSize() { return DefaultSlabSize; } + unsigned GetNumCodeSlabs() { return CodeSlabs.size(); } + unsigned GetNumDataSlabs() { return DataAllocator.GetNumSlabs(); } + unsigned GetNumStubSlabs() { return StubAllocator.GetNumSlabs(); } + + /// startFunctionBody - When a function starts, allocate a block of free + /// executable memory, returning a pointer to it and its actual size. + uint8_t *startFunctionBody(const Function *F, uintptr_t &ActualSize) { + + FreeRangeHeader* candidateBlock = FreeMemoryList; + FreeRangeHeader* head = FreeMemoryList; + FreeRangeHeader* iter = head->Next; + + uintptr_t largest = candidateBlock->BlockSize; + + // Search for the largest free block + while (iter != head) { + if (iter->BlockSize > largest) { + largest = iter->BlockSize; + candidateBlock = iter; + } + iter = iter->Next; + } + + largest = largest - sizeof(MemoryRangeHeader); + + // If this block isn't big enough for the allocation desired, allocate + // another block of memory and add it to the free list. + if (largest < ActualSize || + largest <= FreeRangeHeader::getMinBlockSize()) { + DEBUG(dbgs() << "JIT: Allocating another slab of memory for function."); + candidateBlock = allocateNewCodeSlab((size_t)ActualSize); + } + + // Select this candidate block for allocation + CurBlock = candidateBlock; + + // Allocate the entire memory block. + FreeMemoryList = candidateBlock->AllocateBlock(); + ActualSize = CurBlock->BlockSize - sizeof(MemoryRangeHeader); + return (uint8_t *)(CurBlock + 1); + } + + /// allocateNewCodeSlab - Helper method to allocate a new slab of code + /// memory from the OS and add it to the free list. Returns the new + /// FreeRangeHeader at the base of the slab. + FreeRangeHeader *allocateNewCodeSlab(size_t MinSize) { + // If the user needs at least MinSize free memory, then we account for + // two MemoryRangeHeaders: the one in the user's block, and the one at the + // end of the slab. + size_t PaddedMin = MinSize + 2 * sizeof(MemoryRangeHeader); + size_t SlabSize = std::max(DefaultCodeSlabSize, PaddedMin); + sys::MemoryBlock B = allocateNewSlab(SlabSize); + CodeSlabs.push_back(B); + char *MemBase = (char*)(B.base()); + + // Put a tiny allocated block at the end of the memory chunk, so when + // FreeBlock calls getBlockAfter it doesn't fall off the end. + MemoryRangeHeader *EndBlock = + (MemoryRangeHeader*)(MemBase + B.size()) - 1; + EndBlock->ThisAllocated = 1; + EndBlock->PrevAllocated = 0; + EndBlock->BlockSize = sizeof(MemoryRangeHeader); + + // Start out with a vast new block of free memory. + FreeRangeHeader *NewBlock = (FreeRangeHeader*)MemBase; + NewBlock->ThisAllocated = 0; + // Make sure getFreeBlockBefore doesn't look into unmapped memory. + NewBlock->PrevAllocated = 1; + NewBlock->BlockSize = (uintptr_t)EndBlock - (uintptr_t)NewBlock; + NewBlock->SetEndOfBlockSizeMarker(); + NewBlock->AddToFreeList(FreeMemoryList); + + assert(NewBlock->BlockSize - sizeof(MemoryRangeHeader) >= MinSize && + "The block was too small!"); + return NewBlock; + } + + /// endFunctionBody - The function F is now allocated, and takes the memory + /// in the range [FunctionStart,FunctionEnd). + void endFunctionBody(const Function *F, uint8_t *FunctionStart, + uint8_t *FunctionEnd) { + assert(FunctionEnd > FunctionStart); + assert(FunctionStart == (uint8_t *)(CurBlock+1) && + "Mismatched function start/end!"); + + uintptr_t BlockSize = FunctionEnd - (uint8_t *)CurBlock; + + // Release the memory at the end of this block that isn't needed. + FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); + } + + /// allocateSpace - Allocate a memory block of the given size. This method + /// cannot be called between calls to startFunctionBody and endFunctionBody. + uint8_t *allocateSpace(intptr_t Size, unsigned Alignment) { + CurBlock = FreeMemoryList; + FreeMemoryList = FreeMemoryList->AllocateBlock(); + + uint8_t *result = (uint8_t *)(CurBlock + 1); + + if (Alignment == 0) Alignment = 1; + result = (uint8_t*)(((intptr_t)result+Alignment-1) & + ~(intptr_t)(Alignment-1)); + + uintptr_t BlockSize = result + Size - (uint8_t *)CurBlock; + FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); + + return result; + } + + /// allocateStub - Allocate memory for a function stub. + uint8_t *allocateStub(const GlobalValue* F, unsigned StubSize, + unsigned Alignment) { + return (uint8_t*)StubAllocator.Allocate(StubSize, Alignment); + } + + /// allocateGlobal - Allocate memory for a global. + uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) { + return (uint8_t*)DataAllocator.Allocate(Size, Alignment); + } + + /// startExceptionTable - Use startFunctionBody to allocate memory for the + /// function's exception table. + uint8_t* startExceptionTable(const Function* F, uintptr_t &ActualSize) { + return startFunctionBody(F, ActualSize); + } + + /// endExceptionTable - The exception table of F is now allocated, + /// and takes the memory in the range [TableStart,TableEnd). + void endExceptionTable(const Function *F, uint8_t *TableStart, + uint8_t *TableEnd, uint8_t* FrameRegister) { + assert(TableEnd > TableStart); + assert(TableStart == (uint8_t *)(CurBlock+1) && + "Mismatched table start/end!"); + + uintptr_t BlockSize = TableEnd - (uint8_t *)CurBlock; + + // Release the memory at the end of this block that isn't needed. + FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); + } + + uint8_t *getGOTBase() const { + return GOTBase; + } + + void deallocateBlock(void *Block) { + // Find the block that is allocated for this function. + MemoryRangeHeader *MemRange = static_cast<MemoryRangeHeader*>(Block) - 1; + assert(MemRange->ThisAllocated && "Block isn't allocated!"); + + // Fill the buffer with garbage! + if (PoisonMemory) { + memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange)); + } + + // Free the memory. + FreeMemoryList = MemRange->FreeBlock(FreeMemoryList); + } + + /// deallocateFunctionBody - Deallocate all memory for the specified + /// function body. + void deallocateFunctionBody(void *Body) { + if (Body) deallocateBlock(Body); + } + + /// deallocateExceptionTable - Deallocate memory for the specified + /// exception table. + void deallocateExceptionTable(void *ET) { + if (ET) deallocateBlock(ET); + } + + /// setMemoryWritable - When code generation is in progress, + /// the code pages may need permissions changed. + void setMemoryWritable() + { + for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) + sys::Memory::setWritable(CodeSlabs[i]); + } + /// setMemoryExecutable - When code generation is done and we're ready to + /// start execution, the code pages may need permissions changed. + void setMemoryExecutable() + { + for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) + sys::Memory::setExecutable(CodeSlabs[i]); + } + + /// setPoisonMemory - Controls whether we write garbage over freed memory. + /// + void setPoisonMemory(bool poison) { + PoisonMemory = poison; + } + }; +} + +MemSlab *JITSlabAllocator::Allocate(size_t Size) { + sys::MemoryBlock B = JMM.allocateNewSlab(Size); + MemSlab *Slab = (MemSlab*)B.base(); + Slab->Size = B.size(); + Slab->NextPtr = 0; + return Slab; +} + +void JITSlabAllocator::Deallocate(MemSlab *Slab) { + sys::MemoryBlock B(Slab, Slab->Size); + sys::Memory::ReleaseRWX(B); +} + +DefaultJITMemoryManager::DefaultJITMemoryManager() + : +#ifdef NDEBUG + PoisonMemory(false), +#else + PoisonMemory(true), +#endif + LastSlab(0, 0), + BumpSlabAllocator(*this), + StubAllocator(DefaultSlabSize, DefaultSizeThreshold, BumpSlabAllocator), + DataAllocator(DefaultSlabSize, DefaultSizeThreshold, BumpSlabAllocator) { + + // Allocate space for code. + sys::MemoryBlock MemBlock = allocateNewSlab(DefaultCodeSlabSize); + CodeSlabs.push_back(MemBlock); + uint8_t *MemBase = (uint8_t*)MemBlock.base(); + + // We set up the memory chunk with 4 mem regions, like this: + // [ START + // [ Free #0 ] -> Large space to allocate functions from. + // [ Allocated #1 ] -> Tiny space to separate regions. + // [ Free #2 ] -> Tiny space so there is always at least 1 free block. + // [ Allocated #3 ] -> Tiny space to prevent looking past end of block. + // END ] + // + // The last three blocks are never deallocated or touched. + + // Add MemoryRangeHeader to the end of the memory region, indicating that + // the space after the block of memory is allocated. This is block #3. + MemoryRangeHeader *Mem3 = (MemoryRangeHeader*)(MemBase+MemBlock.size())-1; + Mem3->ThisAllocated = 1; + Mem3->PrevAllocated = 0; + Mem3->BlockSize = sizeof(MemoryRangeHeader); + + /// Add a tiny free region so that the free list always has one entry. + FreeRangeHeader *Mem2 = + (FreeRangeHeader *)(((char*)Mem3)-FreeRangeHeader::getMinBlockSize()); + Mem2->ThisAllocated = 0; + Mem2->PrevAllocated = 1; + Mem2->BlockSize = FreeRangeHeader::getMinBlockSize(); + Mem2->SetEndOfBlockSizeMarker(); + Mem2->Prev = Mem2; // Mem2 *is* the free list for now. + Mem2->Next = Mem2; + + /// Add a tiny allocated region so that Mem2 is never coalesced away. + MemoryRangeHeader *Mem1 = (MemoryRangeHeader*)Mem2-1; + Mem1->ThisAllocated = 1; + Mem1->PrevAllocated = 0; + Mem1->BlockSize = sizeof(MemoryRangeHeader); + + // Add a FreeRangeHeader to the start of the function body region, indicating + // that the space is free. Mark the previous block allocated so we never look + // at it. + FreeRangeHeader *Mem0 = (FreeRangeHeader*)MemBase; + Mem0->ThisAllocated = 0; + Mem0->PrevAllocated = 1; + Mem0->BlockSize = (char*)Mem1-(char*)Mem0; + Mem0->SetEndOfBlockSizeMarker(); + Mem0->AddToFreeList(Mem2); + + // Start out with the freelist pointing to Mem0. + FreeMemoryList = Mem0; + + GOTBase = NULL; +} + +void DefaultJITMemoryManager::AllocateGOT() { + assert(GOTBase == 0 && "Cannot allocate the got multiple times"); + GOTBase = new uint8_t[sizeof(void*) * 8192]; + HasGOT = true; +} + +DefaultJITMemoryManager::~DefaultJITMemoryManager() { + for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) + sys::Memory::ReleaseRWX(CodeSlabs[i]); + + delete[] GOTBase; +} + +sys::MemoryBlock DefaultJITMemoryManager::allocateNewSlab(size_t size) { + // Allocate a new block close to the last one. + std::string ErrMsg; + sys::MemoryBlock *LastSlabPtr = LastSlab.base() ? &LastSlab : 0; + sys::MemoryBlock B = sys::Memory::AllocateRWX(size, LastSlabPtr, &ErrMsg); + if (B.base() == 0) { + report_fatal_error("Allocation failed when allocating new memory in the" + " JIT\n" + Twine(ErrMsg)); + } + LastSlab = B; + ++NumSlabs; + // Initialize the slab to garbage when debugging. + if (PoisonMemory) { + memset(B.base(), 0xCD, B.size()); + } + return B; +} + +/// CheckInvariants - For testing only. Return "" if all internal invariants +/// are preserved, and a helpful error message otherwise. For free and +/// allocated blocks, make sure that adding BlockSize gives a valid block. +/// For free blocks, make sure they're in the free list and that their end of +/// block size marker is correct. This function should return an error before +/// accessing bad memory. This function is defined here instead of in +/// JITMemoryManagerTest.cpp so that we don't have to expose all of the +/// implementation details of DefaultJITMemoryManager. +bool DefaultJITMemoryManager::CheckInvariants(std::string &ErrorStr) { + raw_string_ostream Err(ErrorStr); + + // Construct a the set of FreeRangeHeader pointers so we can query it + // efficiently. + llvm::SmallPtrSet<MemoryRangeHeader*, 16> FreeHdrSet; + FreeRangeHeader* FreeHead = FreeMemoryList; + FreeRangeHeader* FreeRange = FreeHead; + + do { + // Check that the free range pointer is in the blocks we've allocated. + bool Found = false; + for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(), + E = CodeSlabs.end(); I != E && !Found; ++I) { + char *Start = (char*)I->base(); + char *End = Start + I->size(); + Found = (Start <= (char*)FreeRange && (char*)FreeRange < End); + } + if (!Found) { + Err << "Corrupt free list; points to " << FreeRange; + return false; + } + + if (FreeRange->Next->Prev != FreeRange) { + Err << "Next and Prev pointers do not match."; + return false; + } + + // Otherwise, add it to the set. + FreeHdrSet.insert(FreeRange); + FreeRange = FreeRange->Next; + } while (FreeRange != FreeHead); + + // Go over each block, and look at each MemoryRangeHeader. + for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(), + E = CodeSlabs.end(); I != E; ++I) { + char *Start = (char*)I->base(); + char *End = Start + I->size(); + + // Check each memory range. + for (MemoryRangeHeader *Hdr = (MemoryRangeHeader*)Start, *LastHdr = NULL; + Start <= (char*)Hdr && (char*)Hdr < End; + Hdr = &Hdr->getBlockAfter()) { + if (Hdr->ThisAllocated == 0) { + // Check that this range is in the free list. + if (!FreeHdrSet.count(Hdr)) { + Err << "Found free header at " << Hdr << " that is not in free list."; + return false; + } + + // Now make sure the size marker at the end of the block is correct. + uintptr_t *Marker = ((uintptr_t*)&Hdr->getBlockAfter()) - 1; + if (!(Start <= (char*)Marker && (char*)Marker < End)) { + Err << "Block size in header points out of current MemoryBlock."; + return false; + } + if (Hdr->BlockSize != *Marker) { + Err << "End of block size marker (" << *Marker << ") " + << "and BlockSize (" << Hdr->BlockSize << ") don't match."; + return false; + } + } + + if (LastHdr && LastHdr->ThisAllocated != Hdr->PrevAllocated) { + Err << "Hdr->PrevAllocated (" << Hdr->PrevAllocated << ") != " + << "LastHdr->ThisAllocated (" << LastHdr->ThisAllocated << ")"; + return false; + } else if (!LastHdr && !Hdr->PrevAllocated) { + Err << "The first header should have PrevAllocated true."; + return false; + } + + // Remember the last header. + LastHdr = Hdr; + } + } + + // All invariants are preserved. + return true; +} + +JITMemoryManager *JITMemoryManager::CreateDefaultMemManager() { + return new DefaultJITMemoryManager(); +} + +// Allocate memory for code in 512K slabs. +const size_t DefaultJITMemoryManager::DefaultCodeSlabSize = 512 * 1024; + +// Allocate globals and stubs in slabs of 64K. (probably 16 pages) +const size_t DefaultJITMemoryManager::DefaultSlabSize = 64 * 1024; + +// Waste at most 16K at the end of each bump slab. (probably 4 pages) +const size_t DefaultJITMemoryManager::DefaultSizeThreshold = 16 * 1024; |
