diff options
| author | dim <dim@FreeBSD.org> | 2015-12-30 13:13:10 +0000 |
|---|---|---|
| committer | dim <dim@FreeBSD.org> | 2015-12-30 13:13:10 +0000 |
| commit | 9b5bf5c4f53d65d6a48722d7410ed7cb15f5ba3a (patch) | |
| tree | b466a4817f79516eb1df8eae92bccf62ecc84003 /contrib/llvm/lib/ExecutionEngine | |
| parent | f09a28d1de99fda4f5517fb12670fc36552f4927 (diff) | |
| parent | e194cd6d03d91631334d9d5e55b506036f423cc8 (diff) | |
| download | FreeBSD-src-9b5bf5c4f53d65d6a48722d7410ed7cb15f5ba3a.zip FreeBSD-src-9b5bf5c4f53d65d6a48722d7410ed7cb15f5ba3a.tar.gz | |
Update llvm to trunk r256633.
Diffstat (limited to 'contrib/llvm/lib/ExecutionEngine')
29 files changed, 1651 insertions, 712 deletions
diff --git a/contrib/llvm/lib/ExecutionEngine/ExecutionEngine.cpp b/contrib/llvm/lib/ExecutionEngine/ExecutionEngine.cpp index 67a1ca6..41c8da4 100644 --- a/contrib/llvm/lib/ExecutionEngine/ExecutionEngine.cpp +++ b/contrib/llvm/lib/ExecutionEngine/ExecutionEngine.cpp @@ -61,8 +61,7 @@ ExecutionEngine *(*ExecutionEngine::InterpCtor)(std::unique_ptr<Module> M, void JITEventListener::anchor() {} -ExecutionEngine::ExecutionEngine(std::unique_ptr<Module> M) - : LazyFunctionCreator(nullptr) { +void ExecutionEngine::Init(std::unique_ptr<Module> M) { CompilingLazily = false; GVCompilationDisabled = false; SymbolSearchingDisabled = false; @@ -79,6 +78,16 @@ ExecutionEngine::ExecutionEngine(std::unique_ptr<Module> M) Modules.push_back(std::move(M)); } +ExecutionEngine::ExecutionEngine(std::unique_ptr<Module> M) + : DL(M->getDataLayout()), LazyFunctionCreator(nullptr) { + Init(std::move(M)); +} + +ExecutionEngine::ExecutionEngine(DataLayout DL, std::unique_ptr<Module> M) + : DL(std::move(DL)), LazyFunctionCreator(nullptr) { + Init(std::move(M)); +} + ExecutionEngine::~ExecutionEngine() { clearAllGlobalMappings(); } @@ -86,7 +95,7 @@ ExecutionEngine::~ExecutionEngine() { namespace { /// \brief Helper class which uses a value handler to automatically deletes the /// memory block when the GlobalVariable is destroyed. -class GVMemoryBlock : public CallbackVH { +class GVMemoryBlock final : public CallbackVH { GVMemoryBlock(const GlobalVariable *GV) : CallbackVH(const_cast<GlobalVariable*>(GV)) {} @@ -115,7 +124,7 @@ public: } // anonymous namespace char *ExecutionEngine::getMemoryForGV(const GlobalVariable *GV) { - return GVMemoryBlock::Create(GV, *getDataLayout()); + return GVMemoryBlock::Create(GV, getDataLayout()); } void ExecutionEngine::addObjectFile(std::unique_ptr<object::ObjectFile> O) { @@ -187,7 +196,7 @@ std::string ExecutionEngine::getMangledName(const GlobalValue *GV) { const DataLayout &DL = GV->getParent()->getDataLayout().isDefault() - ? *getDataLayout() + ? getDataLayout() : GV->getParent()->getDataLayout(); Mangler::getNameWithPrefix(FullName, GV->getName(), DL); @@ -228,11 +237,10 @@ void ExecutionEngine::clearAllGlobalMappings() { void ExecutionEngine::clearGlobalMappingsFromModule(Module *M) { MutexGuard locked(lock); - for (Module::iterator FI = M->begin(), FE = M->end(); FI != FE; ++FI) - EEState.RemoveMapping(getMangledName(FI)); - for (Module::global_iterator GI = M->global_begin(), GE = M->global_end(); - GI != GE; ++GI) - EEState.RemoveMapping(getMangledName(GI)); + for (Function &FI : *M) + EEState.RemoveMapping(getMangledName(&FI)); + for (GlobalVariable &GI : M->globals()) + EEState.RemoveMapping(getMangledName(&GI)); } uint64_t ExecutionEngine::updateGlobalMapping(const GlobalValue *GV, @@ -333,7 +341,7 @@ void *ArgvArray::reset(LLVMContext &C, ExecutionEngine *EE, const std::vector<std::string> &InputArgv) { Values.clear(); // Free the old contents. Values.reserve(InputArgv.size()); - unsigned PtrSize = EE->getDataLayout()->getPointerSize(); + unsigned PtrSize = EE->getDataLayout().getPointerSize(); Array = make_unique<char[]>((InputArgv.size()+1)*PtrSize); DEBUG(dbgs() << "JIT: ARGV = " << (void*)Array.get() << "\n"); @@ -408,7 +416,7 @@ void ExecutionEngine::runStaticConstructorsDestructors(bool isDtors) { #ifndef NDEBUG /// isTargetNullPtr - Return whether the target pointer stored at Loc is null. static bool isTargetNullPtr(ExecutionEngine *EE, void *Loc) { - unsigned PtrSize = EE->getDataLayout()->getPointerSize(); + unsigned PtrSize = EE->getDataLayout().getPointerSize(); for (unsigned i = 0; i < PtrSize; ++i) if (*(i + (uint8_t*)Loc)) return false; @@ -621,8 +629,8 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) { break; case Type::VectorTyID: // if the whole vector is 'undef' just reserve memory for the value. - const VectorType* VTy = dyn_cast<VectorType>(C->getType()); - const Type *ElemTy = VTy->getElementType(); + auto* VTy = dyn_cast<VectorType>(C->getType()); + Type *ElemTy = VTy->getElementType(); unsigned int elemNum = VTy->getNumElements(); Result.AggregateVal.resize(elemNum); if (ElemTy->isIntegerTy()) @@ -641,8 +649,8 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) { case Instruction::GetElementPtr: { // Compute the index GenericValue Result = getConstantValue(Op0); - APInt Offset(DL->getPointerSizeInBits(), 0); - cast<GEPOperator>(CE)->accumulateConstantOffset(*DL, Offset); + APInt Offset(DL.getPointerSizeInBits(), 0); + cast<GEPOperator>(CE)->accumulateConstantOffset(DL, Offset); char* tmp = (char*) Result.PointerVal; Result = PTOGV(tmp + Offset.getSExtValue()); @@ -729,16 +737,16 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) { } case Instruction::PtrToInt: { GenericValue GV = getConstantValue(Op0); - uint32_t PtrWidth = DL->getTypeSizeInBits(Op0->getType()); + uint32_t PtrWidth = DL.getTypeSizeInBits(Op0->getType()); assert(PtrWidth <= 64 && "Bad pointer width"); GV.IntVal = APInt(PtrWidth, uintptr_t(GV.PointerVal)); - uint32_t IntWidth = DL->getTypeSizeInBits(CE->getType()); + uint32_t IntWidth = DL.getTypeSizeInBits(CE->getType()); GV.IntVal = GV.IntVal.zextOrTrunc(IntWidth); return GV; } case Instruction::IntToPtr: { GenericValue GV = getConstantValue(Op0); - uint32_t PtrWidth = DL->getTypeSizeInBits(CE->getType()); + uint32_t PtrWidth = DL.getTypeSizeInBits(CE->getType()); GV.IntVal = GV.IntVal.zextOrTrunc(PtrWidth); assert(GV.IntVal.getBitWidth() <= 64 && "Bad pointer width"); GV.PointerVal = PointerTy(uintptr_t(GV.IntVal.getZExtValue())); @@ -860,8 +868,7 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) { GV.IntVal = apfLHS.bitcastToAPInt(); break; case Instruction::FRem: - apfLHS.mod(APFloat(Sem, RHS.IntVal), - APFloat::rmNearestTiesToEven); + apfLHS.mod(APFloat(Sem, RHS.IntVal)); GV.IntVal = apfLHS.bitcastToAPInt(); break; } @@ -1040,7 +1047,7 @@ static void StoreIntToMemory(const APInt &IntVal, uint8_t *Dst, void ExecutionEngine::StoreValueToMemory(const GenericValue &Val, GenericValue *Ptr, Type *Ty) { - const unsigned StoreBytes = getDataLayout()->getTypeStoreSize(Ty); + const unsigned StoreBytes = getDataLayout().getTypeStoreSize(Ty); switch (Ty->getTypeID()) { default: @@ -1080,7 +1087,7 @@ void ExecutionEngine::StoreValueToMemory(const GenericValue &Val, break; } - if (sys::IsLittleEndianHost != getDataLayout()->isLittleEndian()) + if (sys::IsLittleEndianHost != getDataLayout().isLittleEndian()) // Host and target are different endian - reverse the stored bytes. std::reverse((uint8_t*)Ptr, StoreBytes + (uint8_t*)Ptr); } @@ -1117,7 +1124,7 @@ static void LoadIntFromMemory(APInt &IntVal, uint8_t *Src, unsigned LoadBytes) { void ExecutionEngine::LoadValueFromMemory(GenericValue &Result, GenericValue *Ptr, Type *Ty) { - const unsigned LoadBytes = getDataLayout()->getTypeStoreSize(Ty); + const unsigned LoadBytes = getDataLayout().getTypeStoreSize(Ty); switch (Ty->getTypeID()) { case Type::IntegerTyID: @@ -1143,8 +1150,8 @@ void ExecutionEngine::LoadValueFromMemory(GenericValue &Result, break; } case Type::VectorTyID: { - const VectorType *VT = cast<VectorType>(Ty); - const Type *ElemT = VT->getElementType(); + auto *VT = cast<VectorType>(Ty); + Type *ElemT = VT->getElementType(); const unsigned numElems = VT->getNumElements(); if (ElemT->isFloatTy()) { Result.AggregateVal.resize(numElems); @@ -1183,20 +1190,20 @@ void ExecutionEngine::InitializeMemory(const Constant *Init, void *Addr) { if (const ConstantVector *CP = dyn_cast<ConstantVector>(Init)) { unsigned ElementSize = - getDataLayout()->getTypeAllocSize(CP->getType()->getElementType()); + getDataLayout().getTypeAllocSize(CP->getType()->getElementType()); for (unsigned i = 0, e = CP->getNumOperands(); i != e; ++i) InitializeMemory(CP->getOperand(i), (char*)Addr+i*ElementSize); return; } if (isa<ConstantAggregateZero>(Init)) { - memset(Addr, 0, (size_t)getDataLayout()->getTypeAllocSize(Init->getType())); + memset(Addr, 0, (size_t)getDataLayout().getTypeAllocSize(Init->getType())); return; } if (const ConstantArray *CPA = dyn_cast<ConstantArray>(Init)) { unsigned ElementSize = - getDataLayout()->getTypeAllocSize(CPA->getType()->getElementType()); + getDataLayout().getTypeAllocSize(CPA->getType()->getElementType()); for (unsigned i = 0, e = CPA->getNumOperands(); i != e; ++i) InitializeMemory(CPA->getOperand(i), (char*)Addr+i*ElementSize); return; @@ -1204,7 +1211,7 @@ void ExecutionEngine::InitializeMemory(const Constant *Init, void *Addr) { if (const ConstantStruct *CPS = dyn_cast<ConstantStruct>(Init)) { const StructLayout *SL = - getDataLayout()->getStructLayout(cast<StructType>(CPS->getType())); + getDataLayout().getStructLayout(cast<StructType>(CPS->getType())); for (unsigned i = 0, e = CPS->getNumOperands(); i != e; ++i) InitializeMemory(CPS->getOperand(i), (char*)Addr+SL->getElementOffset(i)); return; @@ -1349,7 +1356,7 @@ void ExecutionEngine::EmitGlobalVariable(const GlobalVariable *GV) { InitializeMemory(GV->getInitializer(), GA); Type *ElTy = GV->getType()->getElementType(); - size_t GVSize = (size_t)getDataLayout()->getTypeAllocSize(ElTy); + size_t GVSize = (size_t)getDataLayout().getTypeAllocSize(ElTy); NumInitBytes += (unsigned)GVSize; ++NumGlobals; } diff --git a/contrib/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp b/contrib/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp index 55ab5af..ff7c4dc 100644 --- a/contrib/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp +++ b/contrib/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp @@ -29,7 +29,7 @@ using namespace llvm; DEFINE_SIMPLE_CONVERSION_FUNCTIONS(GenericValue, LLVMGenericValueRef) -inline LLVMTargetMachineRef wrap(const TargetMachine *P) { +static LLVMTargetMachineRef wrap(const TargetMachine *P) { return reinterpret_cast<LLVMTargetMachineRef>(const_cast<TargetMachine*>(P)); } @@ -210,35 +210,6 @@ LLVMBool LLVMCreateMCJITCompilerForModule( return 1; } -LLVMBool LLVMCreateExecutionEngine(LLVMExecutionEngineRef *OutEE, - LLVMModuleProviderRef MP, - char **OutError) { - /* The module provider is now actually a module. */ - return LLVMCreateExecutionEngineForModule(OutEE, - reinterpret_cast<LLVMModuleRef>(MP), - OutError); -} - -LLVMBool LLVMCreateInterpreter(LLVMExecutionEngineRef *OutInterp, - LLVMModuleProviderRef MP, - char **OutError) { - /* The module provider is now actually a module. */ - return LLVMCreateInterpreterForModule(OutInterp, - reinterpret_cast<LLVMModuleRef>(MP), - OutError); -} - -LLVMBool LLVMCreateJITCompiler(LLVMExecutionEngineRef *OutJIT, - LLVMModuleProviderRef MP, - unsigned OptLevel, - char **OutError) { - /* The module provider is now actually a module. */ - return LLVMCreateJITCompilerForModule(OutJIT, - reinterpret_cast<LLVMModuleRef>(MP), - OptLevel, OutError); -} - - void LLVMDisposeExecutionEngine(LLVMExecutionEngineRef EE) { delete unwrap(EE); } @@ -282,11 +253,6 @@ void LLVMAddModule(LLVMExecutionEngineRef EE, LLVMModuleRef M){ unwrap(EE)->addModule(std::unique_ptr<Module>(unwrap(M))); } -void LLVMAddModuleProvider(LLVMExecutionEngineRef EE, LLVMModuleProviderRef MP){ - /* The module provider is now actually a module. */ - LLVMAddModule(EE, reinterpret_cast<LLVMModuleRef>(MP)); -} - LLVMBool LLVMRemoveModule(LLVMExecutionEngineRef EE, LLVMModuleRef M, LLVMModuleRef *OutMod, char **OutError) { Module *Mod = unwrap(M); @@ -295,14 +261,6 @@ LLVMBool LLVMRemoveModule(LLVMExecutionEngineRef EE, LLVMModuleRef M, return 0; } -LLVMBool LLVMRemoveModuleProvider(LLVMExecutionEngineRef EE, - LLVMModuleProviderRef MP, - LLVMModuleRef *OutMod, char **OutError) { - /* The module provider is now actually a module. */ - return LLVMRemoveModule(EE, reinterpret_cast<LLVMModuleRef>(MP), OutMod, - OutError); -} - LLVMBool LLVMFindFunction(LLVMExecutionEngineRef EE, const char *Name, LLVMValueRef *OutFn) { if (Function *F = unwrap(EE)->FindFunctionNamed(Name)) { @@ -318,7 +276,7 @@ void *LLVMRecompileAndRelinkFunction(LLVMExecutionEngineRef EE, } LLVMTargetDataRef LLVMGetExecutionEngineTargetData(LLVMExecutionEngineRef EE) { - return wrap(unwrap(EE)->getDataLayout()); + return wrap(&unwrap(EE)->getDataLayout()); } LLVMTargetMachineRef diff --git a/contrib/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp b/contrib/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp index dbfa37e..1eb4f7d 100644 --- a/contrib/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp @@ -593,7 +593,7 @@ static GenericValue executeFCMP_UNO(GenericValue Src1, GenericValue Src2, } static GenericValue executeFCMP_BOOL(GenericValue Src1, GenericValue Src2, - const Type *Ty, const bool val) { + Type *Ty, const bool val) { GenericValue Dest; if(Ty->isVectorTy()) { assert(Src1.AggregateVal.size() == Src2.AggregateVal.size()); @@ -788,7 +788,7 @@ void Interpreter::visitBinaryOperator(BinaryOperator &I) { } static GenericValue executeSelectInst(GenericValue Src1, GenericValue Src2, - GenericValue Src3, const Type *Ty) { + GenericValue Src3, Type *Ty) { GenericValue Dest; if(Ty->isVectorTy()) { assert(Src1.AggregateVal.size() == Src2.AggregateVal.size()); @@ -805,7 +805,7 @@ static GenericValue executeSelectInst(GenericValue Src1, GenericValue Src2, void Interpreter::visitSelectInst(SelectInst &I) { ExecutionContext &SF = ECStack.back(); - const Type * Ty = I.getOperand(0)->getType(); + Type * Ty = I.getOperand(0)->getType(); GenericValue Src1 = getOperandValue(I.getOperand(0), SF); GenericValue Src2 = getOperandValue(I.getOperand(1), SF); GenericValue Src3 = getOperandValue(I.getOperand(2), SF); @@ -968,7 +968,7 @@ void Interpreter::visitAllocaInst(AllocaInst &I) { unsigned NumElements = getOperandValue(I.getOperand(0), SF).IntVal.getZExtValue(); - unsigned TypeSize = (size_t)TD.getTypeAllocSize(Ty); + unsigned TypeSize = (size_t)getDataLayout().getTypeAllocSize(Ty); // Avoid malloc-ing zero bytes, use max()... unsigned MemToAlloc = std::max(1U, NumElements * TypeSize); @@ -1000,7 +1000,7 @@ GenericValue Interpreter::executeGEPOperation(Value *Ptr, gep_type_iterator I, for (; I != E; ++I) { if (StructType *STy = dyn_cast<StructType>(*I)) { - const StructLayout *SLO = TD.getStructLayout(STy); + const StructLayout *SLO = getDataLayout().getStructLayout(STy); const ConstantInt *CPU = cast<ConstantInt>(I.getOperand()); unsigned Index = unsigned(CPU->getZExtValue()); @@ -1020,7 +1020,7 @@ GenericValue Interpreter::executeGEPOperation(Value *Ptr, gep_type_iterator I, assert(BitWidth == 64 && "Invalid index type for getelementptr"); Idx = (int64_t)IdxGV.IntVal.getZExtValue(); } - Total += TD.getTypeAllocSize(ST->getElementType())*Idx; + Total += getDataLayout().getTypeAllocSize(ST->getElementType()) * Idx; } } @@ -1139,7 +1139,7 @@ void Interpreter::visitShl(BinaryOperator &I) { GenericValue Src1 = getOperandValue(I.getOperand(0), SF); GenericValue Src2 = getOperandValue(I.getOperand(1), SF); GenericValue Dest; - const Type *Ty = I.getType(); + Type *Ty = I.getType(); if (Ty->isVectorTy()) { uint32_t src1Size = uint32_t(Src1.AggregateVal.size()); @@ -1166,7 +1166,7 @@ void Interpreter::visitLShr(BinaryOperator &I) { GenericValue Src1 = getOperandValue(I.getOperand(0), SF); GenericValue Src2 = getOperandValue(I.getOperand(1), SF); GenericValue Dest; - const Type *Ty = I.getType(); + Type *Ty = I.getType(); if (Ty->isVectorTy()) { uint32_t src1Size = uint32_t(Src1.AggregateVal.size()); @@ -1193,7 +1193,7 @@ void Interpreter::visitAShr(BinaryOperator &I) { GenericValue Src1 = getOperandValue(I.getOperand(0), SF); GenericValue Src2 = getOperandValue(I.getOperand(1), SF); GenericValue Dest; - const Type *Ty = I.getType(); + Type *Ty = I.getType(); if (Ty->isVectorTy()) { size_t src1Size = Src1.AggregateVal.size(); @@ -1237,10 +1237,10 @@ GenericValue Interpreter::executeTruncInst(Value *SrcVal, Type *DstTy, GenericValue Interpreter::executeSExtInst(Value *SrcVal, Type *DstTy, ExecutionContext &SF) { - const Type *SrcTy = SrcVal->getType(); + Type *SrcTy = SrcVal->getType(); GenericValue Dest, Src = getOperandValue(SrcVal, SF); if (SrcTy->isVectorTy()) { - const Type *DstVecTy = DstTy->getScalarType(); + Type *DstVecTy = DstTy->getScalarType(); unsigned DBitWidth = cast<IntegerType>(DstVecTy)->getBitWidth(); unsigned size = Src.AggregateVal.size(); // the sizes of src and dst vectors must be equal. @@ -1248,7 +1248,7 @@ GenericValue Interpreter::executeSExtInst(Value *SrcVal, Type *DstTy, for (unsigned i = 0; i < size; i++) Dest.AggregateVal[i].IntVal = Src.AggregateVal[i].IntVal.sext(DBitWidth); } else { - const IntegerType *DITy = cast<IntegerType>(DstTy); + auto *DITy = cast<IntegerType>(DstTy); unsigned DBitWidth = DITy->getBitWidth(); Dest.IntVal = Src.IntVal.sext(DBitWidth); } @@ -1257,10 +1257,10 @@ GenericValue Interpreter::executeSExtInst(Value *SrcVal, Type *DstTy, GenericValue Interpreter::executeZExtInst(Value *SrcVal, Type *DstTy, ExecutionContext &SF) { - const Type *SrcTy = SrcVal->getType(); + Type *SrcTy = SrcVal->getType(); GenericValue Dest, Src = getOperandValue(SrcVal, SF); if (SrcTy->isVectorTy()) { - const Type *DstVecTy = DstTy->getScalarType(); + Type *DstVecTy = DstTy->getScalarType(); unsigned DBitWidth = cast<IntegerType>(DstVecTy)->getBitWidth(); unsigned size = Src.AggregateVal.size(); @@ -1269,7 +1269,7 @@ GenericValue Interpreter::executeZExtInst(Value *SrcVal, Type *DstTy, for (unsigned i = 0; i < size; i++) Dest.AggregateVal[i].IntVal = Src.AggregateVal[i].IntVal.zext(DBitWidth); } else { - const IntegerType *DITy = cast<IntegerType>(DstTy); + auto *DITy = cast<IntegerType>(DstTy); unsigned DBitWidth = DITy->getBitWidth(); Dest.IntVal = Src.IntVal.zext(DBitWidth); } @@ -1327,8 +1327,8 @@ GenericValue Interpreter::executeFPToUIInst(Value *SrcVal, Type *DstTy, GenericValue Dest, Src = getOperandValue(SrcVal, SF); if (SrcTy->getTypeID() == Type::VectorTyID) { - const Type *DstVecTy = DstTy->getScalarType(); - const Type *SrcVecTy = SrcTy->getScalarType(); + Type *DstVecTy = DstTy->getScalarType(); + Type *SrcVecTy = SrcTy->getScalarType(); uint32_t DBitWidth = cast<IntegerType>(DstVecTy)->getBitWidth(); unsigned size = Src.AggregateVal.size(); // the sizes of src and dst vectors must be equal. @@ -1365,8 +1365,8 @@ GenericValue Interpreter::executeFPToSIInst(Value *SrcVal, Type *DstTy, GenericValue Dest, Src = getOperandValue(SrcVal, SF); if (SrcTy->getTypeID() == Type::VectorTyID) { - const Type *DstVecTy = DstTy->getScalarType(); - const Type *SrcVecTy = SrcTy->getScalarType(); + Type *DstVecTy = DstTy->getScalarType(); + Type *SrcVecTy = SrcTy->getScalarType(); uint32_t DBitWidth = cast<IntegerType>(DstVecTy)->getBitWidth(); unsigned size = Src.AggregateVal.size(); // the sizes of src and dst vectors must be equal @@ -1401,7 +1401,7 @@ GenericValue Interpreter::executeUIToFPInst(Value *SrcVal, Type *DstTy, GenericValue Dest, Src = getOperandValue(SrcVal, SF); if (SrcVal->getType()->getTypeID() == Type::VectorTyID) { - const Type *DstVecTy = DstTy->getScalarType(); + Type *DstVecTy = DstTy->getScalarType(); unsigned size = Src.AggregateVal.size(); // the sizes of src and dst vectors must be equal Dest.AggregateVal.resize(size); @@ -1433,7 +1433,7 @@ GenericValue Interpreter::executeSIToFPInst(Value *SrcVal, Type *DstTy, GenericValue Dest, Src = getOperandValue(SrcVal, SF); if (SrcVal->getType()->getTypeID() == Type::VectorTyID) { - const Type *DstVecTy = DstTy->getScalarType(); + Type *DstVecTy = DstTy->getScalarType(); unsigned size = Src.AggregateVal.size(); // the sizes of src and dst vectors must be equal Dest.AggregateVal.resize(size); @@ -1477,7 +1477,7 @@ GenericValue Interpreter::executeIntToPtrInst(Value *SrcVal, Type *DstTy, GenericValue Dest, Src = getOperandValue(SrcVal, SF); assert(DstTy->isPointerTy() && "Invalid PtrToInt instruction"); - uint32_t PtrSize = TD.getPointerSizeInBits(); + uint32_t PtrSize = getDataLayout().getPointerSizeInBits(); if (PtrSize != Src.IntVal.getBitWidth()) Src.IntVal = Src.IntVal.zextOrTrunc(PtrSize); @@ -1497,10 +1497,10 @@ GenericValue Interpreter::executeBitCastInst(Value *SrcVal, Type *DstTy, (DstTy->getTypeID() == Type::VectorTyID)) { // vector src bitcast to vector dst or vector src bitcast to scalar dst or // scalar src bitcast to vector dst - bool isLittleEndian = TD.isLittleEndian(); + bool isLittleEndian = getDataLayout().isLittleEndian(); GenericValue TempDst, TempSrc, SrcVec; - const Type *SrcElemTy; - const Type *DstElemTy; + Type *SrcElemTy; + Type *DstElemTy; unsigned SrcBitSize; unsigned DstBitSize; unsigned SrcNum; @@ -2091,7 +2091,7 @@ void Interpreter::callFunction(Function *F, ArrayRef<GenericValue> ArgVals) { } // Get pointers to first LLVM BB & Instruction in function. - StackFrame.CurBB = F->begin(); + StackFrame.CurBB = &F->front(); StackFrame.CurInst = StackFrame.CurBB->begin(); // Run through the function arguments and initialize their values... @@ -2103,7 +2103,7 @@ void Interpreter::callFunction(Function *F, ArrayRef<GenericValue> ArgVals) { unsigned i = 0; for (Function::arg_iterator AI = F->arg_begin(), E = F->arg_end(); AI != E; ++AI, ++i) - SetValue(AI, ArgVals[i], StackFrame); + SetValue(&*AI, ArgVals[i], StackFrame); // Handle varargs arguments... StackFrame.VarArgs.assign(ArgVals.begin()+i, ArgVals.end()); @@ -2121,27 +2121,5 @@ void Interpreter::run() { DEBUG(dbgs() << "About to interpret: " << I); visit(I); // Dispatch to one of the visit* methods... -#if 0 - // This is not safe, as visiting the instruction could lower it and free I. -DEBUG( - if (!isa<CallInst>(I) && !isa<InvokeInst>(I) && - I.getType() != Type::VoidTy) { - dbgs() << " --> "; - const GenericValue &Val = SF.Values[&I]; - switch (I.getType()->getTypeID()) { - default: llvm_unreachable("Invalid GenericValue Type"); - case Type::VoidTyID: dbgs() << "void"; break; - case Type::FloatTyID: dbgs() << "float " << Val.FloatVal; break; - case Type::DoubleTyID: dbgs() << "double " << Val.DoubleVal; break; - case Type::PointerTyID: dbgs() << "void* " << intptr_t(Val.PointerVal); - break; - case Type::IntegerTyID: - dbgs() << "i" << Val.IntVal.getBitWidth() << " " - << Val.IntVal.toStringUnsigned(10) - << " (0x" << Val.IntVal.toStringUnsigned(16) << ")\n"; - break; - } - }); -#endif } } diff --git a/contrib/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp b/contrib/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp index 9b44042..441f0eb 100644 --- a/contrib/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp @@ -178,7 +178,7 @@ static void *ffiValueFor(Type *Ty, const GenericValue &AV, } static bool ffiInvoke(RawFunc Fn, Function *F, ArrayRef<GenericValue> ArgVals, - const DataLayout *TD, GenericValue &Result) { + const DataLayout &TD, GenericValue &Result) { ffi_cif cif; FunctionType *FTy = F->getFunctionType(); const unsigned NumArgs = F->arg_size(); @@ -198,7 +198,7 @@ static bool ffiInvoke(RawFunc Fn, Function *F, ArrayRef<GenericValue> ArgVals, const unsigned ArgNo = A->getArgNo(); Type *ArgTy = FTy->getParamType(ArgNo); args[ArgNo] = ffiTypeFor(ArgTy); - ArgBytes += TD->getTypeStoreSize(ArgTy); + ArgBytes += TD.getTypeStoreSize(ArgTy); } SmallVector<uint8_t, 128> ArgData; @@ -210,7 +210,7 @@ static bool ffiInvoke(RawFunc Fn, Function *F, ArrayRef<GenericValue> ArgVals, const unsigned ArgNo = A->getArgNo(); Type *ArgTy = FTy->getParamType(ArgNo); values[ArgNo] = ffiValueFor(ArgTy, ArgVals[ArgNo], ArgDataPtr); - ArgDataPtr += TD->getTypeStoreSize(ArgTy); + ArgDataPtr += TD.getTypeStoreSize(ArgTy); } Type *RetTy = FTy->getReturnType(); @@ -219,7 +219,7 @@ static bool ffiInvoke(RawFunc Fn, Function *F, ArrayRef<GenericValue> ArgVals, if (ffi_prep_cif(&cif, FFI_DEFAULT_ABI, NumArgs, rtype, &args[0]) == FFI_OK) { SmallVector<uint8_t, 128> ret; if (RetTy->getTypeID() != Type::VoidTyID) - ret.resize(TD->getTypeStoreSize(RetTy)); + ret.resize(TD.getTypeStoreSize(RetTy)); ffi_call(&cif, Fn, ret.data(), values.data()); switch (RetTy->getTypeID()) { case Type::IntegerTyID: @@ -368,7 +368,7 @@ static GenericValue lle_X_sprintf(FunctionType *FT, case 'x': case 'X': if (HowLong >= 1) { if (HowLong == 1 && - TheInterpreter->getDataLayout()->getPointerSizeInBits() == 64 && + TheInterpreter->getDataLayout().getPointerSizeInBits() == 64 && sizeof(long) < sizeof(int64_t)) { // Make sure we use %lld with a 64 bit argument because we might be // compiling LLI on a 32 bit compiler. diff --git a/contrib/llvm/lib/ExecutionEngine/Interpreter/Interpreter.cpp b/contrib/llvm/lib/ExecutionEngine/Interpreter/Interpreter.cpp index f103c09..bc7da2e 100644 --- a/contrib/llvm/lib/ExecutionEngine/Interpreter/Interpreter.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Interpreter/Interpreter.cpp @@ -35,7 +35,7 @@ extern "C" void LLVMLinkInInterpreter() { } ExecutionEngine *Interpreter::create(std::unique_ptr<Module> M, std::string *ErrStr) { // Tell this Module to materialize everything and release the GVMaterializer. - if (std::error_code EC = M->materializeAllPermanently()) { + if (std::error_code EC = M->materializeAll()) { if (ErrStr) *ErrStr = EC.message(); // We got an error, just return 0 @@ -49,16 +49,15 @@ ExecutionEngine *Interpreter::create(std::unique_ptr<Module> M, // Interpreter ctor - Initialize stuff // Interpreter::Interpreter(std::unique_ptr<Module> M) - : ExecutionEngine(std::move(M)), TD(Modules.back().get()) { + : ExecutionEngine(std::move(M)) { memset(&ExitValue.Untyped, 0, sizeof(ExitValue.Untyped)); - setDataLayout(&TD); // Initialize the "backend" initializeExecutionEngine(); initializeExternalFunctions(); emitGlobals(); - IL = new IntrinsicLowering(TD); + IL = new IntrinsicLowering(getDataLayout()); } Interpreter::~Interpreter() { diff --git a/contrib/llvm/lib/ExecutionEngine/Interpreter/Interpreter.h b/contrib/llvm/lib/ExecutionEngine/Interpreter/Interpreter.h index f976641..2e5a867 100644 --- a/contrib/llvm/lib/ExecutionEngine/Interpreter/Interpreter.h +++ b/contrib/llvm/lib/ExecutionEngine/Interpreter/Interpreter.h @@ -26,7 +26,6 @@ namespace llvm { class IntrinsicLowering; -struct FunctionInfo; template<typename T> class generic_gep_type_iterator; class ConstantExpr; typedef generic_gep_type_iterator<User::const_op_iterator> gep_type_iterator; @@ -95,7 +94,6 @@ struct ExecutionContext { // class Interpreter : public ExecutionEngine, public InstVisitor<Interpreter> { GenericValue ExitValue; // The return value of the called function - DataLayout TD; IntrinsicLowering *IL; // The runtime stack of executing code. The top of the stack is the current diff --git a/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp b/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp index f6944ee..6cbebe9 100644 --- a/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp +++ b/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp @@ -65,12 +65,13 @@ MCJIT::createJIT(std::unique_ptr<Module> M, std::move(Resolver)); } -MCJIT::MCJIT(std::unique_ptr<Module> M, std::unique_ptr<TargetMachine> tm, +MCJIT::MCJIT(std::unique_ptr<Module> M, std::unique_ptr<TargetMachine> TM, std::shared_ptr<MCJITMemoryManager> MemMgr, std::shared_ptr<RuntimeDyld::SymbolResolver> Resolver) - : ExecutionEngine(std::move(M)), TM(std::move(tm)), Ctx(nullptr), - MemMgr(std::move(MemMgr)), Resolver(*this, std::move(Resolver)), - Dyld(*this->MemMgr, this->Resolver), ObjCache(nullptr) { + : ExecutionEngine(TM->createDataLayout(), std::move(M)), TM(std::move(TM)), + Ctx(nullptr), MemMgr(std::move(MemMgr)), + Resolver(*this, std::move(Resolver)), Dyld(*this->MemMgr, this->Resolver), + ObjCache(nullptr) { // FIXME: We are managing our modules, so we do not want the base class // ExecutionEngine to manage them as well. To avoid double destruction // of the first (and only) module added in ExecutionEngine constructor @@ -85,7 +86,6 @@ MCJIT::MCJIT(std::unique_ptr<Module> M, std::unique_ptr<TargetMachine> tm, Modules.clear(); OwnedModules.addModule(std::move(First)); - setDataLayout(TM->getDataLayout()); RegisterJITEventListener(JITEventListener::createGDBRegistrationListener()); } @@ -159,7 +159,6 @@ std::unique_ptr<MemoryBuffer> MCJIT::emitObject(Module *M) { // Initialize passes. PM.run(*M); // Flush the output buffer to get the generated code into memory - ObjStream.flush(); std::unique_ptr<MemoryBuffer> CompiledObjBuffer( new ObjectMemoryBuffer(std::move(ObjBufferSV))); @@ -193,7 +192,11 @@ void MCJIT::generateCodeForModule(Module *M) { if (ObjCache) ObjectToLoad = ObjCache->getObject(M); - M->setDataLayout(*TM->getDataLayout()); + if (M->getDataLayout().isDefault()) { + M->setDataLayout(getDataLayout()); + } else { + assert(M->getDataLayout() == getDataLayout() && "DataLayout Mismatch"); + } // If the cache did not contain a suitable object, compile the object if (!ObjectToLoad) { @@ -265,7 +268,7 @@ void MCJIT::finalizeModule(Module *M) { RuntimeDyld::SymbolInfo MCJIT::findExistingSymbol(const std::string &Name) { SmallString<128> FullName; - Mangler::getNameWithPrefix(FullName, Name, *TM->getDataLayout()); + Mangler::getNameWithPrefix(FullName, Name, getDataLayout()); if (void *Addr = getPointerToGlobalIfAvailable(FullName)) return RuntimeDyld::SymbolInfo(static_cast<uint64_t>( @@ -315,10 +318,12 @@ RuntimeDyld::SymbolInfo MCJIT::findSymbol(const std::string &Name, object::Archive *A = OB.getBinary(); // Look for our symbols in each Archive object::Archive::child_iterator ChildIt = A->findSym(Name); + if (std::error_code EC = ChildIt->getError()) + report_fatal_error(EC.message()); if (ChildIt != A->child_end()) { // FIXME: Support nested archives? ErrorOr<std::unique_ptr<object::Binary>> ChildBinOrErr = - ChildIt->getAsBinary(); + (*ChildIt)->getAsBinary(); if (ChildBinOrErr.getError()) continue; std::unique_ptr<object::Binary> &ChildBin = ChildBinOrErr.get(); diff --git a/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.h b/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.h index a45173c..3c9d2fd 100644 --- a/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.h +++ b/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.h @@ -86,7 +86,7 @@ class MCJIT : public ExecutionEngine { ModulePtrSet::iterator begin_added() { return AddedModules.begin(); } ModulePtrSet::iterator end_added() { return AddedModules.end(); } iterator_range<ModulePtrSet::iterator> added() { - return iterator_range<ModulePtrSet::iterator>(begin_added(), end_added()); + return make_range(begin_added(), end_added()); } ModulePtrSet::iterator begin_loaded() { return LoadedModules.begin(); } @@ -223,12 +223,13 @@ public: /// FindFunctionNamed - Search all of the active modules to find the function that /// defines FnName. This is very slow operation and shouldn't be used for /// general code. - virtual Function *FindFunctionNamed(const char *FnName) override; + Function *FindFunctionNamed(const char *FnName) override; - /// FindGlobalVariableNamed - Search all of the active modules to find the global variable - /// that defines Name. This is very slow operation and shouldn't be used for - /// general code. - virtual GlobalVariable *FindGlobalVariableNamed(const char *Name, bool AllowInternal = false) override; + /// FindGlobalVariableNamed - Search all of the active modules to find the + /// global variable that defines Name. This is very slow operation and + /// shouldn't be used for general code. + GlobalVariable *FindGlobalVariableNamed(const char *Name, + bool AllowInternal = false) override; /// Sets the object manager that MCJIT should use to avoid compilation. void setObjectCache(ObjectCache *manager) override; @@ -335,6 +336,6 @@ protected: bool CheckFunctionsOnly); }; -} // End llvm namespace +} // end llvm namespace -#endif +#endif // LLVM_LIB_EXECUTIONENGINE_MCJIT_MCJIT_H diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index b439810..34564e4 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -19,6 +19,9 @@ namespace llvm { namespace orc { +void JITCompileCallbackManager::anchor() {} +void IndirectStubsManager::anchor() {} + Constant* createIRTypedAddress(FunctionType &FT, TargetAddress Addr) { Constant *AddrIntVal = ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr); @@ -37,7 +40,7 @@ GlobalVariable* createImplPointer(PointerType &PT, Module &M, return IP; } -void makeStub(Function &F, GlobalVariable &ImplPointer) { +void makeStub(Function &F, Value &ImplPointer) { assert(F.isDeclaration() && "Can't turn a definition into a stub."); assert(F.getParent() && "Function isn't in a module."); Module &M = *F.getParent(); @@ -61,9 +64,7 @@ class GlobalRenamer { public: static bool needsRenaming(const Value &New) { - if (!New.hasName() || New.getName().startswith("\01L")) - return true; - return false; + return !New.hasName() || New.getName().startswith("\01L"); } const std::string& getRename(const Value &Orig) { @@ -106,6 +107,9 @@ void makeAllSymbolsExternallyAccessible(Module &M) { for (auto &GV : M.globals()) raiseVisibilityOnValue(GV, Renamer); + + for (auto &A : M.aliases()) + raiseVisibilityOnValue(A, Renamer); } Function* cloneFunctionDecl(Module &Dst, const Function &F, @@ -121,7 +125,7 @@ Function* cloneFunctionDecl(Module &Dst, const Function &F, auto NewArgI = NewF->arg_begin(); for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE; ++ArgI, ++NewArgI) - (*VMap)[ArgI] = NewArgI; + (*VMap)[&*ArgI] = &*NewArgI; } return NewF; @@ -177,5 +181,16 @@ void moveGlobalVariableInitializer(GlobalVariable &OrigGV, nullptr, Materializer)); } +GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, + ValueToValueMapTy &VMap) { + assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?"); + auto *NewA = GlobalAlias::create(OrigA.getValueType(), + OrigA.getType()->getPointerAddressSpace(), + OrigA.getLinkage(), OrigA.getName(), &Dst); + NewA->copyAttributesFrom(&OrigA); + VMap[&OrigA] = NewA; + return NewA; +} + } // End namespace orc. } // End namespace llvm. diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp new file mode 100644 index 0000000..d2379cd --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp @@ -0,0 +1,97 @@ +//===----------- OrcCBindings.cpp - C bindings for the Orc APIs -----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "OrcCBindingsStack.h" +#include "llvm-c/OrcBindings.h" + +using namespace llvm; + +LLVMOrcJITStackRef LLVMOrcCreateInstance(LLVMTargetMachineRef TM) { + TargetMachine *TM2(unwrap(TM)); + + Triple T(TM2->getTargetTriple()); + + auto CompileCallbackMgr = OrcCBindingsStack::createCompileCallbackMgr(T); + auto IndirectStubsMgrBuilder = + OrcCBindingsStack::createIndirectStubsMgrBuilder(T); + + OrcCBindingsStack *JITStack = + new OrcCBindingsStack(*TM2, std::move(CompileCallbackMgr), + IndirectStubsMgrBuilder); + + return wrap(JITStack); +} + +void LLVMOrcGetMangledSymbol(LLVMOrcJITStackRef JITStack, char **MangledName, + const char *SymbolName) { + OrcCBindingsStack &J = *unwrap(JITStack); + std::string Mangled = J.mangle(SymbolName); + *MangledName = new char[Mangled.size() + 1]; + strcpy(*MangledName, Mangled.c_str()); +} + +void LLVMOrcDisposeMangledSymbol(char *MangledName) { + delete[] MangledName; +} + +LLVMOrcTargetAddress +LLVMOrcCreateLazyCompileCallback(LLVMOrcJITStackRef JITStack, + LLVMOrcLazyCompileCallbackFn Callback, + void *CallbackCtx) { + OrcCBindingsStack &J = *unwrap(JITStack); + return J.createLazyCompileCallback(Callback, CallbackCtx); +} + +void LLVMOrcCreateIndirectStub(LLVMOrcJITStackRef JITStack, + const char *StubName, + LLVMOrcTargetAddress InitAddr) { + OrcCBindingsStack &J = *unwrap(JITStack); + J.createIndirectStub(StubName, InitAddr); +} + +void LLVMOrcSetIndirectStubPointer(LLVMOrcJITStackRef JITStack, + const char *StubName, + LLVMOrcTargetAddress NewAddr) { + OrcCBindingsStack &J = *unwrap(JITStack); + J.setIndirectStubPointer(StubName, NewAddr); +} + +LLVMOrcModuleHandle +LLVMOrcAddEagerlyCompiledIR(LLVMOrcJITStackRef JITStack, LLVMModuleRef Mod, + LLVMOrcSymbolResolverFn SymbolResolver, + void *SymbolResolverCtx) { + OrcCBindingsStack &J = *unwrap(JITStack); + Module *M(unwrap(Mod)); + return J.addIRModuleEager(M, SymbolResolver, SymbolResolverCtx); +} + +LLVMOrcModuleHandle +LLVMOrcAddLazilyCompiledIR(LLVMOrcJITStackRef JITStack, LLVMModuleRef Mod, + LLVMOrcSymbolResolverFn SymbolResolver, + void *SymbolResolverCtx) { + OrcCBindingsStack &J = *unwrap(JITStack); + Module *M(unwrap(Mod)); + return J.addIRModuleLazy(M, SymbolResolver, SymbolResolverCtx); +} + +void LLVMOrcRemoveModule(LLVMOrcJITStackRef JITStack, LLVMOrcModuleHandle H) { + OrcCBindingsStack &J = *unwrap(JITStack); + J.removeModule(H); +} + +LLVMOrcTargetAddress LLVMOrcGetSymbolAddress(LLVMOrcJITStackRef JITStack, + const char *SymbolName) { + OrcCBindingsStack &J = *unwrap(JITStack); + auto Sym = J.findSymbol(SymbolName, true); + return Sym.getAddress(); +} + +void LLVMOrcDisposeInstance(LLVMOrcJITStackRef JITStack) { + delete unwrap(JITStack); +} diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.cpp new file mode 100644 index 0000000..e519c7f --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.cpp @@ -0,0 +1,43 @@ +//===-------- OrcCBindingsStack.cpp - Orc JIT stack for C bindings --------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "OrcCBindingsStack.h" + +#include "llvm/ExecutionEngine/Orc/OrcTargetSupport.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/DynamicLibrary.h" +#include <cstdio> +#include <system_error> + +using namespace llvm; + +std::unique_ptr<OrcCBindingsStack::CompileCallbackMgr> +OrcCBindingsStack::createCompileCallbackMgr(Triple T) { + switch (T.getArch()) { + default: return nullptr; + + case Triple::x86_64: { + typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64> CCMgrT; + return llvm::make_unique<CCMgrT>(0); + } + } +} + +OrcCBindingsStack::IndirectStubsManagerBuilder +OrcCBindingsStack::createIndirectStubsMgrBuilder(Triple T) { + switch (T.getArch()) { + default: return nullptr; + + case Triple::x86_64: + return [](){ + return llvm::make_unique< + orc::LocalIndirectStubsManager<orc::OrcX86_64>>(); + }; + } +} diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h b/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h new file mode 100644 index 0000000..2e17624 --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h @@ -0,0 +1,282 @@ +//===--- OrcCBindingsStack.h - Orc JIT stack for C bindings ---*- C++ -*---===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H +#define LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H + +#include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" +#include "llvm/ExecutionEngine/Orc/CompileUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm-c/OrcBindings.h" + +namespace llvm { + +class OrcCBindingsStack; + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcCBindingsStack, LLVMOrcJITStackRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) + +class OrcCBindingsStack { +public: + + typedef orc::JITCompileCallbackManager CompileCallbackMgr; + typedef orc::ObjectLinkingLayer<> ObjLayerT; + typedef orc::IRCompileLayer<ObjLayerT> CompileLayerT; + typedef orc::CompileOnDemandLayer<CompileLayerT, CompileCallbackMgr> CODLayerT; + + typedef std::function<std::unique_ptr<CompileCallbackMgr>()> + CallbackManagerBuilder; + + typedef CODLayerT::IndirectStubsManagerBuilderT IndirectStubsManagerBuilder; + +private: + + class GenericHandle { + public: + virtual ~GenericHandle() {} + virtual orc::JITSymbol findSymbolIn(const std::string &Name, + bool ExportedSymbolsOnly) = 0; + virtual void removeModule() = 0; + }; + + template <typename LayerT> + class GenericHandleImpl : public GenericHandle { + public: + GenericHandleImpl(LayerT &Layer, typename LayerT::ModuleSetHandleT Handle) + : Layer(Layer), Handle(std::move(Handle)) {} + + orc::JITSymbol findSymbolIn(const std::string &Name, + bool ExportedSymbolsOnly) override { + return Layer.findSymbolIn(Handle, Name, ExportedSymbolsOnly); + } + + void removeModule() override { + return Layer.removeModuleSet(Handle); + } + + private: + LayerT &Layer; + typename LayerT::ModuleSetHandleT Handle; + }; + + template <typename LayerT> + std::unique_ptr<GenericHandleImpl<LayerT>> + createGenericHandle(LayerT &Layer, typename LayerT::ModuleSetHandleT Handle) { + return llvm::make_unique<GenericHandleImpl<LayerT>>(Layer, + std::move(Handle)); + } + +public: + + // We need a 'ModuleSetHandleT' to conform to the layer concept. + typedef unsigned ModuleSetHandleT; + + typedef unsigned ModuleHandleT; + + static std::unique_ptr<CompileCallbackMgr> createCompileCallbackMgr(Triple T); + static IndirectStubsManagerBuilder createIndirectStubsMgrBuilder(Triple T); + + OrcCBindingsStack(TargetMachine &TM, + std::unique_ptr<CompileCallbackMgr> CCMgr, + IndirectStubsManagerBuilder IndirectStubsMgrBuilder) + : DL(TM.createDataLayout()), CCMgr(std::move(CCMgr)), + ObjectLayer(), + CompileLayer(ObjectLayer, orc::SimpleCompiler(TM)), + CODLayer(CompileLayer, + [](Function &F) { std::set<Function*> S; S.insert(&F); return S; }, + *this->CCMgr, std::move(IndirectStubsMgrBuilder), false), + IndirectStubsMgr(IndirectStubsMgrBuilder()), + CXXRuntimeOverrides([this](const std::string &S) { return mangle(S); }) {} + + ~OrcCBindingsStack() { + // Run any destructors registered with __cxa_atexit. + CXXRuntimeOverrides.runDestructors(); + // Run any IR destructors. + for (auto &DtorRunner : IRStaticDestructorRunners) + DtorRunner.runViaLayer(*this); + } + + std::string mangle(StringRef Name) { + std::string MangledName; + { + raw_string_ostream MangledNameStream(MangledName); + Mangler::getNameWithPrefix(MangledNameStream, Name, DL); + } + return MangledName; + } + + template <typename PtrTy> + static PtrTy fromTargetAddress(orc::TargetAddress Addr) { + return reinterpret_cast<PtrTy>(static_cast<uintptr_t>(Addr)); + } + + orc::TargetAddress + createLazyCompileCallback(LLVMOrcLazyCompileCallbackFn Callback, + void *CallbackCtx) { + auto CCInfo = CCMgr->getCompileCallback(); + CCInfo.setCompileAction( + [=]() -> orc::TargetAddress { + return Callback(wrap(this), CallbackCtx); + }); + return CCInfo.getAddress(); + } + + void createIndirectStub(StringRef StubName, orc::TargetAddress Addr) { + IndirectStubsMgr->createStub(StubName, Addr, JITSymbolFlags::Exported); + } + + void setIndirectStubPointer(StringRef Name, orc::TargetAddress Addr) { + IndirectStubsMgr->updatePointer(Name, Addr); + } + + std::shared_ptr<RuntimeDyld::SymbolResolver> + createResolver(LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) { + auto Resolver = orc::createLambdaResolver( + [this, ExternalResolver, ExternalResolverCtx](const std::string &Name) { + // Search order: + // 1. JIT'd symbols. + // 2. Runtime overrides. + // 3. External resolver (if present). + + if (auto Sym = CODLayer.findSymbol(Name, true)) + return RuntimeDyld::SymbolInfo(Sym.getAddress(), + Sym.getFlags()); + if (auto Sym = CXXRuntimeOverrides.searchOverrides(Name)) + return Sym; + + if (ExternalResolver) + return RuntimeDyld::SymbolInfo(ExternalResolver(Name.c_str(), + ExternalResolverCtx), + llvm::JITSymbolFlags::Exported); + + return RuntimeDyld::SymbolInfo(nullptr); + }, + [](const std::string &Name) { + return RuntimeDyld::SymbolInfo(nullptr); + } + ); + + return std::shared_ptr<RuntimeDyld::SymbolResolver>(std::move(Resolver)); + } + + template <typename LayerT> + ModuleHandleT addIRModule(LayerT &Layer, + Module *M, + std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) { + + // Attach a data-layout if one isn't already present. + if (M->getDataLayout().isDefault()) + M->setDataLayout(DL); + + // Record the static constructors and destructors. We have to do this before + // we hand over ownership of the module to the JIT. + std::vector<std::string> CtorNames, DtorNames; + for (auto Ctor : orc::getConstructors(*M)) + CtorNames.push_back(mangle(Ctor.Func->getName())); + for (auto Dtor : orc::getDestructors(*M)) + DtorNames.push_back(mangle(Dtor.Func->getName())); + + // Create the resolver. + auto Resolver = createResolver(ExternalResolver, ExternalResolverCtx); + + // Add the module to the JIT. + std::vector<Module*> S; + S.push_back(std::move(M)); + + auto LH = Layer.addModuleSet(std::move(S), std::move(MemMgr), + std::move(Resolver)); + ModuleHandleT H = createHandle(Layer, LH); + + // Run the static constructors, and save the static destructor runner for + // execution when the JIT is torn down. + orc::CtorDtorRunner<OrcCBindingsStack> CtorRunner(std::move(CtorNames), H); + CtorRunner.runViaLayer(*this); + + IRStaticDestructorRunners.emplace_back(std::move(DtorNames), H); + + return H; + } + + ModuleHandleT addIRModuleEager(Module* M, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) { + return addIRModule(CompileLayer, std::move(M), + llvm::make_unique<SectionMemoryManager>(), + std::move(ExternalResolver), ExternalResolverCtx); + } + + ModuleHandleT addIRModuleLazy(Module* M, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) { + return addIRModule(CODLayer, std::move(M), nullptr, + std::move(ExternalResolver), ExternalResolverCtx); + } + + void removeModule(ModuleHandleT H) { + GenericHandles[H]->removeModule(); + GenericHandles[H] = nullptr; + FreeHandleIndexes.push_back(H); + } + + orc::JITSymbol findSymbol(const std::string &Name, bool ExportedSymbolsOnly) { + if (auto Sym = IndirectStubsMgr->findStub(Name, ExportedSymbolsOnly)) + return Sym; + return CODLayer.findSymbol(mangle(Name), ExportedSymbolsOnly); + } + + orc::JITSymbol findSymbolIn(ModuleHandleT H, const std::string &Name, + bool ExportedSymbolsOnly) { + return GenericHandles[H]->findSymbolIn(Name, ExportedSymbolsOnly); + } + +private: + + template <typename LayerT> + unsigned createHandle(LayerT &Layer, + typename LayerT::ModuleSetHandleT Handle) { + unsigned NewHandle; + if (!FreeHandleIndexes.empty()) { + NewHandle = FreeHandleIndexes.back(); + FreeHandleIndexes.pop_back(); + GenericHandles[NewHandle] = createGenericHandle(Layer, std::move(Handle)); + return NewHandle; + } else { + NewHandle = GenericHandles.size(); + GenericHandles.push_back(createGenericHandle(Layer, std::move(Handle))); + } + return NewHandle; + } + + DataLayout DL; + SectionMemoryManager CCMgrMemMgr; + + std::unique_ptr<CompileCallbackMgr> CCMgr; + ObjLayerT ObjectLayer; + CompileLayerT CompileLayer; + CODLayerT CODLayer; + + std::unique_ptr<orc::IndirectStubsManager> IndirectStubsMgr; + + std::vector<std::unique_ptr<GenericHandle>> GenericHandles; + std::vector<unsigned> FreeHandleIndexes; + + orc::LocalCXXRuntimeOverrides CXXRuntimeOverrides; + std::vector<orc::CtorDtorRunner<OrcCBindingsStack>> IRStaticDestructorRunners; +}; + +} // end namespace llvm + +#endif // LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h b/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h index 7dc5164..38a27cf 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h +++ b/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h @@ -137,25 +137,26 @@ public: } OrcMCJITReplacement( - std::shared_ptr<MCJITMemoryManager> MemMgr, - std::shared_ptr<RuntimeDyld::SymbolResolver> ClientResolver, - std::unique_ptr<TargetMachine> TM) - : TM(std::move(TM)), MemMgr(*this, std::move(MemMgr)), - Resolver(*this), ClientResolver(std::move(ClientResolver)), - NotifyObjectLoaded(*this), NotifyFinalized(*this), + std::shared_ptr<MCJITMemoryManager> MemMgr, + std::shared_ptr<RuntimeDyld::SymbolResolver> ClientResolver, + std::unique_ptr<TargetMachine> TM) + : ExecutionEngine(TM->createDataLayout()), TM(std::move(TM)), + MemMgr(*this, std::move(MemMgr)), Resolver(*this), + ClientResolver(std::move(ClientResolver)), NotifyObjectLoaded(*this), + NotifyFinalized(*this), ObjectLayer(NotifyObjectLoaded, NotifyFinalized), CompileLayer(ObjectLayer, SimpleCompiler(*this->TM)), - LazyEmitLayer(CompileLayer) { - setDataLayout(this->TM->getDataLayout()); - } + LazyEmitLayer(CompileLayer) {} void addModule(std::unique_ptr<Module> M) override { // If this module doesn't have a DataLayout attached then attach the // default. - if (M->getDataLayout().isDefault()) - M->setDataLayout(*getDataLayout()); - + if (M->getDataLayout().isDefault()) { + M->setDataLayout(getDataLayout()); + } else { + assert(M->getDataLayout() == getDataLayout() && "DataLayout Mismatch"); + } Modules.push_back(std::move(M)); std::vector<Module *> Ms; Ms.push_back(&*Modules.back()); @@ -174,12 +175,7 @@ public: std::tie(Obj, Buf) = O.takeBinary(); std::vector<std::unique_ptr<object::ObjectFile>> Objs; Objs.push_back(std::move(Obj)); - auto H = - ObjectLayer.addObjectSet(std::move(Objs), &MemMgr, &Resolver); - - std::vector<std::unique_ptr<MemoryBuffer>> Bufs; - Bufs.push_back(std::move(Buf)); - ObjectLayer.takeOwnershipOfBuffers(H, std::move(Bufs)); + ObjectLayer.addObjectSet(std::move(Objs), &MemMgr, &Resolver); } void addArchive(object::OwningBinary<object::Archive> A) override { @@ -234,6 +230,10 @@ public: CompileLayer.setObjectCache(NewCache); } + void setProcessAllSections(bool ProcessAllSections) override { + ObjectLayer.setProcessAllSections(ProcessAllSections); + } + private: RuntimeDyld::SymbolInfo findMangledSymbol(StringRef Name) { @@ -252,10 +252,12 @@ private: object::Archive *A = OB.getBinary(); // Look for our symbols in each Archive object::Archive::child_iterator ChildIt = A->findSym(Name); + if (std::error_code EC = ChildIt->getError()) + report_fatal_error(EC.message()); if (ChildIt != A->child_end()) { // FIXME: Support nested archives? ErrorOr<std::unique_ptr<object::Binary>> ChildBinOrErr = - ChildIt->getAsBinary(); + (*ChildIt)->getAsBinary(); if (ChildBinOrErr.getError()) continue; std::unique_ptr<object::Binary> &ChildBin = ChildBinOrErr.get(); @@ -289,7 +291,7 @@ private: "Incorrect number of Infos for Objects."); for (unsigned I = 0; I < Objects.size(); ++I) M.MemMgr.notifyObjectLoaded(&M, *Objects[I]); - }; + } private: OrcMCJITReplacement &M; @@ -310,7 +312,7 @@ private: std::string MangledName; { raw_string_ostream MangledNameStream(MangledName); - Mang.getNameWithPrefix(MangledNameStream, Name, *TM->getDataLayout()); + Mang.getNameWithPrefix(MangledNameStream, Name, getDataLayout()); } return MangledName; } diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp index 258868a..b931f10 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp @@ -1,137 +1,170 @@ +//===------- OrcTargetSupport.cpp - Target support utilities for Orc ------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + #include "llvm/ADT/Triple.h" #include "llvm/ExecutionEngine/Orc/OrcTargetSupport.h" +#include "llvm/Support/Process.h" #include <array> -using namespace llvm::orc; +namespace llvm { +namespace orc { -namespace { +void OrcX86_64::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn, + void *CallbackMgr) { + + const uint8_t ResolverCode[] = { + // resolver_entry: + 0x55, // 0x00: pushq %rbp + 0x48, 0x89, 0xe5, // 0x01: movq %rsp, %rbp + 0x50, // 0x04: pushq %rax + 0x53, // 0x05: pushq %rbx + 0x51, // 0x06: pushq %rcx + 0x52, // 0x07: pushq %rdx + 0x56, // 0x08: pushq %rsi + 0x57, // 0x09: pushq %rdi + 0x41, 0x50, // 0x0a: pushq %r8 + 0x41, 0x51, // 0x0c: pushq %r9 + 0x41, 0x52, // 0x0e: pushq %r10 + 0x41, 0x53, // 0x10: pushq %r11 + 0x41, 0x54, // 0x12: pushq %r12 + 0x41, 0x55, // 0x14: pushq %r13 + 0x41, 0x56, // 0x16: pushq %r14 + 0x41, 0x57, // 0x18: pushq %r15 + 0x48, 0x81, 0xec, 0x08, 0x02, 0x00, 0x00, // 0x1a: subq 20, %rsp + 0x48, 0x0f, 0xae, 0x04, 0x24, // 0x21: fxsave64 (%rsp) + 0x48, 0x8d, 0x3d, 0x43, 0x00, 0x00, 0x00, // 0x26: leaq 67(%rip), %rdi + 0x48, 0x8b, 0x3f, // 0x2d: movq (%rdi), %rdi + 0x48, 0x8b, 0x75, 0x08, // 0x30: movq 8(%rbp), %rsi + 0x48, 0x83, 0xee, 0x06, // 0x34: subq $6, %rsi + 0x48, 0xb8, // 0x38: movabsq $0, %rax + + // 0x3a: JIT re-entry fn addr: + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + 0xff, 0xd0, // 0x42: callq *%rax + 0x48, 0x89, 0x45, 0x08, // 0x44: movq %rax, 8(%rbp) + 0x48, 0x0f, 0xae, 0x0c, 0x24, // 0x48: fxrstor64 (%rsp) + 0x48, 0x81, 0xc4, 0x08, 0x02, 0x00, 0x00, // 0x4d: addq 20, %rsp + 0x41, 0x5f, // 0x54: popq %r15 + 0x41, 0x5e, // 0x56: popq %r14 + 0x41, 0x5d, // 0x58: popq %r13 + 0x41, 0x5c, // 0x5a: popq %r12 + 0x41, 0x5b, // 0x5c: popq %r11 + 0x41, 0x5a, // 0x5e: popq %r10 + 0x41, 0x59, // 0x60: popq %r9 + 0x41, 0x58, // 0x62: popq %r8 + 0x5f, // 0x64: popq %rdi + 0x5e, // 0x65: popq %rsi + 0x5a, // 0x66: popq %rdx + 0x59, // 0x67: popq %rcx + 0x5b, // 0x68: popq %rbx + 0x58, // 0x69: popq %rax + 0x5d, // 0x6a: popq %rbp + 0xc3, // 0x6b: retq + 0x00, 0x00, 0x00, 0x00, // 0x6c: <padding> + + // 0x70: Callback mgr address. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }; -uint64_t executeCompileCallback(JITCompileCallbackManagerBase *JCBM, - TargetAddress CallbackID) { - return JCBM->executeCompileCallback(CallbackID); + const unsigned ReentryFnAddrOffset = 0x3a; + const unsigned CallbackMgrAddrOffset = 0x70; + + memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); + memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryFn, sizeof(ReentryFn)); + memcpy(ResolverMem + CallbackMgrAddrOffset, &CallbackMgr, + sizeof(CallbackMgr)); } -} +void OrcX86_64::writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, + unsigned NumTrampolines) { -namespace llvm { -namespace orc { + unsigned OffsetToPtr = NumTrampolines * TrampolineSize; -const char* OrcX86_64::ResolverBlockName = "orc_resolver_block"; - -void OrcX86_64::insertResolverBlock( - Module &M, JITCompileCallbackManagerBase &JCBM) { - - // Trampoline code-sequence length, used to get trampoline address from return - // address. - const unsigned X86_64_TrampolineLength = 6; - - // List of x86-64 GPRs to save. Note - RBP saved separately below. - std::array<const char *, 14> GPRs = {{ - "rax", "rbx", "rcx", "rdx", - "rsi", "rdi", "r8", "r9", - "r10", "r11", "r12", "r13", - "r14", "r15" - }}; - - // Address of the executeCompileCallback function. - uint64_t CallbackAddr = - static_cast<uint64_t>( - reinterpret_cast<uintptr_t>(executeCompileCallback)); - - std::ostringstream AsmStream; - Triple TT(M.getTargetTriple()); - - // Switch to text section. - if (TT.getOS() == Triple::Darwin) - AsmStream << ".section __TEXT,__text,regular,pure_instructions\n" - << ".align 4, 0x90\n"; - else - AsmStream << ".text\n" - << ".align 16, 0x90\n"; - - // Bake in a pointer to the callback manager immediately before the - // start of the resolver function. - AsmStream << "jit_callback_manager_addr:\n" - << " .quad " << &JCBM << "\n"; - - // Start the resolver function. - AsmStream << ResolverBlockName << ":\n" - << " pushq %rbp\n" - << " movq %rsp, %rbp\n"; - - // Store the GPRs. - for (const auto &GPR : GPRs) - AsmStream << " pushq %" << GPR << "\n"; - - // Store floating-point state with FXSAVE. - // Note: We need to keep the stack 16-byte aligned, so if we've emitted an odd - // number of 64-bit pushes so far (GPRs.size() plus 1 for RBP) then add - // an extra 64 bits of padding to the FXSave area. - unsigned Padding = (GPRs.size() + 1) % 2 ? 8 : 0; - unsigned FXSaveSize = 512 + Padding; - AsmStream << " subq $" << FXSaveSize << ", %rsp\n" - << " fxsave64 (%rsp)\n" - - // Load callback manager address, compute trampoline address, call JIT. - << " lea jit_callback_manager_addr(%rip), %rdi\n" - << " movq (%rdi), %rdi\n" - << " movq 0x8(%rbp), %rsi\n" - << " subq $" << X86_64_TrampolineLength << ", %rsi\n" - << " movabsq $" << CallbackAddr << ", %rax\n" - << " callq *%rax\n" - - // Replace the return to the trampoline with the return address of the - // compiled function body. - << " movq %rax, 0x8(%rbp)\n" - - // Restore the floating point state. - << " fxrstor64 (%rsp)\n" - << " addq $" << FXSaveSize << ", %rsp\n"; - - for (const auto &GPR : make_range(GPRs.rbegin(), GPRs.rend())) - AsmStream << " popq %" << GPR << "\n"; - - // Restore original RBP and return to compiled function body. - AsmStream << " popq %rbp\n" - << " retq\n"; - - M.appendModuleInlineAsm(AsmStream.str()); -} + memcpy(TrampolineMem + OffsetToPtr, &ResolverAddr, sizeof(void*)); -OrcX86_64::LabelNameFtor -OrcX86_64::insertCompileCallbackTrampolines(Module &M, - TargetAddress ResolverBlockAddr, - unsigned NumCalls, - unsigned StartIndex) { - const char *ResolverBlockPtrName = "Lorc_resolve_block_addr"; - - std::ostringstream AsmStream; - Triple TT(M.getTargetTriple()); - - if (TT.getOS() == Triple::Darwin) - AsmStream << ".section __TEXT,__text,regular,pure_instructions\n" - << ".align 4, 0x90\n"; - else - AsmStream << ".text\n" - << ".align 16, 0x90\n"; - - AsmStream << ResolverBlockPtrName << ":\n" - << " .quad " << ResolverBlockAddr << "\n"; - - auto GetLabelName = - [=](unsigned I) { - std::ostringstream LabelStream; - LabelStream << "orc_jcc_" << (StartIndex + I); - return LabelStream.str(); - }; + uint64_t *Trampolines = reinterpret_cast<uint64_t*>(TrampolineMem); + uint64_t CallIndirPCRel = 0xf1c40000000015ff; - for (unsigned I = 0; I < NumCalls; ++I) - AsmStream << GetLabelName(I) << ":\n" - << " callq *" << ResolverBlockPtrName << "(%rip)\n"; - - M.appendModuleInlineAsm(AsmStream.str()); + for (unsigned I = 0; I < NumTrampolines; ++I, OffsetToPtr -= TrampolineSize) + Trampolines[I] = CallIndirPCRel | ((OffsetToPtr - 6) << 16); +} - return GetLabelName; +std::error_code OrcX86_64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, + unsigned MinStubs, + void *InitialPtrVal) { + // Stub format is: + // + // .section __orc_stubs + // stub1: + // jmpq *ptr1(%rip) + // .byte 0xC4 ; <- Invalid opcode padding. + // .byte 0xF1 + // stub2: + // jmpq *ptr2(%rip) + // + // ... + // + // .section __orc_ptrs + // ptr1: + // .quad 0x0 + // ptr2: + // .quad 0x0 + // + // ... + + const unsigned StubSize = IndirectStubsInfo::StubSize; + + // Emit at least MinStubs, rounded up to fill the pages allocated. + unsigned PageSize = sys::Process::getPageSize(); + unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; + unsigned NumStubs = (NumPages * PageSize) / StubSize; + + // Allocate memory for stubs and pointers in one call. + std::error_code EC; + auto StubsMem = + sys::OwningMemoryBlock( + sys::Memory::allocateMappedMemory(2 * NumPages * PageSize, nullptr, + sys::Memory::MF_READ | + sys::Memory::MF_WRITE, + EC)); + + if (EC) + return EC; + + // Create separate MemoryBlocks representing the stubs and pointers. + sys::MemoryBlock StubsBlock(StubsMem.base(), NumPages * PageSize); + sys::MemoryBlock PtrsBlock(static_cast<char*>(StubsMem.base()) + + NumPages * PageSize, + NumPages * PageSize); + + // Populate the stubs page stubs and mark it executable. + uint64_t *Stub = reinterpret_cast<uint64_t*>(StubsBlock.base()); + uint64_t PtrOffsetField = + static_cast<uint64_t>(NumPages * PageSize - 6) << 16; + for (unsigned I = 0; I < NumStubs; ++I) + Stub[I] = 0xF1C40000000025ff | PtrOffsetField; + + if (auto EC = sys::Memory::protectMappedMemory(StubsBlock, + sys::Memory::MF_READ | + sys::Memory::MF_EXEC)) + return EC; + + // Initialize all pointers to point at FailureAddress. + void **Ptr = reinterpret_cast<void**>(PtrsBlock.base()); + for (unsigned I = 0; I < NumStubs; ++I) + Ptr[I] = InitialPtrVal; + + StubsInfo.NumStubs = NumStubs; + StubsInfo.StubsMem = std::move(StubsMem); + + return std::error_code(); } } // End namespace orc. diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp index 93287a3..a95f3bb 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp @@ -41,20 +41,21 @@ void RuntimeDyldImpl::deregisterEHFrames() {} #ifndef NDEBUG static void dumpSectionMemory(const SectionEntry &S, StringRef State) { - dbgs() << "----- Contents of section " << S.Name << " " << State << " -----"; + dbgs() << "----- Contents of section " << S.getName() << " " << State + << " -----"; - if (S.Address == nullptr) { + if (S.getAddress() == nullptr) { dbgs() << "\n <section not emitted>\n"; return; } const unsigned ColsPerRow = 16; - uint8_t *DataAddr = S.Address; - uint64_t LoadAddr = S.LoadAddress; + uint8_t *DataAddr = S.getAddress(); + uint64_t LoadAddr = S.getLoadAddress(); unsigned StartPadding = LoadAddr & (ColsPerRow - 1); - unsigned BytesRemaining = S.Size; + unsigned BytesRemaining = S.getSize(); if (StartPadding) { dbgs() << "\n" << format("0x%016" PRIx64, @@ -82,30 +83,41 @@ static void dumpSectionMemory(const SectionEntry &S, StringRef State) { void RuntimeDyldImpl::resolveRelocations() { MutexGuard locked(lock); + // Print out the sections prior to relocation. + DEBUG( + for (int i = 0, e = Sections.size(); i != e; ++i) + dumpSectionMemory(Sections[i], "before relocations"); + ); + // First, resolve relocations associated with external symbols. resolveExternalSymbols(); - // Just iterate over the sections we have and resolve all the relocations - // in them. Gross overkill, but it gets the job done. - for (int i = 0, e = Sections.size(); i != e; ++i) { + // Iterate over all outstanding relocations + for (auto it = Relocations.begin(), e = Relocations.end(); it != e; ++it) { // The Section here (Sections[i]) refers to the section in which the // symbol for the relocation is located. The SectionID in the relocation // entry provides the section to which the relocation will be applied. - uint64_t Addr = Sections[i].LoadAddress; - DEBUG(dbgs() << "Resolving relocations Section #" << i << "\t" + int Idx = it->first; + uint64_t Addr = Sections[Idx].getLoadAddress(); + DEBUG(dbgs() << "Resolving relocations Section #" << Idx << "\t" << format("%p", (uintptr_t)Addr) << "\n"); - DEBUG(dumpSectionMemory(Sections[i], "before relocations")); - resolveRelocationList(Relocations[i], Addr); - DEBUG(dumpSectionMemory(Sections[i], "after relocations")); - Relocations.erase(i); + resolveRelocationList(it->second, Addr); } + Relocations.clear(); + + // Print out sections after relocation. + DEBUG( + for (int i = 0, e = Sections.size(); i != e; ++i) + dumpSectionMemory(Sections[i], "after relocations"); + ); + } void RuntimeDyldImpl::mapSectionAddress(const void *LocalAddress, uint64_t TargetAddress) { MutexGuard locked(lock); for (unsigned i = 0, e = Sections.size(); i != e; ++i) { - if (Sections[i].Address == LocalAddress) { + if (Sections[i].getAddress() == LocalAddress) { reassignSectionAddress(i, TargetAddress); return; } @@ -122,14 +134,10 @@ static std::error_code getOffset(const SymbolRef &Sym, SectionRef Sec, return std::error_code(); } -std::pair<unsigned, unsigned> +RuntimeDyldImpl::ObjSectionToIDMap RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) { MutexGuard locked(lock); - // Grab the first Section ID. We'll use this later to construct the underlying - // range for the returned LoadedObjectInfo. - unsigned SectionsAddedBeginIdx = Sections.size(); - // Save information about our target Arch = (Triple::ArchType)Obj.getArch(); IsTargetLittleEndian = Obj.isLittleEndian(); @@ -155,39 +163,56 @@ RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) { ++I) { uint32_t Flags = I->getFlags(); - bool IsCommon = Flags & SymbolRef::SF_Common; - if (IsCommon) + if (Flags & SymbolRef::SF_Common) CommonSymbols.push_back(*I); else { object::SymbolRef::Type SymType = I->getType(); - if (SymType == object::SymbolRef::ST_Function || - SymType == object::SymbolRef::ST_Data || - SymType == object::SymbolRef::ST_Unknown) { - - ErrorOr<StringRef> NameOrErr = I->getName(); - Check(NameOrErr.getError()); - StringRef Name = *NameOrErr; - section_iterator SI = Obj.section_end(); - Check(I->getSection(SI)); + // Get symbol name. + ErrorOr<StringRef> NameOrErr = I->getName(); + Check(NameOrErr.getError()); + StringRef Name = *NameOrErr; + + // Compute JIT symbol flags. + JITSymbolFlags RTDyldSymFlags = JITSymbolFlags::None; + if (Flags & SymbolRef::SF_Weak) + RTDyldSymFlags |= JITSymbolFlags::Weak; + if (Flags & SymbolRef::SF_Exported) + RTDyldSymFlags |= JITSymbolFlags::Exported; + + if (Flags & SymbolRef::SF_Absolute && + SymType != object::SymbolRef::ST_File) { + auto Addr = I->getAddress(); + Check(Addr.getError()); + uint64_t SectOffset = *Addr; + unsigned SectionID = AbsoluteSymbolSection; + + DEBUG(dbgs() << "\tType: " << SymType << " (absolute) Name: " << Name + << " SID: " << SectionID << " Offset: " + << format("%p", (uintptr_t)SectOffset) + << " flags: " << Flags << "\n"); + GlobalSymbolTable[Name] = + SymbolTableEntry(SectionID, SectOffset, RTDyldSymFlags); + } else if (SymType == object::SymbolRef::ST_Function || + SymType == object::SymbolRef::ST_Data || + SymType == object::SymbolRef::ST_Unknown || + SymType == object::SymbolRef::ST_Other) { + + ErrorOr<section_iterator> SIOrErr = I->getSection(); + Check(SIOrErr.getError()); + section_iterator SI = *SIOrErr; if (SI == Obj.section_end()) continue; + // Get symbol offset. uint64_t SectOffset; Check(getOffset(*I, *SI, SectOffset)); - StringRef SectionData; - Check(SI->getContents(SectionData)); bool IsCode = SI->isText(); - unsigned SectionID = - findOrEmitSection(Obj, *SI, IsCode, LocalSections); + unsigned SectionID = findOrEmitSection(Obj, *SI, IsCode, LocalSections); + DEBUG(dbgs() << "\tType: " << SymType << " Name: " << Name << " SID: " << SectionID << " Offset: " << format("%p", (uintptr_t)SectOffset) << " flags: " << Flags << "\n"); - JITSymbolFlags RTDyldSymFlags = JITSymbolFlags::None; - if (Flags & SymbolRef::SF_Weak) - RTDyldSymFlags |= JITSymbolFlags::Weak; - if (Flags & SymbolRef::SF_Exported) - RTDyldSymFlags |= JITSymbolFlags::Exported; GlobalSymbolTable[Name] = SymbolTableEntry(SectionID, SectOffset, RTDyldSymFlags); } @@ -231,9 +256,10 @@ RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) { // Give the subclasses a chance to tie-up any loose ends. finalizeLoad(Obj, LocalSections); - unsigned SectionsAddedEndIdx = Sections.size(); +// for (auto E : LocalSections) +// llvm::dbgs() << "Added: " << E.first.getRawDataRefImpl() << " -> " << E.second << "\n"; - return std::make_pair(SectionsAddedBeginIdx, SectionsAddedEndIdx); + return LocalSections; } // A helper method for computeTotalAllocSize. @@ -406,10 +432,9 @@ unsigned RuntimeDyldImpl::computeSectionStubBufSize(const ObjectFile &Obj, if (!(RelSecI == Section)) continue; - for (const RelocationRef &Reloc : SI->relocations()) { - (void)Reloc; - StubBufSize += StubSize; - } + for (const RelocationRef &Reloc : SI->relocations()) + if (relocationNeedsStub(Reloc)) + StubBufSize += StubSize; } // Get section data size and alignment @@ -492,7 +517,8 @@ void RuntimeDyldImpl::emitCommonSymbols(const ObjectFile &Obj, if (!Addr) report_fatal_error("Unable to allocate memory for common symbols!"); uint64_t Offset = 0; - Sections.push_back(SectionEntry("<common symbols>", Addr, CommonSize, 0)); + Sections.push_back( + SectionEntry("<common symbols>", Addr, CommonSize, CommonSize, 0)); memset(Addr, 0, CommonSize); DEBUG(dbgs() << "emitCommonSection SectionID: " << SectionID << " new addr: " @@ -524,6 +550,9 @@ void RuntimeDyldImpl::emitCommonSymbols(const ObjectFile &Obj, Offset += Size; Addr += Size; } + + if (Checker) + Checker->registerSection(Obj.getFileName(), SectionID); } unsigned RuntimeDyldImpl::emitSection(const ObjectFile &Obj, @@ -556,12 +585,20 @@ unsigned RuntimeDyldImpl::emitSection(const ObjectFile &Obj, uint8_t *Addr; const char *pData = nullptr; - // In either case, set the location of the unrelocated section in memory, - // since we still process relocations for it even if we're not applying them. - Check(Section.getContents(data)); - // Virtual sections have no data in the object image, so leave pData = 0 - if (!IsVirtual) + // If this section contains any bits (i.e. isn't a virtual or bss section), + // grab a reference to them. + if (!IsVirtual && !IsZeroInit) { + // In either case, set the location of the unrelocated section in memory, + // since we still process relocations for it even if we're not applying them. + Check(Section.getContents(data)); pData = data.data(); + } + + // Code section alignment needs to be at least as high as stub alignment or + // padding calculations may by incorrect when the section is remapped to a + // higher alignment. + if (IsCode) + Alignment = std::max(Alignment, getStubAlignment()); // Some sections, such as debug info, don't need to be loaded for execution. // Leave those where they are. @@ -606,7 +643,8 @@ unsigned RuntimeDyldImpl::emitSection(const ObjectFile &Obj, << " Allocate: " << Allocate << "\n"); } - Sections.push_back(SectionEntry(Name, Addr, DataSize, (uintptr_t)pData)); + Sections.push_back( + SectionEntry(Name, Addr, DataSize, Allocate, (uintptr_t)pData)); if (Checker) Checker->registerSection(Obj.getFileName(), SectionID); @@ -742,11 +780,11 @@ void RuntimeDyldImpl::reassignSectionAddress(unsigned SectionID, // Addr is a uint64_t because we can't assume the pointer width // of the target is the same as that of the host. Just use a generic // "big enough" type. - DEBUG(dbgs() << "Reassigning address for section " - << SectionID << " (" << Sections[SectionID].Name << "): " - << format("0x%016" PRIx64, Sections[SectionID].LoadAddress) << " -> " - << format("0x%016" PRIx64, Addr) << "\n"); - Sections[SectionID].LoadAddress = Addr; + DEBUG(dbgs() << "Reassigning address for section " << SectionID << " (" + << Sections[SectionID].getName() << "): " + << format("0x%016" PRIx64, Sections[SectionID].getLoadAddress()) + << " -> " << format("0x%016" PRIx64, Addr) << "\n"); + Sections[SectionID].setLoadAddress(Addr); } void RuntimeDyldImpl::resolveRelocationList(const RelocationList &Relocs, @@ -754,7 +792,7 @@ void RuntimeDyldImpl::resolveRelocationList(const RelocationList &Relocs, for (unsigned i = 0, e = Relocs.size(); i != e; ++i) { const RelocationEntry &RE = Relocs[i]; // Ignore relocations for sections that were not loaded - if (Sections[RE.SectionID].Address == nullptr) + if (Sections[RE.SectionID].getAddress() == nullptr) continue; resolveRelocation(RE, Value); } @@ -818,10 +856,11 @@ void RuntimeDyldImpl::resolveExternalSymbols() { // RuntimeDyld class implementation uint64_t RuntimeDyld::LoadedObjectInfo::getSectionLoadAddress( - StringRef SectionName) const { - for (unsigned I = BeginIdx; I != EndIdx; ++I) - if (RTDyld.Sections[I].Name == SectionName) - return RTDyld.Sections[I].LoadAddress; + const object::SectionRef &Sec) const { + + auto I = ObjSecToIDMap.find(Sec); + if (I != ObjSecToIDMap.end()) + return RTDyld.Sections[I->second].getLoadAddress(); return 0; } diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp index 1dacc13..e5fab92 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "RuntimeDyldCOFF.h" +#include "Targets/RuntimeDyldCOFFI386.h" #include "Targets/RuntimeDyldCOFFX86_64.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Triple.h" @@ -24,12 +25,11 @@ using namespace llvm::object; namespace { -class LoadedCOFFObjectInfo +class LoadedCOFFObjectInfo final : public RuntimeDyld::LoadedObjectInfoHelper<LoadedCOFFObjectInfo> { public: - LoadedCOFFObjectInfo(RuntimeDyldImpl &RTDyld, unsigned BeginIdx, - unsigned EndIdx) - : LoadedObjectInfoHelper(RTDyld, BeginIdx, EndIdx) {} + LoadedCOFFObjectInfo(RuntimeDyldImpl &RTDyld, ObjSectionToIDMap ObjSecToIDMap) + : LoadedObjectInfoHelper(RTDyld, std::move(ObjSecToIDMap)) {} OwningBinary<ObjectFile> getObjectForDebug(const ObjectFile &Obj) const override { @@ -48,6 +48,8 @@ llvm::RuntimeDyldCOFF::create(Triple::ArchType Arch, default: llvm_unreachable("Unsupported target for RuntimeDyldCOFF."); break; + case Triple::x86: + return make_unique<RuntimeDyldCOFFI386>(MemMgr, Resolver); case Triple::x86_64: return make_unique<RuntimeDyldCOFFX86_64>(MemMgr, Resolver); } @@ -55,10 +57,7 @@ llvm::RuntimeDyldCOFF::create(Triple::ArchType Arch, std::unique_ptr<RuntimeDyld::LoadedObjectInfo> RuntimeDyldCOFF::loadObject(const object::ObjectFile &O) { - unsigned SectionStartIdx, SectionEndIdx; - std::tie(SectionStartIdx, SectionEndIdx) = loadObjectImpl(O); - return llvm::make_unique<LoadedCOFFObjectInfo>(*this, SectionStartIdx, - SectionEndIdx); + return llvm::make_unique<LoadedCOFFObjectInfo>(*this, loadObjectImpl(O)); } uint64_t RuntimeDyldCOFF::getSymbolOffset(const SymbolRef &Sym) { diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp index ae199b7..58ce88a 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp @@ -727,7 +727,7 @@ bool RuntimeDyldCheckerImpl::checkAllRulesInBuffer(StringRef RulePrefix, } bool RuntimeDyldCheckerImpl::isSymbolValid(StringRef Symbol) const { - if (getRTDyld().getSymbolLocalAddress(Symbol)) + if (getRTDyld().getSymbol(Symbol)) return true; return !!getRTDyld().Resolver.findSymbol(Symbol); } @@ -799,11 +799,10 @@ std::pair<uint64_t, std::string> RuntimeDyldCheckerImpl::getSectionAddr( unsigned SectionID = SectionInfo->SectionID; uint64_t Addr; if (IsInsideLoad) - Addr = - static_cast<uint64_t>( - reinterpret_cast<uintptr_t>(getRTDyld().Sections[SectionID].Address)); + Addr = static_cast<uint64_t>(reinterpret_cast<uintptr_t>( + getRTDyld().Sections[SectionID].getAddress())); else - Addr = getRTDyld().Sections[SectionID].LoadAddress; + Addr = getRTDyld().Sections[SectionID].getLoadAddress(); return std::make_pair(Addr, std::string("")); } @@ -835,11 +834,11 @@ std::pair<uint64_t, std::string> RuntimeDyldCheckerImpl::getStubAddrFor( uint64_t Addr; if (IsInsideLoad) { - uintptr_t SectionBase = - reinterpret_cast<uintptr_t>(getRTDyld().Sections[SectionID].Address); + uintptr_t SectionBase = reinterpret_cast<uintptr_t>( + getRTDyld().Sections[SectionID].getAddress()); Addr = static_cast<uint64_t>(SectionBase) + StubOffset; } else { - uint64_t SectionBase = getRTDyld().Sections[SectionID].LoadAddress; + uint64_t SectionBase = getRTDyld().Sections[SectionID].getLoadAddress(); Addr = SectionBase + StubOffset; } @@ -855,16 +854,16 @@ RuntimeDyldCheckerImpl::getSubsectionStartingAt(StringRef Name) const { const auto &SymInfo = pos->second; uint8_t *SectionAddr = getRTDyld().getSectionAddress(SymInfo.getSectionID()); return StringRef(reinterpret_cast<const char *>(SectionAddr) + - SymInfo.getOffset(), - getRTDyld().Sections[SymInfo.getSectionID()].Size - - SymInfo.getOffset()); + SymInfo.getOffset(), + getRTDyld().Sections[SymInfo.getSectionID()].getSize() - + SymInfo.getOffset()); } void RuntimeDyldCheckerImpl::registerSection( StringRef FilePath, unsigned SectionID) { StringRef FileName = sys::path::filename(FilePath); const SectionEntry &Section = getRTDyld().Sections[SectionID]; - StringRef SectionName = Section.Name; + StringRef SectionName = Section.getName(); Stubs[FileName][SectionName].SectionID = SectionID; } @@ -874,7 +873,7 @@ void RuntimeDyldCheckerImpl::registerStubMap( const RuntimeDyldImpl::StubMap &RTDyldStubs) { StringRef FileName = sys::path::filename(FilePath); const SectionEntry &Section = getRTDyld().Sections[SectionID]; - StringRef SectionName = Section.Name; + StringRef SectionName = Section.getName(); Stubs[FileName][SectionName].SectionID = SectionID; diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp index 3787950..e09b71a 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp @@ -66,7 +66,6 @@ public: static inline bool classof(const ELFObjectFile<ELFT> *v) { return v->isDyldType(); } - }; @@ -104,12 +103,11 @@ void DyldELFObject<ELFT>::updateSymbolAddress(const SymbolRef &SymRef, sym->st_value = static_cast<addr_type>(Addr); } -class LoadedELFObjectInfo +class LoadedELFObjectInfo final : public RuntimeDyld::LoadedObjectInfoHelper<LoadedELFObjectInfo> { public: - LoadedELFObjectInfo(RuntimeDyldImpl &RTDyld, unsigned BeginIdx, - unsigned EndIdx) - : LoadedObjectInfoHelper(RTDyld, BeginIdx, EndIdx) {} + LoadedELFObjectInfo(RuntimeDyldImpl &RTDyld, ObjSectionToIDMap ObjSecToIDMap) + : LoadedObjectInfoHelper(RTDyld, std::move(ObjSecToIDMap)) {} OwningBinary<ObjectFile> getObjectForDebug(const ObjectFile &Obj) const override; @@ -118,6 +116,7 @@ public: template <typename ELFT> std::unique_ptr<DyldELFObject<ELFT>> createRTDyldELFObject(MemoryBufferRef Buffer, + const ObjectFile &SourceObject, const LoadedELFObjectInfo &L, std::error_code &ec) { typedef typename ELFFile<ELFT>::Elf_Shdr Elf_Shdr; @@ -127,6 +126,7 @@ createRTDyldELFObject(MemoryBufferRef Buffer, llvm::make_unique<DyldELFObject<ELFT>>(Buffer, ec); // Iterate over all sections in the object. + auto SI = SourceObject.section_begin(); for (const auto &Sec : Obj->sections()) { StringRef SectionName; Sec.getName(SectionName); @@ -135,12 +135,13 @@ createRTDyldELFObject(MemoryBufferRef Buffer, Elf_Shdr *shdr = const_cast<Elf_Shdr *>( reinterpret_cast<const Elf_Shdr *>(ShdrRef.p)); - if (uint64_t SecLoadAddr = L.getSectionLoadAddress(SectionName)) { + if (uint64_t SecLoadAddr = L.getSectionLoadAddress(*SI)) { // This assumes that the address passed in matches the target address // bitness. The template-based type cast handles everything else. shdr->sh_addr = static_cast<addr_type>(SecLoadAddr); } } + ++SI; } return Obj; @@ -158,16 +159,20 @@ OwningBinary<ObjectFile> createELFDebugObject(const ObjectFile &Obj, std::unique_ptr<ObjectFile> DebugObj; if (Obj.getBytesInAddress() == 4 && Obj.isLittleEndian()) { typedef ELFType<support::little, false> ELF32LE; - DebugObj = createRTDyldELFObject<ELF32LE>(Buffer->getMemBufferRef(), L, ec); + DebugObj = createRTDyldELFObject<ELF32LE>(Buffer->getMemBufferRef(), Obj, L, + ec); } else if (Obj.getBytesInAddress() == 4 && !Obj.isLittleEndian()) { typedef ELFType<support::big, false> ELF32BE; - DebugObj = createRTDyldELFObject<ELF32BE>(Buffer->getMemBufferRef(), L, ec); + DebugObj = createRTDyldELFObject<ELF32BE>(Buffer->getMemBufferRef(), Obj, L, + ec); } else if (Obj.getBytesInAddress() == 8 && !Obj.isLittleEndian()) { typedef ELFType<support::big, true> ELF64BE; - DebugObj = createRTDyldELFObject<ELF64BE>(Buffer->getMemBufferRef(), L, ec); + DebugObj = createRTDyldELFObject<ELF64BE>(Buffer->getMemBufferRef(), Obj, L, + ec); } else if (Obj.getBytesInAddress() == 8 && Obj.isLittleEndian()) { typedef ELFType<support::little, true> ELF64LE; - DebugObj = createRTDyldELFObject<ELF64LE>(Buffer->getMemBufferRef(), L, ec); + DebugObj = createRTDyldELFObject<ELF64LE>(Buffer->getMemBufferRef(), Obj, L, + ec); } else llvm_unreachable("Unexpected ELF format"); @@ -181,7 +186,7 @@ LoadedELFObjectInfo::getObjectForDebug(const ObjectFile &Obj) const { return createELFDebugObject(Obj, *this); } -} // namespace +} // anonymous namespace namespace llvm { @@ -193,9 +198,9 @@ RuntimeDyldELF::~RuntimeDyldELF() {} void RuntimeDyldELF::registerEHFrames() { for (int i = 0, e = UnregisteredEHFrameSections.size(); i != e; ++i) { SID EHFrameSID = UnregisteredEHFrameSections[i]; - uint8_t *EHFrameAddr = Sections[EHFrameSID].Address; - uint64_t EHFrameLoadAddr = Sections[EHFrameSID].LoadAddress; - size_t EHFrameSize = Sections[EHFrameSID].Size; + uint8_t *EHFrameAddr = Sections[EHFrameSID].getAddress(); + uint64_t EHFrameLoadAddr = Sections[EHFrameSID].getLoadAddress(); + size_t EHFrameSize = Sections[EHFrameSID].getSize(); MemMgr.registerEHFrames(EHFrameAddr, EHFrameLoadAddr, EHFrameSize); RegisteredEHFrameSections.push_back(EHFrameSID); } @@ -205,9 +210,9 @@ void RuntimeDyldELF::registerEHFrames() { void RuntimeDyldELF::deregisterEHFrames() { for (int i = 0, e = RegisteredEHFrameSections.size(); i != e; ++i) { SID EHFrameSID = RegisteredEHFrameSections[i]; - uint8_t *EHFrameAddr = Sections[EHFrameSID].Address; - uint64_t EHFrameLoadAddr = Sections[EHFrameSID].LoadAddress; - size_t EHFrameSize = Sections[EHFrameSID].Size; + uint8_t *EHFrameAddr = Sections[EHFrameSID].getAddress(); + uint64_t EHFrameLoadAddr = Sections[EHFrameSID].getLoadAddress(); + size_t EHFrameSize = Sections[EHFrameSID].getSize(); MemMgr.deregisterEHFrames(EHFrameAddr, EHFrameLoadAddr, EHFrameSize); } RegisteredEHFrameSections.clear(); @@ -215,10 +220,7 @@ void RuntimeDyldELF::deregisterEHFrames() { std::unique_ptr<RuntimeDyld::LoadedObjectInfo> RuntimeDyldELF::loadObject(const object::ObjectFile &O) { - unsigned SectionStartIdx, SectionEndIdx; - std::tie(SectionStartIdx, SectionEndIdx) = loadObjectImpl(O); - return llvm::make_unique<LoadedELFObjectInfo>(*this, SectionStartIdx, - SectionEndIdx); + return llvm::make_unique<LoadedELFObjectInfo>(*this, loadObjectImpl(O)); } void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section, @@ -230,9 +232,10 @@ void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section, llvm_unreachable("Relocation type not implemented yet!"); break; case ELF::R_X86_64_64: { - support::ulittle64_t::ref(Section.Address + Offset) = Value + Addend; + support::ulittle64_t::ref(Section.getAddressWithOffset(Offset)) = + Value + Addend; DEBUG(dbgs() << "Writing " << format("%p", (Value + Addend)) << " at " - << format("%p\n", Section.Address + Offset)); + << format("%p\n", Section.getAddressWithOffset(Offset))); break; } case ELF::R_X86_64_32: @@ -242,23 +245,34 @@ void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section, (Type == ELF::R_X86_64_32S && ((int64_t)Value <= INT32_MAX && (int64_t)Value >= INT32_MIN))); uint32_t TruncatedAddr = (Value & 0xFFFFFFFF); - support::ulittle32_t::ref(Section.Address + Offset) = TruncatedAddr; + support::ulittle32_t::ref(Section.getAddressWithOffset(Offset)) = + TruncatedAddr; DEBUG(dbgs() << "Writing " << format("%p", TruncatedAddr) << " at " - << format("%p\n", Section.Address + Offset)); + << format("%p\n", Section.getAddressWithOffset(Offset))); + break; + } + case ELF::R_X86_64_PC8: { + uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset); + int64_t RealOffset = Value + Addend - FinalAddress; + assert(isInt<8>(RealOffset)); + int8_t TruncOffset = (RealOffset & 0xFF); + Section.getAddress()[Offset] = TruncOffset; break; } case ELF::R_X86_64_PC32: { - uint64_t FinalAddress = Section.LoadAddress + Offset; + uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset); int64_t RealOffset = Value + Addend - FinalAddress; assert(isInt<32>(RealOffset)); int32_t TruncOffset = (RealOffset & 0xFFFFFFFF); - support::ulittle32_t::ref(Section.Address + Offset) = TruncOffset; + support::ulittle32_t::ref(Section.getAddressWithOffset(Offset)) = + TruncOffset; break; } case ELF::R_X86_64_PC64: { - uint64_t FinalAddress = Section.LoadAddress + Offset; + uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset); int64_t RealOffset = Value + Addend - FinalAddress; - support::ulittle64_t::ref(Section.Address + Offset) = RealOffset; + support::ulittle64_t::ref(Section.getAddressWithOffset(Offset)) = + RealOffset; break; } } @@ -269,13 +283,16 @@ void RuntimeDyldELF::resolveX86Relocation(const SectionEntry &Section, uint32_t Type, int32_t Addend) { switch (Type) { case ELF::R_386_32: { - support::ulittle32_t::ref(Section.Address + Offset) = Value + Addend; + support::ulittle32_t::ref(Section.getAddressWithOffset(Offset)) = + Value + Addend; break; } case ELF::R_386_PC32: { - uint32_t FinalAddress = ((Section.LoadAddress + Offset) & 0xFFFFFFFF); + uint32_t FinalAddress = + Section.getLoadAddressWithOffset(Offset) & 0xFFFFFFFF; uint32_t RealOffset = Value + Addend - FinalAddress; - support::ulittle32_t::ref(Section.Address + Offset) = RealOffset; + support::ulittle32_t::ref(Section.getAddressWithOffset(Offset)) = + RealOffset; break; } default: @@ -289,11 +306,12 @@ void RuntimeDyldELF::resolveX86Relocation(const SectionEntry &Section, void RuntimeDyldELF::resolveAArch64Relocation(const SectionEntry &Section, uint64_t Offset, uint64_t Value, uint32_t Type, int64_t Addend) { - uint32_t *TargetPtr = reinterpret_cast<uint32_t *>(Section.Address + Offset); - uint64_t FinalAddress = Section.LoadAddress + Offset; + uint32_t *TargetPtr = + reinterpret_cast<uint32_t *>(Section.getAddressWithOffset(Offset)); + uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset); DEBUG(dbgs() << "resolveAArch64Relocation, LocalAddress: 0x" - << format("%llx", Section.Address + Offset) + << format("%llx", Section.getAddressWithOffset(Offset)) << " FinalAddress: 0x" << format("%llx", FinalAddress) << " Value: 0x" << format("%llx", Value) << " Type: 0x" << format("%x", Type) << " Addend: 0x" << format("%llx", Addend) @@ -305,7 +323,7 @@ void RuntimeDyldELF::resolveAArch64Relocation(const SectionEntry &Section, break; case ELF::R_AARCH64_ABS64: { uint64_t *TargetPtr = - reinterpret_cast<uint64_t *>(Section.Address + Offset); + reinterpret_cast<uint64_t *>(Section.getAddressWithOffset(Offset)); *TargetPtr = Value + Addend; break; } @@ -428,12 +446,13 @@ void RuntimeDyldELF::resolveARMRelocation(const SectionEntry &Section, uint64_t Offset, uint32_t Value, uint32_t Type, int32_t Addend) { // TODO: Add Thumb relocations. - uint32_t *TargetPtr = (uint32_t *)(Section.Address + Offset); - uint32_t FinalAddress = ((Section.LoadAddress + Offset) & 0xFFFFFFFF); + uint32_t *TargetPtr = + reinterpret_cast<uint32_t *>(Section.getAddressWithOffset(Offset)); + uint32_t FinalAddress = Section.getLoadAddressWithOffset(Offset) & 0xFFFFFFFF; Value += Addend; DEBUG(dbgs() << "resolveARMRelocation, LocalAddress: " - << Section.Address + Offset + << Section.getAddressWithOffset(Offset) << " FinalAddress: " << format("%p", FinalAddress) << " Value: " << format("%x", Value) << " Type: " << format("%x", Type) << " Addend: " << format("%x", Addend) << "\n"); @@ -477,13 +496,14 @@ void RuntimeDyldELF::resolveARMRelocation(const SectionEntry &Section, void RuntimeDyldELF::resolveMIPSRelocation(const SectionEntry &Section, uint64_t Offset, uint32_t Value, uint32_t Type, int32_t Addend) { - uint8_t *TargetPtr = Section.Address + Offset; + uint8_t *TargetPtr = Section.getAddressWithOffset(Offset); Value += Addend; DEBUG(dbgs() << "resolveMIPSRelocation, LocalAddress: " - << Section.Address + Offset << " FinalAddress: " - << format("%p", Section.LoadAddress + Offset) << " Value: " - << format("%x", Value) << " Type: " << format("%x", Type) + << Section.getAddressWithOffset(Offset) << " FinalAddress: " + << format("%p", Section.getLoadAddressWithOffset(Offset)) + << " Value: " << format("%x", Value) + << " Type: " << format("%x", Type) << " Addend: " << format("%x", Addend) << "\n"); uint32_t Insn = readBytesUnaligned(TargetPtr, 4); @@ -512,47 +532,47 @@ void RuntimeDyldELF::resolveMIPSRelocation(const SectionEntry &Section, writeBytesUnaligned(Insn, TargetPtr, 4); break; case ELF::R_MIPS_PC32: { - uint32_t FinalAddress = (Section.LoadAddress + Offset); + uint32_t FinalAddress = Section.getLoadAddressWithOffset(Offset); writeBytesUnaligned(Value - FinalAddress, (uint8_t *)TargetPtr, 4); break; } case ELF::R_MIPS_PC16: { - uint32_t FinalAddress = (Section.LoadAddress + Offset); + uint32_t FinalAddress = Section.getLoadAddressWithOffset(Offset); Insn &= 0xffff0000; Insn |= ((Value - FinalAddress) >> 2) & 0xffff; writeBytesUnaligned(Insn, TargetPtr, 4); break; } case ELF::R_MIPS_PC19_S2: { - uint32_t FinalAddress = (Section.LoadAddress + Offset); + uint32_t FinalAddress = Section.getLoadAddressWithOffset(Offset); Insn &= 0xfff80000; Insn |= ((Value - (FinalAddress & ~0x3)) >> 2) & 0x7ffff; writeBytesUnaligned(Insn, TargetPtr, 4); break; } case ELF::R_MIPS_PC21_S2: { - uint32_t FinalAddress = (Section.LoadAddress + Offset); + uint32_t FinalAddress = Section.getLoadAddressWithOffset(Offset); Insn &= 0xffe00000; Insn |= ((Value - FinalAddress) >> 2) & 0x1fffff; writeBytesUnaligned(Insn, TargetPtr, 4); break; } case ELF::R_MIPS_PC26_S2: { - uint32_t FinalAddress = (Section.LoadAddress + Offset); + uint32_t FinalAddress = Section.getLoadAddressWithOffset(Offset); Insn &= 0xfc000000; Insn |= ((Value - FinalAddress) >> 2) & 0x3ffffff; writeBytesUnaligned(Insn, TargetPtr, 4); break; } case ELF::R_MIPS_PCHI16: { - uint32_t FinalAddress = (Section.LoadAddress + Offset); + uint32_t FinalAddress = Section.getLoadAddressWithOffset(Offset); Insn &= 0xffff0000; Insn |= ((Value - FinalAddress + 0x8000) >> 16) & 0xffff; writeBytesUnaligned(Insn, TargetPtr, 4); break; } case ELF::R_MIPS_PCLO16: { - uint32_t FinalAddress = (Section.LoadAddress + Offset); + uint32_t FinalAddress = Section.getLoadAddressWithOffset(Offset); Insn &= 0xffff0000; Insn |= (Value - FinalAddress) & 0xffff; writeBytesUnaligned(Insn, TargetPtr, 4); @@ -603,7 +623,8 @@ void RuntimeDyldELF::resolveMIPS64Relocation(const SectionEntry &Section, CalculatedValue, SymOffset, SectionID); } - applyMIPS64Relocation(Section.Address + Offset, CalculatedValue, RelType); + applyMIPS64Relocation(Section.getAddressWithOffset(Offset), CalculatedValue, + RelType); } int64_t @@ -613,13 +634,12 @@ RuntimeDyldELF::evaluateMIPS64Relocation(const SectionEntry &Section, uint64_t SymOffset, SID SectionID) { DEBUG(dbgs() << "evaluateMIPS64Relocation, LocalAddress: 0x" - << format("%llx", Section.Address + Offset) + << format("%llx", Section.getAddressWithOffset(Offset)) << " FinalAddress: 0x" - << format("%llx", Section.LoadAddress + Offset) + << format("%llx", Section.getLoadAddressWithOffset(Offset)) << " Value: 0x" << format("%llx", Value) << " Type: 0x" << format("%x", Type) << " Addend: 0x" << format("%llx", Addend) - << " SymOffset: " << format("%x", SymOffset) - << "\n"); + << " SymOffset: " << format("%x", SymOffset) << "\n"); switch (Type) { default: @@ -672,35 +692,35 @@ RuntimeDyldELF::evaluateMIPS64Relocation(const SectionEntry &Section, return Value + Addend - (GOTAddr + 0x7ff0); } case ELF::R_MIPS_PC16: { - uint64_t FinalAddress = (Section.LoadAddress + Offset); + uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset); return ((Value + Addend - FinalAddress) >> 2) & 0xffff; } case ELF::R_MIPS_PC32: { - uint64_t FinalAddress = (Section.LoadAddress + Offset); + uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset); return Value + Addend - FinalAddress; } case ELF::R_MIPS_PC18_S3: { - uint64_t FinalAddress = (Section.LoadAddress + Offset); - return ((Value + Addend - ((FinalAddress | 7) ^ 7)) >> 3) & 0x3ffff; + uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset); + return ((Value + Addend - (FinalAddress & ~0x7)) >> 3) & 0x3ffff; } case ELF::R_MIPS_PC19_S2: { - uint64_t FinalAddress = (Section.LoadAddress + Offset); - return ((Value + Addend - FinalAddress) >> 2) & 0x7ffff; + uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset); + return ((Value + Addend - (FinalAddress & ~0x3)) >> 2) & 0x7ffff; } case ELF::R_MIPS_PC21_S2: { - uint64_t FinalAddress = (Section.LoadAddress + Offset); + uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset); return ((Value + Addend - FinalAddress) >> 2) & 0x1fffff; } case ELF::R_MIPS_PC26_S2: { - uint64_t FinalAddress = (Section.LoadAddress + Offset); + uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset); return ((Value + Addend - FinalAddress) >> 2) & 0x3ffffff; } case ELF::R_MIPS_PCHI16: { - uint64_t FinalAddress = (Section.LoadAddress + Offset); + uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset); return ((Value + Addend - FinalAddress + 0x8000) >> 16) & 0xffff; } case ELF::R_MIPS_PCLO16: { - uint64_t FinalAddress = (Section.LoadAddress + Offset); + uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset); return (Value + Addend - FinalAddress) & 0xffff; } } @@ -769,7 +789,7 @@ void RuntimeDyldELF::findPPC64TOCSection(const ELFObjectFileBase &Obj, // relocation) without a .toc directive. In this case just use the // first section (which is usually the .odp) since the code won't // reference the .toc base directly. - Rel.SymbolName = NULL; + Rel.SymbolName = nullptr; Rel.SectionID = 0; // The TOC consists of sections .got, .toc, .tocbss, .plt in that @@ -842,8 +862,9 @@ void RuntimeDyldELF::findOPDEntrySection(const ELFObjectFileBase &Obj, if (Rel.Addend != (int64_t)TargetSymbolOffset) continue; - section_iterator tsi(Obj.section_end()); - check(TargetSymbol->getSection(tsi)); + ErrorOr<section_iterator> TSIOrErr = TargetSymbol->getSection(); + check(TSIOrErr.getError()); + section_iterator tsi = *TSIOrErr; bool IsCode = tsi->isText(); Rel.SectionID = findOrEmitSection(Obj, (*tsi), IsCode, LocalSections); Rel.Addend = (intptr_t)Addend; @@ -884,10 +905,30 @@ static inline uint16_t applyPPChighesta (uint64_t value) { return ((value + 0x8000) >> 48) & 0xffff; } +void RuntimeDyldELF::resolvePPC32Relocation(const SectionEntry &Section, + uint64_t Offset, uint64_t Value, + uint32_t Type, int64_t Addend) { + uint8_t *LocalAddress = Section.getAddressWithOffset(Offset); + switch (Type) { + default: + llvm_unreachable("Relocation type not implemented yet!"); + break; + case ELF::R_PPC_ADDR16_LO: + writeInt16BE(LocalAddress, applyPPClo(Value + Addend)); + break; + case ELF::R_PPC_ADDR16_HI: + writeInt16BE(LocalAddress, applyPPChi(Value + Addend)); + break; + case ELF::R_PPC_ADDR16_HA: + writeInt16BE(LocalAddress, applyPPCha(Value + Addend)); + break; + } +} + void RuntimeDyldELF::resolvePPC64Relocation(const SectionEntry &Section, uint64_t Offset, uint64_t Value, uint32_t Type, int64_t Addend) { - uint8_t *LocalAddress = Section.Address + Offset; + uint8_t *LocalAddress = Section.getAddressWithOffset(Offset); switch (Type) { default: llvm_unreachable("Relocation type not implemented yet!"); @@ -929,17 +970,17 @@ void RuntimeDyldELF::resolvePPC64Relocation(const SectionEntry &Section, writeInt16BE(LocalAddress + 2, (aalk & 3) | ((Value + Addend) & 0xfffc)); } break; case ELF::R_PPC64_REL16_LO: { - uint64_t FinalAddress = (Section.LoadAddress + Offset); + uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset); uint64_t Delta = Value - FinalAddress + Addend; writeInt16BE(LocalAddress, applyPPClo(Delta)); } break; case ELF::R_PPC64_REL16_HI: { - uint64_t FinalAddress = (Section.LoadAddress + Offset); + uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset); uint64_t Delta = Value - FinalAddress + Addend; writeInt16BE(LocalAddress, applyPPChi(Delta)); } break; case ELF::R_PPC64_REL16_HA: { - uint64_t FinalAddress = (Section.LoadAddress + Offset); + uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset); uint64_t Delta = Value - FinalAddress + Addend; writeInt16BE(LocalAddress, applyPPCha(Delta)); } break; @@ -950,22 +991,22 @@ void RuntimeDyldELF::resolvePPC64Relocation(const SectionEntry &Section, writeInt32BE(LocalAddress, Result); } break; case ELF::R_PPC64_REL24: { - uint64_t FinalAddress = (Section.LoadAddress + Offset); + uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset); int32_t delta = static_cast<int32_t>(Value - FinalAddress + Addend); - if (SignExtend32<24>(delta) != delta) + if (SignExtend32<26>(delta) != delta) llvm_unreachable("Relocation R_PPC64_REL24 overflow"); // Generates a 'bl <address>' instruction writeInt32BE(LocalAddress, 0x48000001 | (delta & 0x03FFFFFC)); } break; case ELF::R_PPC64_REL32: { - uint64_t FinalAddress = (Section.LoadAddress + Offset); + uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset); int32_t delta = static_cast<int32_t>(Value - FinalAddress + Addend); if (SignExtend32<32>(delta) != delta) llvm_unreachable("Relocation R_PPC64_REL32 overflow"); writeInt32BE(LocalAddress, delta); } break; case ELF::R_PPC64_REL64: { - uint64_t FinalAddress = (Section.LoadAddress + Offset); + uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset); uint64_t Delta = Value - FinalAddress + Addend; writeInt64BE(LocalAddress, Delta); } break; @@ -978,27 +1019,27 @@ void RuntimeDyldELF::resolvePPC64Relocation(const SectionEntry &Section, void RuntimeDyldELF::resolveSystemZRelocation(const SectionEntry &Section, uint64_t Offset, uint64_t Value, uint32_t Type, int64_t Addend) { - uint8_t *LocalAddress = Section.Address + Offset; + uint8_t *LocalAddress = Section.getAddressWithOffset(Offset); switch (Type) { default: llvm_unreachable("Relocation type not implemented yet!"); break; case ELF::R_390_PC16DBL: case ELF::R_390_PLT16DBL: { - int64_t Delta = (Value + Addend) - (Section.LoadAddress + Offset); + int64_t Delta = (Value + Addend) - Section.getLoadAddressWithOffset(Offset); assert(int16_t(Delta / 2) * 2 == Delta && "R_390_PC16DBL overflow"); writeInt16BE(LocalAddress, Delta / 2); break; } case ELF::R_390_PC32DBL: case ELF::R_390_PLT32DBL: { - int64_t Delta = (Value + Addend) - (Section.LoadAddress + Offset); + int64_t Delta = (Value + Addend) - Section.getLoadAddressWithOffset(Offset); assert(int32_t(Delta / 2) * 2 == Delta && "R_390_PC32DBL overflow"); writeInt32BE(LocalAddress, Delta / 2); break; } case ELF::R_390_PC32: { - int64_t Delta = (Value + Addend) - (Section.LoadAddress + Offset); + int64_t Delta = (Value + Addend) - Section.getLoadAddressWithOffset(Offset); assert(int32_t(Delta) == Delta && "R_390_PC32 overflow"); writeInt32BE(LocalAddress, Delta); break; @@ -1072,6 +1113,9 @@ void RuntimeDyldELF::resolveRelocation(const SectionEntry &Section, else llvm_unreachable("Mips ABI not handled"); break; + case Triple::ppc: + resolvePPC32Relocation(Section, Offset, Value, Type, Addend); + break; case Triple::ppc64: // Fall through. case Triple::ppc64le: resolvePPC64Relocation(Section, Offset, Value, Type, Addend); @@ -1085,7 +1129,7 @@ void RuntimeDyldELF::resolveRelocation(const SectionEntry &Section, } void *RuntimeDyldELF::computePlaceholderAddress(unsigned SectionID, uint64_t Offset) const { - return (void*)(Sections[SectionID].ObjAddress + Offset); + return (void *)(Sections[SectionID].getObjAddress() + Offset); } void RuntimeDyldELF::processSimpleRelocation(unsigned SectionID, uint64_t Offset, unsigned RelType, RelocationValueRef Value) { @@ -1096,6 +1140,29 @@ void RuntimeDyldELF::processSimpleRelocation(unsigned SectionID, uint64_t Offset addRelocationForSection(RE, Value.SectionID); } +uint32_t RuntimeDyldELF::getMatchingLoRelocation(uint32_t RelType, + bool IsLocal) const { + switch (RelType) { + case ELF::R_MICROMIPS_GOT16: + if (IsLocal) + return ELF::R_MICROMIPS_LO16; + break; + case ELF::R_MICROMIPS_HI16: + return ELF::R_MICROMIPS_LO16; + case ELF::R_MIPS_GOT16: + if (IsLocal) + return ELF::R_MIPS_LO16; + break; + case ELF::R_MIPS_HI16: + return ELF::R_MIPS_LO16; + case ELF::R_MIPS_PCHI16: + return ELF::R_MIPS_PCLO16; + default: + break; + } + return ELF::R_MIPS_NONE; +} + relocation_iterator RuntimeDyldELF::processRelocationRef( unsigned SectionID, relocation_iterator RelI, const ObjectFile &O, ObjSectionToIDMap &ObjSectionToID, StubMap &Stubs) { @@ -1136,8 +1203,7 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( // TODO: Now ELF SymbolRef::ST_Debug = STT_SECTION, it's not obviously // and can be changed by another developers. Maybe best way is add // a new symbol type ST_Section to SymbolRef and use it. - section_iterator si(Obj.section_end()); - Symbol->getSection(si); + section_iterator si = *Symbol->getSection(); if (si == Obj.section_end()) llvm_unreachable("Symbol section not found, bad object file format!"); DEBUG(dbgs() << "\t\tThis is section symbol\n"); @@ -1178,24 +1244,28 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( // Look for an existing stub. StubMap::const_iterator i = Stubs.find(Value); if (i != Stubs.end()) { - resolveRelocation(Section, Offset, (uint64_t)Section.Address + i->second, + resolveRelocation(Section, Offset, + (uint64_t)Section.getAddressWithOffset(i->second), RelType, 0); DEBUG(dbgs() << " Stub function found\n"); } else { // Create a new stub function. DEBUG(dbgs() << " Create a new stub function\n"); - Stubs[Value] = Section.StubOffset; - uint8_t *StubTargetAddr = - createStubFunction(Section.Address + Section.StubOffset); + Stubs[Value] = Section.getStubOffset(); + uint8_t *StubTargetAddr = createStubFunction( + Section.getAddressWithOffset(Section.getStubOffset())); - RelocationEntry REmovz_g3(SectionID, StubTargetAddr - Section.Address, + RelocationEntry REmovz_g3(SectionID, + StubTargetAddr - Section.getAddress(), ELF::R_AARCH64_MOVW_UABS_G3, Value.Addend); - RelocationEntry REmovk_g2(SectionID, StubTargetAddr - Section.Address + 4, + RelocationEntry REmovk_g2(SectionID, StubTargetAddr - + Section.getAddress() + 4, ELF::R_AARCH64_MOVW_UABS_G2_NC, Value.Addend); - RelocationEntry REmovk_g1(SectionID, StubTargetAddr - Section.Address + 8, + RelocationEntry REmovk_g1(SectionID, StubTargetAddr - + Section.getAddress() + 8, ELF::R_AARCH64_MOVW_UABS_G1_NC, Value.Addend); - RelocationEntry REmovk_g0(SectionID, - StubTargetAddr - Section.Address + 12, + RelocationEntry REmovk_g0(SectionID, StubTargetAddr - + Section.getAddress() + 12, ELF::R_AARCH64_MOVW_UABS_G0_NC, Value.Addend); if (Value.SymbolName) { @@ -1210,9 +1280,10 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( addRelocationForSection(REmovk_g0, Value.SectionID); } resolveRelocation(Section, Offset, - (uint64_t)Section.Address + Section.StubOffset, RelType, - 0); - Section.StubOffset += getMaxStubSize(); + reinterpret_cast<uint64_t>(Section.getAddressWithOffset( + Section.getStubOffset())), + RelType, 0); + Section.advanceStubOffset(getMaxStubSize()); } } else if (Arch == Triple::arm) { if (RelType == ELF::R_ARM_PC24 || RelType == ELF::R_ARM_CALL || @@ -1224,26 +1295,29 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( // Look for an existing stub. StubMap::const_iterator i = Stubs.find(Value); if (i != Stubs.end()) { - resolveRelocation(Section, Offset, (uint64_t)Section.Address + i->second, - RelType, 0); + resolveRelocation( + Section, Offset, + reinterpret_cast<uint64_t>(Section.getAddressWithOffset(i->second)), + RelType, 0); DEBUG(dbgs() << " Stub function found\n"); } else { // Create a new stub function. DEBUG(dbgs() << " Create a new stub function\n"); - Stubs[Value] = Section.StubOffset; - uint8_t *StubTargetAddr = - createStubFunction(Section.Address + Section.StubOffset); - RelocationEntry RE(SectionID, StubTargetAddr - Section.Address, - ELF::R_ARM_ABS32, Value.Addend); + Stubs[Value] = Section.getStubOffset(); + uint8_t *StubTargetAddr = createStubFunction( + Section.getAddressWithOffset(Section.getStubOffset())); + RelocationEntry RE(SectionID, StubTargetAddr - Section.getAddress(), + ELF::R_ARM_ABS32, Value.Addend); if (Value.SymbolName) addRelocationForSymbol(RE, Value.SymbolName); else addRelocationForSection(RE, Value.SectionID); - resolveRelocation(Section, Offset, - (uint64_t)Section.Address + Section.StubOffset, RelType, - 0); - Section.StubOffset += getMaxStubSize(); + resolveRelocation(Section, Offset, reinterpret_cast<uint64_t>( + Section.getAddressWithOffset( + Section.getStubOffset())), + RelType, 0); + Section.advanceStubOffset(getMaxStubSize()); } } else { uint32_t *Placeholder = @@ -1282,15 +1356,16 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( } else { // Create a new stub function. DEBUG(dbgs() << " Create a new stub function\n"); - Stubs[Value] = Section.StubOffset; - uint8_t *StubTargetAddr = - createStubFunction(Section.Address + Section.StubOffset); + Stubs[Value] = Section.getStubOffset(); + uint8_t *StubTargetAddr = createStubFunction( + Section.getAddressWithOffset(Section.getStubOffset())); // Creating Hi and Lo relocations for the filled stub instructions. - RelocationEntry REHi(SectionID, StubTargetAddr - Section.Address, - ELF::R_MIPS_HI16, Value.Addend); - RelocationEntry RELo(SectionID, StubTargetAddr - Section.Address + 4, - ELF::R_MIPS_LO16, Value.Addend); + RelocationEntry REHi(SectionID, StubTargetAddr - Section.getAddress(), + ELF::R_MIPS_HI16, Value.Addend); + RelocationEntry RELo(SectionID, + StubTargetAddr - Section.getAddress() + 4, + ELF::R_MIPS_LO16, Value.Addend); if (Value.SymbolName) { addRelocationForSymbol(REHi, Value.SymbolName); @@ -1301,21 +1376,39 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( addRelocationForSection(RELo, Value.SectionID); } - RelocationEntry RE(SectionID, Offset, RelType, Section.StubOffset); + RelocationEntry RE(SectionID, Offset, RelType, Section.getStubOffset()); addRelocationForSection(RE, SectionID); - Section.StubOffset += getMaxStubSize(); + Section.advanceStubOffset(getMaxStubSize()); } + } else if (RelType == ELF::R_MIPS_HI16 || RelType == ELF::R_MIPS_PCHI16) { + int64_t Addend = (Opcode & 0x0000ffff) << 16; + RelocationEntry RE(SectionID, Offset, RelType, Addend); + PendingRelocs.push_back(std::make_pair(Value, RE)); + } else if (RelType == ELF::R_MIPS_LO16 || RelType == ELF::R_MIPS_PCLO16) { + int64_t Addend = Value.Addend + SignExtend32<16>(Opcode & 0x0000ffff); + for (auto I = PendingRelocs.begin(); I != PendingRelocs.end();) { + const RelocationValueRef &MatchingValue = I->first; + RelocationEntry &Reloc = I->second; + if (MatchingValue == Value && + RelType == getMatchingLoRelocation(Reloc.RelType) && + SectionID == Reloc.SectionID) { + Reloc.Addend += Addend; + if (Value.SymbolName) + addRelocationForSymbol(Reloc, Value.SymbolName); + else + addRelocationForSection(Reloc, Value.SectionID); + I = PendingRelocs.erase(I); + } else + ++I; + } + RelocationEntry RE(SectionID, Offset, RelType, Addend); + if (Value.SymbolName) + addRelocationForSymbol(RE, Value.SymbolName); + else + addRelocationForSection(RE, Value.SectionID); } else { - // FIXME: Calculate correct addends for R_MIPS_HI16, R_MIPS_LO16, - // R_MIPS_PCHI16 and R_MIPS_PCLO16 relocations. - if (RelType == ELF::R_MIPS_HI16 || RelType == ELF::R_MIPS_PCHI16) - Value.Addend += (Opcode & 0x0000ffff) << 16; - else if (RelType == ELF::R_MIPS_LO16) - Value.Addend += (Opcode & 0x0000ffff); - else if (RelType == ELF::R_MIPS_32) + if (RelType == ELF::R_MIPS_32) Value.Addend += Opcode; - else if (RelType == ELF::R_MIPS_PCLO16) - Value.Addend += SignExtend32<16>((Opcode & 0x0000ffff)); else if (RelType == ELF::R_MIPS_PC16) Value.Addend += SignExtend32<18>((Opcode & 0x0000ffff) << 2); else if (RelType == ELF::R_MIPS_PC19_S2) @@ -1353,7 +1446,7 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( // an external symbol (Symbol::ST_Unknown) or if the target address // is not within the signed 24-bits branch address. SectionEntry &Section = Sections[SectionID]; - uint8_t *Target = Section.Address + Offset; + uint8_t *Target = Section.getAddressWithOffset(Offset); bool RangeOverflow = false; if (SymType != SymbolRef::ST_Unknown) { if (AbiVariant != 2) { @@ -1367,10 +1460,11 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( uint8_t SymOther = Symbol->getOther(); Value.Addend += ELF::decodePPC64LocalEntryOffset(SymOther); } - uint8_t *RelocTarget = Sections[Value.SectionID].Address + Value.Addend; + uint8_t *RelocTarget = + Sections[Value.SectionID].getAddressWithOffset(Value.Addend); int32_t delta = static_cast<int32_t>(Target - RelocTarget); - // If it is within 24-bits branch range, just set the branch target - if (SignExtend32<24>(delta) == delta) { + // If it is within 26-bits branch range, just set the branch target + if (SignExtend32<26>(delta) == delta) { RelocationEntry RE(SectionID, Offset, RelType, Value.Addend); if (Value.SymbolName) addRelocationForSymbol(RE, Value.SymbolName); @@ -1387,23 +1481,25 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( if (i != Stubs.end()) { // Symbol function stub already created, just relocate to it resolveRelocation(Section, Offset, - (uint64_t)Section.Address + i->second, RelType, 0); + reinterpret_cast<uint64_t>( + Section.getAddressWithOffset(i->second)), + RelType, 0); DEBUG(dbgs() << " Stub function found\n"); } else { // Create a new stub function. DEBUG(dbgs() << " Create a new stub function\n"); - Stubs[Value] = Section.StubOffset; - uint8_t *StubTargetAddr = - createStubFunction(Section.Address + Section.StubOffset, - AbiVariant); - RelocationEntry RE(SectionID, StubTargetAddr - Section.Address, + Stubs[Value] = Section.getStubOffset(); + uint8_t *StubTargetAddr = createStubFunction( + Section.getAddressWithOffset(Section.getStubOffset()), + AbiVariant); + RelocationEntry RE(SectionID, StubTargetAddr - Section.getAddress(), ELF::R_PPC64_ADDR64, Value.Addend); // Generates the 64-bits address loads as exemplified in section // 4.5.1 in PPC64 ELF ABI. Note that the relocations need to // apply to the low part of the instructions, so we have to update // the offset according to the target endianness. - uint64_t StubRelocOffset = StubTargetAddr - Section.Address; + uint64_t StubRelocOffset = StubTargetAddr - Section.getAddress(); if (!IsTargetLittleEndian) StubRelocOffset += 2; @@ -1428,10 +1524,11 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( addRelocationForSection(REl, Value.SectionID); } - resolveRelocation(Section, Offset, - (uint64_t)Section.Address + Section.StubOffset, + resolveRelocation(Section, Offset, reinterpret_cast<uint64_t>( + Section.getAddressWithOffset( + Section.getStubOffset())), RelType, 0); - Section.StubOffset += getMaxStubSize(); + Section.advanceStubOffset(getMaxStubSize()); } if (SymType == SymbolRef::ST_Unknown) { // Restore the TOC for external calls @@ -1450,11 +1547,11 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( // These relocations are supposed to subtract the TOC address from // the final value. This does not fit cleanly into the RuntimeDyld // scheme, since there may be *two* sections involved in determining - // the relocation value (the section of the symbol refered to by the + // the relocation value (the section of the symbol referred to by the // relocation, and the TOC section associated with the current module). // // Fortunately, these relocations are currently only ever generated - // refering to symbols that themselves reside in the TOC, which means + // referring to symbols that themselves reside in the TOC, which means // that the two sections are actually the same. Thus they cancel out // and we can immediately resolve the relocation right now. switch (RelType) { @@ -1511,16 +1608,17 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( StubMap::const_iterator i = Stubs.find(Value); uintptr_t StubAddress; if (i != Stubs.end()) { - StubAddress = uintptr_t(Section.Address) + i->second; + StubAddress = uintptr_t(Section.getAddressWithOffset(i->second)); DEBUG(dbgs() << " Stub function found\n"); } else { // Create a new stub function. DEBUG(dbgs() << " Create a new stub function\n"); - uintptr_t BaseAddress = uintptr_t(Section.Address); + uintptr_t BaseAddress = uintptr_t(Section.getAddress()); uintptr_t StubAlignment = getStubAlignment(); - StubAddress = (BaseAddress + Section.StubOffset + StubAlignment - 1) & - -StubAlignment; + StubAddress = + (BaseAddress + Section.getStubOffset() + StubAlignment - 1) & + -StubAlignment; unsigned StubOffset = StubAddress - BaseAddress; Stubs[Value] = StubOffset; @@ -1531,7 +1629,7 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( addRelocationForSymbol(RE, Value.SymbolName); else addRelocationForSection(RE, Value.SectionID); - Section.StubOffset = StubOffset + getMaxStubSize(); + Section.advanceStubOffset(getMaxStubSize()); } if (RelType == ELF::R_390_GOTENT) @@ -1564,37 +1662,39 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( StubMap::const_iterator i = Stubs.find(Value); uintptr_t StubAddress; if (i != Stubs.end()) { - StubAddress = uintptr_t(Section.Address) + i->second; - DEBUG(dbgs() << " Stub function found\n"); + StubAddress = uintptr_t(Section.getAddress()) + i->second; + DEBUG(dbgs() << " Stub function found\n"); } else { - // Create a new stub function (equivalent to a PLT entry). - DEBUG(dbgs() << " Create a new stub function\n"); + // Create a new stub function (equivalent to a PLT entry). + DEBUG(dbgs() << " Create a new stub function\n"); - uintptr_t BaseAddress = uintptr_t(Section.Address); - uintptr_t StubAlignment = getStubAlignment(); - StubAddress = (BaseAddress + Section.StubOffset + StubAlignment - 1) & - -StubAlignment; - unsigned StubOffset = StubAddress - BaseAddress; - Stubs[Value] = StubOffset; - createStubFunction((uint8_t *)StubAddress); + uintptr_t BaseAddress = uintptr_t(Section.getAddress()); + uintptr_t StubAlignment = getStubAlignment(); + StubAddress = + (BaseAddress + Section.getStubOffset() + StubAlignment - 1) & + -StubAlignment; + unsigned StubOffset = StubAddress - BaseAddress; + Stubs[Value] = StubOffset; + createStubFunction((uint8_t *)StubAddress); - // Bump our stub offset counter - Section.StubOffset = StubOffset + getMaxStubSize(); + // Bump our stub offset counter + Section.advanceStubOffset(getMaxStubSize()); - // Allocate a GOT Entry - uint64_t GOTOffset = allocateGOTEntries(SectionID, 1); + // Allocate a GOT Entry + uint64_t GOTOffset = allocateGOTEntries(SectionID, 1); - // The load of the GOT address has an addend of -4 - resolveGOTOffsetRelocation(SectionID, StubOffset + 2, GOTOffset - 4); + // The load of the GOT address has an addend of -4 + resolveGOTOffsetRelocation(SectionID, StubOffset + 2, GOTOffset - 4); - // Fill in the value of the symbol we're targeting into the GOT - addRelocationForSymbol(computeGOTOffsetRE(SectionID,GOTOffset,0,ELF::R_X86_64_64), - Value.SymbolName); + // Fill in the value of the symbol we're targeting into the GOT + addRelocationForSymbol( + computeGOTOffsetRE(SectionID, GOTOffset, 0, ELF::R_X86_64_64), + Value.SymbolName); } // Make the target call a call into the stub table. resolveRelocation(Section, Offset, StubAddress, ELF::R_X86_64_PC32, - Addend); + Addend); } else { RelocationEntry RE(SectionID, Offset, ELF::R_X86_64_PC32, Value.Addend, Value.Offset); @@ -1670,7 +1770,7 @@ uint64_t RuntimeDyldELF::allocateGOTEntries(unsigned SectionID, unsigned no) GOTSectionID = Sections.size(); // Reserve a section id. We'll allocate the section later // once we know the total size - Sections.push_back(SectionEntry(".got", 0, 0, 0)); + Sections.push_back(SectionEntry(".got", nullptr, 0, 0, 0)); } uint64_t StartOffset = CurrentGOTIndex * getGOTEntrySize(); CurrentGOTIndex += no; @@ -1693,6 +1793,10 @@ RelocationEntry RuntimeDyldELF::computeGOTOffsetRE(unsigned SectionID, uint64_t void RuntimeDyldELF::finalizeLoad(const ObjectFile &Obj, ObjSectionToIDMap &SectionMap) { + if (IsMipsO32ABI) + if (!PendingRelocs.empty()) + report_fatal_error("Can't find matching LO16 reloc"); + // If necessary, allocate the global offset table if (GOTSectionID != 0) { // Allocate memory for the section @@ -1702,7 +1806,8 @@ void RuntimeDyldELF::finalizeLoad(const ObjectFile &Obj, if (!Addr) report_fatal_error("Unable to allocate memory for GOT!"); - Sections[GOTSectionID] = SectionEntry(".got", Addr, TotalSize, 0); + Sections[GOTSectionID] = + SectionEntry(".got", Addr, TotalSize, TotalSize, 0); if (Checker) Checker->registerSection(Obj.getFileName(), GOTSectionID); @@ -1746,4 +1851,23 @@ bool RuntimeDyldELF::isCompatibleFile(const object::ObjectFile &Obj) const { return Obj.isELF(); } +bool RuntimeDyldELF::relocationNeedsStub(const RelocationRef &R) const { + if (Arch != Triple::x86_64) + return true; // Conservative answer + + switch (R.getType()) { + default: + return true; // Conservative answer + + + case ELF::R_X86_64_GOTPCREL: + case ELF::R_X86_64_PC32: + case ELF::R_X86_64_PC64: + case ELF::R_X86_64_64: + // We know that these reloation types won't need a stub function. This list + // can be extended as needed. + return false; + } +} + } // namespace llvm diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h index 1a2552d..041811d 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h @@ -43,6 +43,9 @@ class RuntimeDyldELF : public RuntimeDyldImpl { void resolveMIPSRelocation(const SectionEntry &Section, uint64_t Offset, uint32_t Value, uint32_t Type, int32_t Addend); + void resolvePPC32Relocation(const SectionEntry &Section, uint64_t Offset, + uint64_t Value, uint32_t Type, int64_t Addend); + void resolvePPC64Relocation(const SectionEntry &Section, uint64_t Offset, uint64_t Value, uint32_t Type, int64_t Addend); @@ -120,6 +123,10 @@ class RuntimeDyldELF : public RuntimeDyldImpl { // no particular advanced processing. void processSimpleRelocation(unsigned SectionID, uint64_t Offset, unsigned RelType, RelocationValueRef Value); + // Return matching *LO16 relocation (Mips specific) + uint32_t getMatchingLoRelocation(uint32_t RelType, + bool IsLocal = false) const; + // The tentative ID for the GOT section unsigned GOTSectionID; @@ -135,12 +142,18 @@ class RuntimeDyldELF : public RuntimeDyldImpl { // A map to avoid duplicate got entries (Mips64 specific) StringMap<uint64_t> GOTSymbolOffsets; + // *HI16 relocations will be added for resolving when we find matching + // *LO16 part. (Mips specific) + SmallVector<std::pair<RelocationValueRef, RelocationEntry>, 8> PendingRelocs; + // When a module is loaded we save the SectionID of the EH frame section // in a table until we receive a request to register all unregistered // EH frame sections with the memory manager. SmallVector<SID, 2> UnregisteredEHFrameSections; SmallVector<SID, 2> RegisteredEHFrameSections; + bool relocationNeedsStub(const RelocationRef &R) const override; + public: RuntimeDyldELF(RuntimeDyld::MemoryManager &MemMgr, RuntimeDyld::SymbolResolver &Resolver); diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h index e085a92..dafd3c8 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h @@ -30,6 +30,7 @@ #include "llvm/Support/SwapByteOrder.h" #include "llvm/Support/raw_ostream.h" #include <map> +#include <unordered_map> #include <system_error> using namespace llvm; @@ -50,7 +51,6 @@ class Twine; /// SectionEntry - represents a section emitted into memory by the dynamic /// linker. class SectionEntry { -public: /// Name - section name. std::string Name; @@ -70,15 +70,54 @@ public: /// relocations (like ARM). uintptr_t StubOffset; + /// The total amount of space allocated for this section. This includes the + /// section size and the maximum amount of space that the stubs can occupy. + size_t AllocationSize; + /// ObjAddress - address of the section in the in-memory object file. Used /// for calculating relocations in some object formats (like MachO). uintptr_t ObjAddress; +public: SectionEntry(StringRef name, uint8_t *address, size_t size, - uintptr_t objAddress) + size_t allocationSize, uintptr_t objAddress) : Name(name), Address(address), Size(size), LoadAddress(reinterpret_cast<uintptr_t>(address)), StubOffset(size), - ObjAddress(objAddress) {} + AllocationSize(allocationSize), ObjAddress(objAddress) { + // AllocationSize is used only in asserts, prevent an "unused private field" + // warning: + (void)AllocationSize; + } + + StringRef getName() const { return Name; } + + uint8_t *getAddress() const { return Address; } + + /// \brief Return the address of this section with an offset. + uint8_t *getAddressWithOffset(unsigned OffsetBytes) const { + assert(OffsetBytes <= AllocationSize && "Offset out of bounds!"); + return Address + OffsetBytes; + } + + size_t getSize() const { return Size; } + + uint64_t getLoadAddress() const { return LoadAddress; } + void setLoadAddress(uint64_t LA) { LoadAddress = LA; } + + /// \brief Return the load address of this section with an offset. + uint64_t getLoadAddressWithOffset(unsigned OffsetBytes) const { + assert(OffsetBytes <= AllocationSize && "Offset out of bounds!"); + return LoadAddress + OffsetBytes; + } + + uintptr_t getStubOffset() const { return StubOffset; } + + void advanceStubOffset(unsigned StubSize) { + StubOffset += StubSize; + assert(StubOffset <= AllocationSize && "Not enough space allocated!"); + } + + uintptr_t getObjAddress() const { return ObjAddress; } }; /// RelocationEntry - used to represent relocations internally in the dynamic @@ -188,6 +227,8 @@ class RuntimeDyldImpl { friend class RuntimeDyld::LoadedObjectInfo; friend class RuntimeDyldCheckerImpl; protected: + static const unsigned AbsoluteSymbolSection = ~0U; + // The MemoryManager to load objects into. RuntimeDyld::MemoryManager &MemMgr; @@ -224,7 +265,7 @@ protected: // Relocations to sections already loaded. Indexed by SectionID which is the // source of the address. The target where the address will be written is // SectionID/Offset in the relocation itself. - DenseMap<unsigned, RelocationList> Relocations; + std::unordered_map<unsigned, RelocationList> Relocations; // Relocations to external symbols that are not yet resolved. Symbols are // external when they aren't found in the global symbol table of all loaded @@ -269,11 +310,11 @@ protected: } uint64_t getSectionLoadAddress(unsigned SectionID) const { - return Sections[SectionID].LoadAddress; + return Sections[SectionID].getLoadAddress(); } uint8_t *getSectionAddress(unsigned SectionID) const { - return (uint8_t *)Sections[SectionID].Address; + return Sections[SectionID].getAddress(); } void writeInt16BE(uint8_t *Addr, uint16_t Value) { @@ -378,7 +419,12 @@ protected: const SectionRef &Section); // \brief Implementation of the generic part of the loadObject algorithm. - std::pair<unsigned, unsigned> loadObjectImpl(const object::ObjectFile &Obj); + ObjSectionToIDMap loadObjectImpl(const object::ObjectFile &Obj); + + // \brief Return true if the relocation R may require allocating a stub. + virtual bool relocationNeedsStub(const RelocationRef &R) const { + return true; // Conservative answer + } public: RuntimeDyldImpl(RuntimeDyld::MemoryManager &MemMgr, @@ -407,6 +453,9 @@ public: if (pos == GlobalSymbolTable.end()) return nullptr; const auto &SymInfo = pos->second; + // Absolute symbols do not have a local address. + if (SymInfo.getSectionID() == AbsoluteSymbolSection) + return nullptr; return getSectionAddress(SymInfo.getSectionID()) + SymInfo.getOffset(); } @@ -417,8 +466,10 @@ public: if (pos == GlobalSymbolTable.end()) return nullptr; const auto &SymEntry = pos->second; - uint64_t TargetAddr = - getSectionLoadAddress(SymEntry.getSectionID()) + SymEntry.getOffset(); + uint64_t SectionAddr = 0; + if (SymEntry.getSectionID() != AbsoluteSymbolSection) + SectionAddr = getSectionLoadAddress(SymEntry.getSectionID()); + uint64_t TargetAddr = SectionAddr + SymEntry.getOffset(); return RuntimeDyld::SymbolInfo(TargetAddr, SymEntry.getFlags()); } diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp index c074114..739e8d6 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp @@ -26,12 +26,12 @@ using namespace llvm::object; namespace { -class LoadedMachOObjectInfo +class LoadedMachOObjectInfo final : public RuntimeDyld::LoadedObjectInfoHelper<LoadedMachOObjectInfo> { public: - LoadedMachOObjectInfo(RuntimeDyldImpl &RTDyld, unsigned BeginIdx, - unsigned EndIdx) - : LoadedObjectInfoHelper(RTDyld, BeginIdx, EndIdx) {} + LoadedMachOObjectInfo(RuntimeDyldImpl &RTDyld, + ObjSectionToIDMap ObjSecToIDMap) + : LoadedObjectInfoHelper(RTDyld, std::move(ObjSecToIDMap)) {} OwningBinary<ObjectFile> getObjectForDebug(const ObjectFile &Obj) const override { @@ -45,11 +45,47 @@ namespace llvm { int64_t RuntimeDyldMachO::memcpyAddend(const RelocationEntry &RE) const { unsigned NumBytes = 1 << RE.Size; - uint8_t *Src = Sections[RE.SectionID].Address + RE.Offset; + uint8_t *Src = Sections[RE.SectionID].getAddress() + RE.Offset; return static_cast<int64_t>(readBytesUnaligned(Src, NumBytes)); } +relocation_iterator RuntimeDyldMachO::processScatteredVANILLA( + unsigned SectionID, relocation_iterator RelI, + const ObjectFile &BaseObjT, + RuntimeDyldMachO::ObjSectionToIDMap &ObjSectionToID) { + const MachOObjectFile &Obj = + static_cast<const MachOObjectFile&>(BaseObjT); + MachO::any_relocation_info RE = + Obj.getRelocation(RelI->getRawDataRefImpl()); + + SectionEntry &Section = Sections[SectionID]; + uint32_t RelocType = Obj.getAnyRelocationType(RE); + bool IsPCRel = Obj.getAnyRelocationPCRel(RE); + unsigned Size = Obj.getAnyRelocationLength(RE); + uint64_t Offset = RelI->getOffset(); + uint8_t *LocalAddress = Section.getAddressWithOffset(Offset); + unsigned NumBytes = 1 << Size; + int64_t Addend = readBytesUnaligned(LocalAddress, NumBytes); + + unsigned SymbolBaseAddr = Obj.getScatteredRelocationValue(RE); + section_iterator TargetSI = getSectionByAddress(Obj, SymbolBaseAddr); + assert(TargetSI != Obj.section_end() && "Can't find section for symbol"); + uint64_t SectionBaseAddr = TargetSI->getAddress(); + SectionRef TargetSection = *TargetSI; + bool IsCode = TargetSection.isText(); + uint32_t TargetSectionID = + findOrEmitSection(Obj, TargetSection, IsCode, ObjSectionToID); + + Addend -= SectionBaseAddr; + RelocationEntry R(SectionID, Offset, RelocType, Addend, IsPCRel, Size); + + addRelocationForSection(R, TargetSectionID); + + return ++RelI; +} + + RelocationValueRef RuntimeDyldMachO::getRelocationValueRef( const ObjectFile &BaseTObj, const relocation_iterator &RI, const RelocationEntry &RE, ObjSectionToIDMap &ObjSectionToID) { @@ -99,8 +135,8 @@ void RuntimeDyldMachO::makeValueAddendPCRel(RelocationValueRef &Value, void RuntimeDyldMachO::dumpRelocationToResolve(const RelocationEntry &RE, uint64_t Value) const { const SectionEntry &Section = Sections[RE.SectionID]; - uint8_t *LocalAddress = Section.Address + RE.Offset; - uint64_t FinalAddress = Section.LoadAddress + RE.Offset; + uint8_t *LocalAddress = Section.getAddress() + RE.Offset; + uint64_t FinalAddress = Section.getLoadAddress() + RE.Offset; dbgs() << "resolveRelocation Section: " << RE.SectionID << " LocalAddress: " << format("%p", LocalAddress) @@ -147,10 +183,9 @@ void RuntimeDyldMachO::populateIndirectSymbolPointersSection( "Pointers section does not contain a whole number of stubs?"); DEBUG(dbgs() << "Populating pointer table section " - << Sections[PTSectionID].Name - << ", Section ID " << PTSectionID << ", " - << NumPTEntries << " entries, " << PTEntrySize - << " bytes each:\n"); + << Sections[PTSectionID].getName() << ", Section ID " + << PTSectionID << ", " << NumPTEntries << " entries, " + << PTEntrySize << " bytes each:\n"); for (unsigned i = 0; i < NumPTEntries; ++i) { unsigned SymbolIndex = @@ -204,7 +239,7 @@ void RuntimeDyldMachOCRTPBase<Impl>::finalizeLoad(const ObjectFile &Obj, } template <typename Impl> -unsigned char *RuntimeDyldMachOCRTPBase<Impl>::processFDE(unsigned char *P, +unsigned char *RuntimeDyldMachOCRTPBase<Impl>::processFDE(uint8_t *P, int64_t DeltaForText, int64_t DeltaForEH) { typedef typename Impl::TargetPtrT TargetPtrT; @@ -213,7 +248,7 @@ unsigned char *RuntimeDyldMachOCRTPBase<Impl>::processFDE(unsigned char *P, << ", Delta for EH: " << DeltaForEH << "\n"); uint32_t Length = readBytesUnaligned(P, 4); P += 4; - unsigned char *Ret = P + Length; + uint8_t *Ret = P + Length; uint32_t Offset = readBytesUnaligned(P, 4); if (Offset == 0) // is a CIE return Ret; @@ -240,9 +275,9 @@ unsigned char *RuntimeDyldMachOCRTPBase<Impl>::processFDE(unsigned char *P, } static int64_t computeDelta(SectionEntry *A, SectionEntry *B) { - int64_t ObjDistance = - static_cast<int64_t>(A->ObjAddress) - static_cast<int64_t>(B->ObjAddress); - int64_t MemDistance = A->LoadAddress - B->LoadAddress; + int64_t ObjDistance = static_cast<int64_t>(A->getObjAddress()) - + static_cast<int64_t>(B->getObjAddress()); + int64_t MemDistance = A->getLoadAddress() - B->getLoadAddress(); return ObjDistance - MemDistance; } @@ -265,14 +300,14 @@ void RuntimeDyldMachOCRTPBase<Impl>::registerEHFrames() { if (ExceptTab) DeltaForEH = computeDelta(ExceptTab, EHFrame); - unsigned char *P = EHFrame->Address; - unsigned char *End = P + EHFrame->Size; + uint8_t *P = EHFrame->getAddress(); + uint8_t *End = P + EHFrame->getSize(); do { P = processFDE(P, DeltaForText, DeltaForEH); } while (P != End); - MemMgr.registerEHFrames(EHFrame->Address, EHFrame->LoadAddress, - EHFrame->Size); + MemMgr.registerEHFrames(EHFrame->getAddress(), EHFrame->getLoadAddress(), + EHFrame->getSize()); } UnregisteredEHFrameSections.clear(); } @@ -298,10 +333,7 @@ RuntimeDyldMachO::create(Triple::ArchType Arch, std::unique_ptr<RuntimeDyld::LoadedObjectInfo> RuntimeDyldMachO::loadObject(const object::ObjectFile &O) { - unsigned SectionStartIdx, SectionEndIdx; - std::tie(SectionStartIdx, SectionEndIdx) = loadObjectImpl(O); - return llvm::make_unique<LoadedMachOObjectInfo>(*this, SectionStartIdx, - SectionEndIdx); + return llvm::make_unique<LoadedMachOObjectInfo>(*this, loadObjectImpl(O)); } } // end namespace llvm diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h index 0d7364f..c8ae47b 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h @@ -79,6 +79,12 @@ protected: return RelocationEntry(SectionID, Offset, RelType, 0, IsPCRel, Size); } + /// Process a scattered vanilla relocation. + relocation_iterator processScatteredVANILLA( + unsigned SectionID, relocation_iterator RelI, + const ObjectFile &BaseObjT, + RuntimeDyldMachO::ObjSectionToIDMap &ObjSectionToID); + /// Construct a RelocationValueRef representing the relocation target. /// For Symbols in known sections, this will return a RelocationValueRef /// representing a (SectionID, Offset) pair. @@ -140,7 +146,7 @@ private: Impl &impl() { return static_cast<Impl &>(*this); } const Impl &impl() const { return static_cast<const Impl &>(*this); } - unsigned char *processFDE(unsigned char *P, int64_t DeltaForText, + unsigned char *processFDE(uint8_t *P, int64_t DeltaForText, int64_t DeltaForEH); public: diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h new file mode 100644 index 0000000..fbfbb32 --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h @@ -0,0 +1,201 @@ +//===--- RuntimeDyldCOFFI386.h --- COFF/X86_64 specific code ---*- C++ --*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// COFF x86 support for MC-JIT runtime dynamic linker. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDCOFFI386_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDCOFFI386_H + +#include "llvm/Object/COFF.h" +#include "llvm/Support/COFF.h" +#include "../RuntimeDyldCOFF.h" + +#define DEBUG_TYPE "dyld" + +namespace llvm { + +class RuntimeDyldCOFFI386 : public RuntimeDyldCOFF { +public: + RuntimeDyldCOFFI386(RuntimeDyld::MemoryManager &MM, + RuntimeDyld::SymbolResolver &Resolver) + : RuntimeDyldCOFF(MM, Resolver) {} + + unsigned getMaxStubSize() override { + return 8; // 2-byte jmp instruction + 32-bit relative address + 2 byte pad + } + + unsigned getStubAlignment() override { return 1; } + + relocation_iterator processRelocationRef(unsigned SectionID, + relocation_iterator RelI, + const ObjectFile &Obj, + ObjSectionToIDMap &ObjSectionToID, + StubMap &Stubs) override { + auto Symbol = RelI->getSymbol(); + if (Symbol == Obj.symbol_end()) + report_fatal_error("Unknown symbol in relocation"); + + ErrorOr<StringRef> TargetNameOrErr = Symbol->getName(); + if (auto EC = TargetNameOrErr.getError()) + report_fatal_error(EC.message()); + StringRef TargetName = *TargetNameOrErr; + + auto Section = *Symbol->getSection(); + + uint64_t RelType = RelI->getType(); + uint64_t Offset = RelI->getOffset(); + +#if !defined(NDEBUG) + SmallString<32> RelTypeName; + RelI->getTypeName(RelTypeName); +#endif + DEBUG(dbgs() << "\t\tIn Section " << SectionID << " Offset " << Offset + << " RelType: " << RelTypeName << " TargetName: " << TargetName + << "\n"); + + unsigned TargetSectionID = -1; + if (Section == Obj.section_end()) { + RelocationEntry RE(SectionID, Offset, RelType, 0, -1, 0, 0, 0, false, 0); + addRelocationForSymbol(RE, TargetName); + } else { + TargetSectionID = + findOrEmitSection(Obj, *Section, Section->isText(), ObjSectionToID); + + switch (RelType) { + case COFF::IMAGE_REL_I386_ABSOLUTE: + // This relocation is ignored. + break; + case COFF::IMAGE_REL_I386_DIR32: + case COFF::IMAGE_REL_I386_DIR32NB: + case COFF::IMAGE_REL_I386_REL32: { + RelocationEntry RE = + RelocationEntry(SectionID, Offset, RelType, 0, TargetSectionID, + getSymbolOffset(*Symbol), 0, 0, false, 0); + addRelocationForSection(RE, TargetSectionID); + break; + } + case COFF::IMAGE_REL_I386_SECTION: { + RelocationEntry RE = + RelocationEntry(TargetSectionID, Offset, RelType, 0); + addRelocationForSection(RE, TargetSectionID); + break; + } + case COFF::IMAGE_REL_I386_SECREL: { + RelocationEntry RE = RelocationEntry(SectionID, Offset, RelType, + getSymbolOffset(*Symbol)); + addRelocationForSection(RE, TargetSectionID); + break; + } + default: + llvm_unreachable("unsupported relocation type"); + } + + } + + return ++RelI; + } + + void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override { + const auto Section = Sections[RE.SectionID]; + uint8_t *Target = Section.getAddressWithOffset(RE.Offset); + + switch (RE.RelType) { + case COFF::IMAGE_REL_I386_ABSOLUTE: + // This relocation is ignored. + break; + case COFF::IMAGE_REL_I386_DIR32: { + // The target's 32-bit VA. + uint64_t Result = + RE.Sections.SectionA == static_cast<uint32_t>(-1) + ? Value + : Sections[RE.Sections.SectionA].getLoadAddressWithOffset( + RE.Addend); + assert(static_cast<int32_t>(Result) <= INT32_MAX && + "relocation overflow"); + assert(static_cast<int32_t>(Result) >= INT32_MIN && + "relocation underflow"); + DEBUG(dbgs() << "\t\tOffset: " << RE.Offset + << " RelType: IMAGE_REL_I386_DIR32" + << " TargetSection: " << RE.Sections.SectionA + << " Value: " << format("0x%08" PRIx32, Result) << '\n'); + writeBytesUnaligned(Result, Target, 4); + break; + } + case COFF::IMAGE_REL_I386_DIR32NB: { + // The target's 32-bit RVA. + // NOTE: use Section[0].getLoadAddress() as an approximation of ImageBase + uint64_t Result = + Sections[RE.Sections.SectionA].getLoadAddressWithOffset(RE.Addend) - + Sections[0].getLoadAddress(); + assert(static_cast<int32_t>(Result) <= INT32_MAX && + "relocation overflow"); + assert(static_cast<int32_t>(Result) >= INT32_MIN && + "relocation underflow"); + DEBUG(dbgs() << "\t\tOffset: " << RE.Offset + << " RelType: IMAGE_REL_I386_DIR32NB" + << " TargetSection: " << RE.Sections.SectionA + << " Value: " << format("0x%08" PRIx32, Result) << '\n'); + writeBytesUnaligned(Result, Target, 4); + break; + } + case COFF::IMAGE_REL_I386_REL32: { + // 32-bit relative displacement to the target. + uint64_t Result = Sections[RE.Sections.SectionA].getLoadAddress() - + Section.getLoadAddress() + RE.Addend - 4 - RE.Offset; + assert(static_cast<int32_t>(Result) <= INT32_MAX && + "relocation overflow"); + assert(static_cast<int32_t>(Result) >= INT32_MIN && + "relocation underflow"); + DEBUG(dbgs() << "\t\tOffset: " << RE.Offset + << " RelType: IMAGE_REL_I386_REL32" + << " TargetSection: " << RE.Sections.SectionA + << " Value: " << format("0x%08" PRIx32, Result) << '\n'); + writeBytesUnaligned(Result, Target, 4); + break; + } + case COFF::IMAGE_REL_I386_SECTION: + // 16-bit section index of the section that contains the target. + assert(static_cast<int32_t>(RE.SectionID) <= INT16_MAX && + "relocation overflow"); + assert(static_cast<int32_t>(RE.SectionID) >= INT16_MIN && + "relocation underflow"); + DEBUG(dbgs() << "\t\tOffset: " << RE.Offset + << " RelType: IMAGE_REL_I386_SECTION Value: " << RE.SectionID + << '\n'); + writeBytesUnaligned(RE.SectionID, Target, 2); + break; + case COFF::IMAGE_REL_I386_SECREL: + // 32-bit offset of the target from the beginning of its section. + assert(static_cast<int32_t>(RE.Addend) <= INT32_MAX && + "relocation overflow"); + assert(static_cast<int32_t>(RE.Addend) >= INT32_MIN && + "relocation underflow"); + DEBUG(dbgs() << "\t\tOffset: " << RE.Offset + << " RelType: IMAGE_REL_I386_SECREL Value: " << RE.Addend + << '\n'); + writeBytesUnaligned(RE.Addend, Target, 2); + break; + default: + llvm_unreachable("unsupported relocation type"); + } + } + + void registerEHFrames() override {} + void deregisterEHFrames() override {} + + void finalizeLoad(const ObjectFile &Obj, + ObjSectionToIDMap &SectionMap) override {} +}; + +} + +#endif + diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h index 408227e..25f538d 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h @@ -62,7 +62,7 @@ public: // symbol in the target address space. void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override { const SectionEntry &Section = Sections[RE.SectionID]; - uint8_t *Target = Section.Address + RE.Offset; + uint8_t *Target = Section.getAddressWithOffset(RE.Offset); switch (RE.RelType) { @@ -72,8 +72,7 @@ public: case COFF::IMAGE_REL_AMD64_REL32_3: case COFF::IMAGE_REL_AMD64_REL32_4: case COFF::IMAGE_REL_AMD64_REL32_5: { - uint32_t *TargetAddress = (uint32_t *)Target; - uint64_t FinalAddress = Section.LoadAddress + RE.Offset; + uint64_t FinalAddress = Section.getLoadAddressWithOffset(RE.Offset); // Delta is the distance from the start of the reloc to the end of the // instruction with the reloc. uint64_t Delta = 4 + (RE.RelType - COFF::IMAGE_REL_AMD64_REL32); @@ -81,7 +80,7 @@ public: uint64_t Result = Value + RE.Addend; assert(((int64_t)Result <= INT32_MAX) && "Relocation overflow"); assert(((int64_t)Result >= INT32_MIN) && "Relocation underflow"); - *TargetAddress = Result; + writeBytesUnaligned(Result, Target, 4); break; } @@ -92,14 +91,12 @@ public: // within a 32 bit offset from the base. // // For now we just set these to zero. - uint32_t *TargetAddress = (uint32_t *)Target; - *TargetAddress = 0; + writeBytesUnaligned(0, Target, 4); break; } case COFF::IMAGE_REL_AMD64_ADDR64: { - uint64_t *TargetAddress = (uint64_t *)Target; - *TargetAddress = Value + RE.Addend; + writeBytesUnaligned(Value + RE.Addend, Target, 8); break; } @@ -119,8 +116,7 @@ public: symbol_iterator Symbol = RelI->getSymbol(); if (Symbol == Obj.symbol_end()) report_fatal_error("Unknown symbol in relocation"); - section_iterator SecI(Obj.section_end()); - Symbol->getSection(SecI); + section_iterator SecI = *Symbol->getSection(); // If there is no section, this must be an external reference. const bool IsExtern = SecI == Obj.section_end(); @@ -129,7 +125,7 @@ public: uint64_t Offset = RelI->getOffset(); uint64_t Addend = 0; SectionEntry &Section = Sections[SectionID]; - uintptr_t ObjTarget = Section.ObjAddress + Offset; + uintptr_t ObjTarget = Section.getObjAddress() + Offset; switch (RelType) { @@ -140,14 +136,14 @@ public: case COFF::IMAGE_REL_AMD64_REL32_4: case COFF::IMAGE_REL_AMD64_REL32_5: case COFF::IMAGE_REL_AMD64_ADDR32NB: { - uint32_t *Displacement = (uint32_t *)ObjTarget; - Addend = *Displacement; + uint8_t *Displacement = (uint8_t *)ObjTarget; + Addend = readBytesUnaligned(Displacement, 4); break; } case COFF::IMAGE_REL_AMD64_ADDR64: { - uint64_t *Displacement = (uint64_t *)ObjTarget; - Addend = *Displacement; + uint8_t *Displacement = (uint8_t *)ObjTarget; + Addend = readBytesUnaligned(Displacement, 8); break; } @@ -182,9 +178,9 @@ public: unsigned getStubAlignment() override { return 1; } void registerEHFrames() override { for (auto const &EHFrameSID : UnregisteredEHFrameSections) { - uint8_t *EHFrameAddr = Sections[EHFrameSID].Address; - uint64_t EHFrameLoadAddr = Sections[EHFrameSID].LoadAddress; - size_t EHFrameSize = Sections[EHFrameSID].Size; + uint8_t *EHFrameAddr = Sections[EHFrameSID].getAddress(); + uint64_t EHFrameLoadAddr = Sections[EHFrameSID].getLoadAddress(); + size_t EHFrameSize = Sections[EHFrameSID].getSize(); MemMgr.registerEHFrames(EHFrameAddr, EHFrameLoadAddr, EHFrameSize); RegisteredEHFrameSections.push_back(EHFrameSID); } diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h index 7bf7641..dbca377 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h @@ -34,7 +34,7 @@ public: /// Extract the addend encoded in the instruction / memory location. int64_t decodeAddend(const RelocationEntry &RE) const { const SectionEntry &Section = Sections[RE.SectionID]; - uint8_t *LocalAddress = Section.Address + RE.Offset; + uint8_t *LocalAddress = Section.getAddressWithOffset(RE.Offset); unsigned NumBytes = 1 << RE.Size; int64_t Addend = 0; // Verify that the relocation has the correct size and alignment. @@ -272,15 +272,14 @@ public: RelocationEntry RE(getRelocationEntry(SectionID, Obj, RelI)); RE.Addend = decodeAddend(RE); - RelocationValueRef Value( - getRelocationValueRef(Obj, RelI, RE, ObjSectionToID)); assert((ExplicitAddend == 0 || RE.Addend == 0) && "Relocation has "\ "ARM64_RELOC_ADDEND and embedded addend in the instruction."); - if (ExplicitAddend) { + if (ExplicitAddend) RE.Addend = ExplicitAddend; - Value.Offset = ExplicitAddend; - } + + RelocationValueRef Value( + getRelocationValueRef(Obj, RelI, RE, ObjSectionToID)); bool IsExtern = Obj.getPlainRelocationExternal(RelInfo); if (!IsExtern && RE.IsPCRel) @@ -305,7 +304,7 @@ public: DEBUG(dumpRelocationToResolve(RE, Value)); const SectionEntry &Section = Sections[RE.SectionID]; - uint8_t *LocalAddress = Section.Address + RE.Offset; + uint8_t *LocalAddress = Section.getAddressWithOffset(RE.Offset); MachO::RelocationInfoType RelType = static_cast<MachO::RelocationInfoType>(RE.RelType); @@ -325,7 +324,7 @@ public: case MachO::ARM64_RELOC_BRANCH26: { assert(RE.IsPCRel && "not PCRel and ARM64_RELOC_BRANCH26 not supported"); // Check if branch is in range. - uint64_t FinalAddress = Section.LoadAddress + RE.Offset; + uint64_t FinalAddress = Section.getLoadAddressWithOffset(RE.Offset); int64_t PCRelVal = Value - FinalAddress + RE.Addend; encodeAddend(LocalAddress, /*Size=*/4, RelType, PCRelVal); break; @@ -334,7 +333,7 @@ public: case MachO::ARM64_RELOC_PAGE21: { assert(RE.IsPCRel && "not PCRel and ARM64_RELOC_PAGE21 not supported"); // Adjust for PC-relative relocation and offset. - uint64_t FinalAddress = Section.LoadAddress + RE.Offset; + uint64_t FinalAddress = Section.getLoadAddressWithOffset(RE.Offset); int64_t PCRelVal = ((Value + RE.Addend) & (-4096)) - (FinalAddress & (-4096)); encodeAddend(LocalAddress, /*Size=*/4, RelType, PCRelVal); @@ -376,10 +375,10 @@ private: else { // FIXME: There must be a better way to do this then to check and fix the // alignment every time!!! - uintptr_t BaseAddress = uintptr_t(Section.Address); + uintptr_t BaseAddress = uintptr_t(Section.getAddress()); uintptr_t StubAlignment = getStubAlignment(); uintptr_t StubAddress = - (BaseAddress + Section.StubOffset + StubAlignment - 1) & + (BaseAddress + Section.getStubOffset() + StubAlignment - 1) & -StubAlignment; unsigned StubOffset = StubAddress - BaseAddress; Stubs[Value] = StubOffset; @@ -392,7 +391,7 @@ private: addRelocationForSymbol(GOTRE, Value.SymbolName); else addRelocationForSection(GOTRE, Value.SectionID); - Section.StubOffset = StubOffset + getMaxStubSize(); + Section.advanceStubOffset(getMaxStubSize()); Offset = static_cast<int64_t>(StubOffset); } RelocationEntry TargetRE(RE.SectionID, RE.Offset, RE.RelType, Offset, diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h index 0a24bb2..7731df0 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h @@ -35,7 +35,7 @@ public: int64_t decodeAddend(const RelocationEntry &RE) const { const SectionEntry &Section = Sections[RE.SectionID]; - uint8_t *LocalAddress = Section.Address + RE.Offset; + uint8_t *LocalAddress = Section.getAddressWithOffset(RE.Offset); switch (RE.RelType) { default: @@ -64,8 +64,10 @@ public: if (RelType == MachO::ARM_RELOC_HALF_SECTDIFF) return processHALFSECTDIFFRelocation(SectionID, RelI, Obj, ObjSectionToID); + else if (RelType == MachO::GENERIC_RELOC_VANILLA) + return processScatteredVANILLA(SectionID, RelI, Obj, ObjSectionToID); else - return ++++RelI; + return ++RelI; } RelocationEntry RE(getRelocationEntry(SectionID, Obj, RelI)); @@ -92,12 +94,12 @@ public: void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override { DEBUG(dumpRelocationToResolve(RE, Value)); const SectionEntry &Section = Sections[RE.SectionID]; - uint8_t *LocalAddress = Section.Address + RE.Offset; + uint8_t *LocalAddress = Section.getAddressWithOffset(RE.Offset); // If the relocation is PC-relative, the value to be encoded is the // pointer difference. if (RE.IsPCRel) { - uint64_t FinalAddress = Section.LoadAddress + RE.Offset; + uint64_t FinalAddress = Section.getLoadAddressWithOffset(RE.Offset); Value -= FinalAddress; // ARM PCRel relocations have an effective-PC offset of two instructions // (four bytes in Thumb mode, 8 bytes in ARM mode). @@ -130,8 +132,8 @@ public: break; } case MachO::ARM_RELOC_HALF_SECTDIFF: { - uint64_t SectionABase = Sections[RE.Sections.SectionA].LoadAddress; - uint64_t SectionBBase = Sections[RE.Sections.SectionB].LoadAddress; + uint64_t SectionABase = Sections[RE.Sections.SectionA].getLoadAddress(); + uint64_t SectionBBase = Sections[RE.Sections.SectionB].getLoadAddress(); assert((Value == SectionABase || Value == SectionBBase) && "Unexpected HALFSECTDIFF relocation value."); Value = SectionABase - SectionBBase + RE.Addend; @@ -178,21 +180,21 @@ private: RuntimeDyldMachO::StubMap::const_iterator i = Stubs.find(Value); uint8_t *Addr; if (i != Stubs.end()) { - Addr = Section.Address + i->second; + Addr = Section.getAddressWithOffset(i->second); } else { // Create a new stub function. - Stubs[Value] = Section.StubOffset; - uint8_t *StubTargetAddr = - createStubFunction(Section.Address + Section.StubOffset); - RelocationEntry StubRE(RE.SectionID, StubTargetAddr - Section.Address, - MachO::GENERIC_RELOC_VANILLA, Value.Offset, false, - 2); + Stubs[Value] = Section.getStubOffset(); + uint8_t *StubTargetAddr = createStubFunction( + Section.getAddressWithOffset(Section.getStubOffset())); + RelocationEntry StubRE( + RE.SectionID, StubTargetAddr - Section.getAddress(), + MachO::GENERIC_RELOC_VANILLA, Value.Offset, false, 2); if (Value.SymbolName) addRelocationForSymbol(StubRE, Value.SymbolName); else addRelocationForSection(StubRE, Value.SectionID); - Addr = Section.Address + Section.StubOffset; - Section.StubOffset += getMaxStubSize(); + Addr = Section.getAddressWithOffset(Section.getStubOffset()); + Section.advanceStubOffset(getMaxStubSize()); } RelocationEntry TargetRE(RE.SectionID, RE.Offset, RE.RelType, 0, RE.IsPCRel, RE.Size); @@ -221,7 +223,7 @@ private: uint32_t RelocType = MachO.getAnyRelocationType(RE); bool IsPCRel = MachO.getAnyRelocationPCRel(RE); uint64_t Offset = RelI->getOffset(); - uint8_t *LocalAddress = Section.Address + Offset; + uint8_t *LocalAddress = Section.getAddressWithOffset(Offset); int64_t Immediate = readBytesUnaligned(LocalAddress, 4); // Copy the whole instruction out. Immediate = ((Immediate >> 4) & 0xf000) | (Immediate & 0xfff); diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h index 569a078..85059d7 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h @@ -47,8 +47,7 @@ public: return processSECTDIFFRelocation(SectionID, RelI, Obj, ObjSectionToID); else if (RelType == MachO::GENERIC_RELOC_VANILLA) - return processI386ScatteredVANILLA(SectionID, RelI, Obj, - ObjSectionToID); + return processScatteredVANILLA(SectionID, RelI, Obj, ObjSectionToID); llvm_unreachable("Unhandled scattered relocation."); } @@ -84,10 +83,10 @@ public: DEBUG(dumpRelocationToResolve(RE, Value)); const SectionEntry &Section = Sections[RE.SectionID]; - uint8_t *LocalAddress = Section.Address + RE.Offset; + uint8_t *LocalAddress = Section.getAddressWithOffset(RE.Offset); if (RE.IsPCRel) { - uint64_t FinalAddress = Section.LoadAddress + RE.Offset; + uint64_t FinalAddress = Section.getLoadAddressWithOffset(RE.Offset); Value -= FinalAddress + 4; // see MachOX86_64::resolveRelocation. } @@ -99,8 +98,8 @@ public: break; case MachO::GENERIC_RELOC_SECTDIFF: case MachO::GENERIC_RELOC_LOCAL_SECTDIFF: { - uint64_t SectionABase = Sections[RE.Sections.SectionA].LoadAddress; - uint64_t SectionBBase = Sections[RE.Sections.SectionB].LoadAddress; + uint64_t SectionABase = Sections[RE.Sections.SectionA].getLoadAddress(); + uint64_t SectionBBase = Sections[RE.Sections.SectionB].getLoadAddress(); assert((Value == SectionABase || Value == SectionBBase) && "Unexpected SECTDIFF relocation value."); Value = SectionABase - SectionBBase + RE.Addend; @@ -139,7 +138,7 @@ private: bool IsPCRel = Obj.getAnyRelocationPCRel(RE); unsigned Size = Obj.getAnyRelocationLength(RE); uint64_t Offset = RelI->getOffset(); - uint8_t *LocalAddress = Section.Address + Offset; + uint8_t *LocalAddress = Section.getAddressWithOffset(Offset); unsigned NumBytes = 1 << Size; uint64_t Addend = readBytesUnaligned(LocalAddress, NumBytes); @@ -183,41 +182,6 @@ private: return ++RelI; } - relocation_iterator processI386ScatteredVANILLA( - unsigned SectionID, relocation_iterator RelI, - const ObjectFile &BaseObjT, - RuntimeDyldMachO::ObjSectionToIDMap &ObjSectionToID) { - const MachOObjectFile &Obj = - static_cast<const MachOObjectFile&>(BaseObjT); - MachO::any_relocation_info RE = - Obj.getRelocation(RelI->getRawDataRefImpl()); - - SectionEntry &Section = Sections[SectionID]; - uint32_t RelocType = Obj.getAnyRelocationType(RE); - bool IsPCRel = Obj.getAnyRelocationPCRel(RE); - unsigned Size = Obj.getAnyRelocationLength(RE); - uint64_t Offset = RelI->getOffset(); - uint8_t *LocalAddress = Section.Address + Offset; - unsigned NumBytes = 1 << Size; - int64_t Addend = readBytesUnaligned(LocalAddress, NumBytes); - - unsigned SymbolBaseAddr = Obj.getScatteredRelocationValue(RE); - section_iterator TargetSI = getSectionByAddress(Obj, SymbolBaseAddr); - assert(TargetSI != Obj.section_end() && "Can't find section for symbol"); - uint64_t SectionBaseAddr = TargetSI->getAddress(); - SectionRef TargetSection = *TargetSI; - bool IsCode = TargetSection.isText(); - uint32_t TargetSectionID = - findOrEmitSection(Obj, TargetSection, IsCode, ObjSectionToID); - - Addend -= SectionBaseAddr; - RelocationEntry R(SectionID, Offset, RelocType, Addend, IsPCRel, Size); - - addRelocationForSection(R, TargetSectionID); - - return ++RelI; - } - // Populate stubs in __jump_table section. void populateJumpTable(const MachOObjectFile &Obj, const SectionRef &JTSection, unsigned JTSectionID) { diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h index dd56e72..2242295 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h @@ -39,6 +39,10 @@ public: static_cast<const MachOObjectFile &>(BaseObjT); MachO::any_relocation_info RelInfo = Obj.getRelocation(RelI->getRawDataRefImpl()); + uint32_t RelType = Obj.getAnyRelocationType(RelInfo); + + if (RelType == MachO::X86_64_RELOC_SUBTRACTOR) + return processSubtractRelocation(SectionID, RelI, Obj, ObjSectionToID); assert(!Obj.isRelocationScattered(RelInfo) && "Scattered relocations not supported on X86_64"); @@ -69,14 +73,14 @@ public: void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override { DEBUG(dumpRelocationToResolve(RE, Value)); const SectionEntry &Section = Sections[RE.SectionID]; - uint8_t *LocalAddress = Section.Address + RE.Offset; + uint8_t *LocalAddress = Section.getAddressWithOffset(RE.Offset); // If the relocation is PC-relative, the value to be encoded is the // pointer difference. if (RE.IsPCRel) { // FIXME: It seems this value needs to be adjusted by 4 for an effective // PC address. Is that expected? Only for branches, perhaps? - uint64_t FinalAddress = Section.LoadAddress + RE.Offset; + uint64_t FinalAddress = Section.getLoadAddressWithOffset(RE.Offset); Value -= FinalAddress + 4; } @@ -91,9 +95,17 @@ public: case MachO::X86_64_RELOC_BRANCH: writeBytesUnaligned(Value + RE.Addend, LocalAddress, 1 << RE.Size); break; + case MachO::X86_64_RELOC_SUBTRACTOR: { + uint64_t SectionABase = Sections[RE.Sections.SectionA].getLoadAddress(); + uint64_t SectionBBase = Sections[RE.Sections.SectionB].getLoadAddress(); + assert((Value == SectionABase || Value == SectionBBase) && + "Unexpected SUBTRACTOR relocation value."); + Value = SectionABase - SectionBBase + RE.Addend; + writeBytesUnaligned(Value, LocalAddress, 1 << RE.Size); + break; + } case MachO::X86_64_RELOC_GOT_LOAD: case MachO::X86_64_RELOC_GOT: - case MachO::X86_64_RELOC_SUBTRACTOR: case MachO::X86_64_RELOC_TLV: Error("Relocation type not implemented yet!"); } @@ -112,24 +124,65 @@ private: RuntimeDyldMachO::StubMap::const_iterator i = Stubs.find(Value); uint8_t *Addr; if (i != Stubs.end()) { - Addr = Section.Address + i->second; + Addr = Section.getAddressWithOffset(i->second); } else { - Stubs[Value] = Section.StubOffset; - uint8_t *GOTEntry = Section.Address + Section.StubOffset; - RelocationEntry GOTRE(RE.SectionID, Section.StubOffset, + Stubs[Value] = Section.getStubOffset(); + uint8_t *GOTEntry = Section.getAddressWithOffset(Section.getStubOffset()); + RelocationEntry GOTRE(RE.SectionID, Section.getStubOffset(), MachO::X86_64_RELOC_UNSIGNED, Value.Offset, false, 3); if (Value.SymbolName) addRelocationForSymbol(GOTRE, Value.SymbolName); else addRelocationForSection(GOTRE, Value.SectionID); - Section.StubOffset += 8; + Section.advanceStubOffset(8); Addr = GOTEntry; } RelocationEntry TargetRE(RE.SectionID, RE.Offset, MachO::X86_64_RELOC_UNSIGNED, RE.Addend, true, 2); resolveRelocation(TargetRE, (uint64_t)Addr); } + + relocation_iterator + processSubtractRelocation(unsigned SectionID, relocation_iterator RelI, + const ObjectFile &BaseObjT, + ObjSectionToIDMap &ObjSectionToID) { + const MachOObjectFile &Obj = + static_cast<const MachOObjectFile&>(BaseObjT); + MachO::any_relocation_info RE = + Obj.getRelocation(RelI->getRawDataRefImpl()); + + unsigned Size = Obj.getAnyRelocationLength(RE); + uint64_t Offset = RelI->getOffset(); + uint8_t *LocalAddress = Sections[SectionID].getAddressWithOffset(Offset); + unsigned NumBytes = 1 << Size; + + ErrorOr<StringRef> SubtrahendNameOrErr = RelI->getSymbol()->getName(); + if (auto EC = SubtrahendNameOrErr.getError()) + report_fatal_error(EC.message()); + auto SubtrahendI = GlobalSymbolTable.find(*SubtrahendNameOrErr); + unsigned SectionBID = SubtrahendI->second.getSectionID(); + uint64_t SectionBOffset = SubtrahendI->second.getOffset(); + int64_t Addend = + SignExtend64(readBytesUnaligned(LocalAddress, NumBytes), NumBytes * 8); + + ++RelI; + ErrorOr<StringRef> MinuendNameOrErr = RelI->getSymbol()->getName(); + if (auto EC = MinuendNameOrErr.getError()) + report_fatal_error(EC.message()); + auto MinuendI = GlobalSymbolTable.find(*MinuendNameOrErr); + unsigned SectionAID = MinuendI->second.getSectionID(); + uint64_t SectionAOffset = MinuendI->second.getOffset(); + + RelocationEntry R(SectionID, Offset, MachO::X86_64_RELOC_SUBTRACTOR, (uint64_t)Addend, + SectionAID, SectionAOffset, SectionBID, SectionBOffset, + false, Size); + + addRelocationForSection(R, SectionAID); + + return ++RelI; + } + }; } diff --git a/contrib/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp b/contrib/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp index 5986084..e2f2208 100644 --- a/contrib/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp +++ b/contrib/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp @@ -15,6 +15,7 @@ #include "llvm/Config/config.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" #include "llvm/Support/MathExtras.h" +#include "llvm/Support/Process.h" namespace llvm { @@ -48,16 +49,27 @@ uint8_t *SectionMemoryManager::allocateSection(MemoryGroup &MemGroup, // Look in the list of free memory regions and use a block there if one // is available. - for (int i = 0, e = MemGroup.FreeMem.size(); i != e; ++i) { - sys::MemoryBlock &MB = MemGroup.FreeMem[i]; - if (MB.size() >= RequiredSize) { - Addr = (uintptr_t)MB.base(); - uintptr_t EndOfBlock = Addr + MB.size(); + for (FreeMemBlock &FreeMB : MemGroup.FreeMem) { + if (FreeMB.Free.size() >= RequiredSize) { + Addr = (uintptr_t)FreeMB.Free.base(); + uintptr_t EndOfBlock = Addr + FreeMB.Free.size(); // Align the address. Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); - // Store cutted free memory block. - MemGroup.FreeMem[i] = sys::MemoryBlock((void*)(Addr + Size), - EndOfBlock - Addr - Size); + + if (FreeMB.PendingPrefixIndex == (unsigned)-1) { + // The part of the block we're giving out to the user is now pending + MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size)); + + // Remember this pending block, such that future allocations can just + // modify it rather than creating a new one + FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1; + } else { + sys::MemoryBlock &PendingMB = MemGroup.PendingMem[FreeMB.PendingPrefixIndex]; + PendingMB = sys::MemoryBlock(PendingMB.base(), Addr + Size - (uintptr_t)PendingMB.base()); + } + + // Remember how much free space is now left in this block + FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size); return (uint8_t*)Addr; } } @@ -85,6 +97,7 @@ uint8_t *SectionMemoryManager::allocateSection(MemoryGroup &MemGroup, // Save this address as the basis for our next request MemGroup.Near = MB; + // Remember that we allocated this memory MemGroup.AllocatedMem.push_back(MB); Addr = (uintptr_t)MB.base(); uintptr_t EndOfBlock = Addr + MB.size(); @@ -92,11 +105,18 @@ uint8_t *SectionMemoryManager::allocateSection(MemoryGroup &MemGroup, // Align the address. Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); + // The part of the block we're giving out to the user is now pending + MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size)); + // The allocateMappedMemory may allocate much more memory than we need. In // this case, we store the unused memory as a free memory block. unsigned FreeSize = EndOfBlock-Addr-Size; - if (FreeSize > 16) - MemGroup.FreeMem.push_back(sys::MemoryBlock((void*)(Addr + Size), FreeSize)); + if (FreeSize > 16) { + FreeMemBlock FreeMB; + FreeMB.Free = sys::MemoryBlock((void*)(Addr + Size), FreeSize); + FreeMB.PendingPrefixIndex = (unsigned)-1; + MemGroup.FreeMem.push_back(FreeMB); + } // Return aligned address return (uint8_t*)Addr; @@ -107,9 +127,6 @@ bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) // FIXME: Should in-progress permissions be reverted if an error occurs? std::error_code ec; - // Don't allow free memory blocks to be used after setting protection flags. - CodeMem.FreeMem.clear(); - // Make code memory executable. ec = applyMemoryGroupPermissions(CodeMem, sys::Memory::MF_READ | sys::Memory::MF_EXEC); @@ -143,36 +160,62 @@ bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) return false; } +static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) { + static const size_t PageSize = sys::Process::getPageSize(); + + size_t StartOverlap = + (PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize; + + size_t TrimmedSize = M.size(); + TrimmedSize -= StartOverlap; + TrimmedSize -= TrimmedSize % PageSize; + + sys::MemoryBlock Trimmed((void *)((uintptr_t)M.base() + StartOverlap), TrimmedSize); + + assert(((uintptr_t)Trimmed.base() % PageSize) == 0); + assert((Trimmed.size() % PageSize) == 0); + assert(M.base() <= Trimmed.base() && Trimmed.size() <= M.size()); + + return Trimmed; +} + + std::error_code SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup, unsigned Permissions) { - - for (int i = 0, e = MemGroup.AllocatedMem.size(); i != e; ++i) { - std::error_code ec; - ec = - sys::Memory::protectMappedMemory(MemGroup.AllocatedMem[i], Permissions); - if (ec) { - return ec; - } + for (sys::MemoryBlock &MB : MemGroup.PendingMem) + if (std::error_code EC = sys::Memory::protectMappedMemory(MB, Permissions)) + return EC; + + MemGroup.PendingMem.clear(); + + // Now go through free blocks and trim any of them that don't span the entire + // page because one of the pending blocks may have overlapped it. + for (FreeMemBlock &FreeMB : MemGroup.FreeMem) { + FreeMB.Free = trimBlockToPageSize(FreeMB.Free); + // We cleared the PendingMem list, so all these pointers are now invalid + FreeMB.PendingPrefixIndex = (unsigned)-1; } + // Remove all blocks which are now empty + MemGroup.FreeMem.erase( + std::remove_if(MemGroup.FreeMem.begin(), MemGroup.FreeMem.end(), + [](FreeMemBlock &FreeMB) { return FreeMB.Free.size() == 0; }), + MemGroup.FreeMem.end()); + return std::error_code(); } void SectionMemoryManager::invalidateInstructionCache() { - for (int i = 0, e = CodeMem.AllocatedMem.size(); i != e; ++i) - sys::Memory::InvalidateInstructionCache(CodeMem.AllocatedMem[i].base(), - CodeMem.AllocatedMem[i].size()); + for (sys::MemoryBlock &Block : CodeMem.PendingMem) + sys::Memory::InvalidateInstructionCache(Block.base(), Block.size()); } SectionMemoryManager::~SectionMemoryManager() { - for (unsigned i = 0, e = CodeMem.AllocatedMem.size(); i != e; ++i) - sys::Memory::releaseMappedMemory(CodeMem.AllocatedMem[i]); - for (unsigned i = 0, e = RWDataMem.AllocatedMem.size(); i != e; ++i) - sys::Memory::releaseMappedMemory(RWDataMem.AllocatedMem[i]); - for (unsigned i = 0, e = RODataMem.AllocatedMem.size(); i != e; ++i) - sys::Memory::releaseMappedMemory(RODataMem.AllocatedMem[i]); + for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) { + for (sys::MemoryBlock &Block : Group->AllocatedMem) + sys::Memory::releaseMappedMemory(Block); + } } } // namespace llvm - |
