diff options
Diffstat (limited to 'contrib/llvm/tools/clang/lib/CodeGen')
31 files changed, 1135 insertions, 388 deletions
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp b/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp index 8cece0d..f5edea7 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp @@ -201,8 +201,14 @@ static void addSanitizerCoveragePass(const PassManagerBuilder &Builder, static void addAddressSanitizerPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { - PM.add(createAddressSanitizerFunctionPass()); - PM.add(createAddressSanitizerModulePass()); + PM.add(createAddressSanitizerFunctionPass(/*CompileKernel*/false)); + PM.add(createAddressSanitizerModulePass(/*CompileKernel*/false)); +} + +static void addKernelAddressSanitizerPasses(const PassManagerBuilder &Builder, + legacy::PassManagerBase &PM) { + PM.add(createAddressSanitizerFunctionPass(/*CompileKernel*/true)); + PM.add(createAddressSanitizerModulePass(/*CompileKernel*/true)); } static void addMemorySanitizerPass(const PassManagerBuilder &Builder, @@ -283,7 +289,6 @@ void EmitAssemblyHelper::CreatePasses() { PMBuilder.SLPVectorize = CodeGenOpts.VectorizeSLP; PMBuilder.LoopVectorize = CodeGenOpts.VectorizeLoop; - PMBuilder.DisableTailCalls = CodeGenOpts.DisableTailCalls; PMBuilder.DisableUnitAtATime = !CodeGenOpts.UnitAtATime; PMBuilder.DisableUnrollLoops = !CodeGenOpts.UnrollLoops; PMBuilder.MergeFunctions = CodeGenOpts.MergeFunctions; @@ -329,6 +334,13 @@ void EmitAssemblyHelper::CreatePasses() { addAddressSanitizerPasses); } + if (LangOpts.Sanitize.has(SanitizerKind::KernelAddress)) { + PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, + addKernelAddressSanitizerPasses); + PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, + addKernelAddressSanitizerPasses); + } + if (LangOpts.Sanitize.has(SanitizerKind::Memory)) { PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, addMemorySanitizerPass); @@ -478,6 +490,9 @@ TargetMachine *EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) { llvm::TargetOptions Options; + if (!TargetOpts.Reciprocals.empty()) + Options.Reciprocals = TargetRecip(TargetOpts.Reciprocals); + Options.ThreadModel = llvm::StringSwitch<llvm::ThreadModel::Model>(CodeGenOpts.ThreadModel) .Case("posix", llvm::ThreadModel::POSIX) @@ -521,7 +536,6 @@ TargetMachine *EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) { Options.NoZerosInBSS = CodeGenOpts.NoZeroInitializedInBSS; Options.UnsafeFPMath = CodeGenOpts.UnsafeFPMath; Options.StackAlignmentOverride = CodeGenOpts.StackAlignment; - Options.DisableTailCalls = CodeGenOpts.DisableTailCalls; Options.TrapFuncName = CodeGenOpts.TrapFuncName; Options.PositionIndependentExecutable = LangOpts.PIELevel != 0; Options.FunctionSections = CodeGenOpts.FunctionSections; diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp index d86534d..a14daac 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp @@ -1877,7 +1877,7 @@ Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID, return EmitPPCBuiltinExpr(BuiltinID, E); case llvm::Triple::r600: case llvm::Triple::amdgcn: - return EmitR600BuiltinExpr(BuiltinID, E); + return EmitAMDGPUBuiltinExpr(BuiltinID, E); case llvm::Triple::systemz: return EmitSystemZBuiltinExpr(BuiltinID, E); default: @@ -3279,6 +3279,66 @@ Value *CodeGenFunction::GetValueForARMHint(unsigned BuiltinID) { } } +// Generates the IR for the read/write special register builtin, +// ValueType is the type of the value that is to be written or read, +// RegisterType is the type of the register being written to or read from. +static Value *EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, + const CallExpr *E, + llvm::Type *RegisterType, + llvm::Type *ValueType, bool IsRead) { + // write and register intrinsics only support 32 and 64 bit operations. + assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64)) + && "Unsupported size for register."); + + CodeGen::CGBuilderTy &Builder = CGF.Builder; + CodeGen::CodeGenModule &CGM = CGF.CGM; + LLVMContext &Context = CGM.getLLVMContext(); + + const Expr *SysRegStrExpr = E->getArg(0)->IgnoreParenCasts(); + StringRef SysReg = cast<StringLiteral>(SysRegStrExpr)->getString(); + + llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) }; + llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); + llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName); + + llvm::Type *Types[] = { RegisterType }; + + bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32); + assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64)) + && "Can't fit 64-bit value in 32-bit register"); + + if (IsRead) { + llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types); + llvm::Value *Call = Builder.CreateCall(F, Metadata); + + if (MixedTypes) + // Read into 64 bit register and then truncate result to 32 bit. + return Builder.CreateTrunc(Call, ValueType); + + if (ValueType->isPointerTy()) + // Have i32/i64 result (Call) but want to return a VoidPtrTy (i8*). + return Builder.CreateIntToPtr(Call, ValueType); + + return Call; + } + + llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types); + llvm::Value *ArgValue = CGF.EmitScalarExpr(E->getArg(1)); + if (MixedTypes) { + // Extend 32 bit write value to 64 bit to pass to write. + ArgValue = Builder.CreateZExt(ArgValue, RegisterType); + return Builder.CreateCall(F, { Metadata, ArgValue }); + } + + if (ValueType->isPointerTy()) { + // Have VoidPtrTy ArgValue but want to return an i32/i64. + ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType); + return Builder.CreateCall(F, { Metadata, ArgValue }); + } + + return Builder.CreateCall(F, { Metadata, ArgValue }); +} + Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E) { if (auto Hint = GetValueForARMHint(BuiltinID)) @@ -3491,6 +3551,37 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, } } + if (BuiltinID == ARM::BI__builtin_arm_rsr || + BuiltinID == ARM::BI__builtin_arm_rsr64 || + BuiltinID == ARM::BI__builtin_arm_rsrp || + BuiltinID == ARM::BI__builtin_arm_wsr || + BuiltinID == ARM::BI__builtin_arm_wsr64 || + BuiltinID == ARM::BI__builtin_arm_wsrp) { + + bool IsRead = BuiltinID == ARM::BI__builtin_arm_rsr || + BuiltinID == ARM::BI__builtin_arm_rsr64 || + BuiltinID == ARM::BI__builtin_arm_rsrp; + + bool IsPointerBuiltin = BuiltinID == ARM::BI__builtin_arm_rsrp || + BuiltinID == ARM::BI__builtin_arm_wsrp; + + bool Is64Bit = BuiltinID == ARM::BI__builtin_arm_rsr64 || + BuiltinID == ARM::BI__builtin_arm_wsr64; + + llvm::Type *ValueType; + llvm::Type *RegisterType; + if (IsPointerBuiltin) { + ValueType = VoidPtrTy; + RegisterType = Int32Ty; + } else if (Is64Bit) { + ValueType = RegisterType = Int64Ty; + } else { + ValueType = RegisterType = Int32Ty; + } + + return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType, IsRead); + } + // Find out if any arguments are required to be integer constant // expressions. unsigned ICEArguments = 0; @@ -4239,6 +4330,36 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, return Builder.CreateCall(F, {Arg0, Arg1}); } + if (BuiltinID == AArch64::BI__builtin_arm_rsr || + BuiltinID == AArch64::BI__builtin_arm_rsr64 || + BuiltinID == AArch64::BI__builtin_arm_rsrp || + BuiltinID == AArch64::BI__builtin_arm_wsr || + BuiltinID == AArch64::BI__builtin_arm_wsr64 || + BuiltinID == AArch64::BI__builtin_arm_wsrp) { + + bool IsRead = BuiltinID == AArch64::BI__builtin_arm_rsr || + BuiltinID == AArch64::BI__builtin_arm_rsr64 || + BuiltinID == AArch64::BI__builtin_arm_rsrp; + + bool IsPointerBuiltin = BuiltinID == AArch64::BI__builtin_arm_rsrp || + BuiltinID == AArch64::BI__builtin_arm_wsrp; + + bool Is64Bit = BuiltinID != AArch64::BI__builtin_arm_rsr && + BuiltinID != AArch64::BI__builtin_arm_wsr; + + llvm::Type *ValueType; + llvm::Type *RegisterType = Int64Ty; + if (IsPointerBuiltin) { + ValueType = VoidPtrTy; + } else if (Is64Bit) { + ValueType = Int64Ty; + } else { + ValueType = Int32Ty; + } + + return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType, IsRead); + } + // Find out if any arguments are required to be integer constant // expressions. unsigned ICEArguments = 0; @@ -6427,11 +6548,11 @@ static Value *emitFPIntBuiltin(CodeGenFunction &CGF, return CGF.Builder.CreateCall(F, {Src0, Src1}); } -Value *CodeGenFunction::EmitR600BuiltinExpr(unsigned BuiltinID, - const CallExpr *E) { +Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, + const CallExpr *E) { switch (BuiltinID) { - case R600::BI__builtin_amdgpu_div_scale: - case R600::BI__builtin_amdgpu_div_scalef: { + case AMDGPU::BI__builtin_amdgpu_div_scale: + case AMDGPU::BI__builtin_amdgpu_div_scalef: { // Translate from the intrinsics's struct return to the builtin's out // argument. @@ -6458,8 +6579,8 @@ Value *CodeGenFunction::EmitR600BuiltinExpr(unsigned BuiltinID, FlagStore->setAlignment(FlagOutPtr.second); return Result; } - case R600::BI__builtin_amdgpu_div_fmas: - case R600::BI__builtin_amdgpu_div_fmasf: { + case AMDGPU::BI__builtin_amdgpu_div_fmas: + case AMDGPU::BI__builtin_amdgpu_div_fmasf: { llvm::Value *Src0 = EmitScalarExpr(E->getArg(0)); llvm::Value *Src1 = EmitScalarExpr(E->getArg(1)); llvm::Value *Src2 = EmitScalarExpr(E->getArg(2)); @@ -6470,26 +6591,26 @@ Value *CodeGenFunction::EmitR600BuiltinExpr(unsigned BuiltinID, llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3); return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool}); } - case R600::BI__builtin_amdgpu_div_fixup: - case R600::BI__builtin_amdgpu_div_fixupf: + case AMDGPU::BI__builtin_amdgpu_div_fixup: + case AMDGPU::BI__builtin_amdgpu_div_fixupf: return emitTernaryFPBuiltin(*this, E, Intrinsic::AMDGPU_div_fixup); - case R600::BI__builtin_amdgpu_trig_preop: - case R600::BI__builtin_amdgpu_trig_preopf: + case AMDGPU::BI__builtin_amdgpu_trig_preop: + case AMDGPU::BI__builtin_amdgpu_trig_preopf: return emitFPIntBuiltin(*this, E, Intrinsic::AMDGPU_trig_preop); - case R600::BI__builtin_amdgpu_rcp: - case R600::BI__builtin_amdgpu_rcpf: + case AMDGPU::BI__builtin_amdgpu_rcp: + case AMDGPU::BI__builtin_amdgpu_rcpf: return emitUnaryFPBuiltin(*this, E, Intrinsic::AMDGPU_rcp); - case R600::BI__builtin_amdgpu_rsq: - case R600::BI__builtin_amdgpu_rsqf: + case AMDGPU::BI__builtin_amdgpu_rsq: + case AMDGPU::BI__builtin_amdgpu_rsqf: return emitUnaryFPBuiltin(*this, E, Intrinsic::AMDGPU_rsq); - case R600::BI__builtin_amdgpu_rsq_clamped: - case R600::BI__builtin_amdgpu_rsq_clampedf: + case AMDGPU::BI__builtin_amdgpu_rsq_clamped: + case AMDGPU::BI__builtin_amdgpu_rsq_clampedf: return emitUnaryFPBuiltin(*this, E, Intrinsic::AMDGPU_rsq_clamped); - case R600::BI__builtin_amdgpu_ldexp: - case R600::BI__builtin_amdgpu_ldexpf: + case AMDGPU::BI__builtin_amdgpu_ldexp: + case AMDGPU::BI__builtin_amdgpu_ldexpf: return emitFPIntBuiltin(*this, E, Intrinsic::AMDGPU_ldexp); - case R600::BI__builtin_amdgpu_class: - case R600::BI__builtin_amdgpu_classf: + case AMDGPU::BI__builtin_amdgpu_class: + case AMDGPU::BI__builtin_amdgpu_classf: return emitFPIntBuiltin(*this, E, Intrinsic::AMDGPU_class); default: return nullptr; diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.h b/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.h index 2c73921..b6b4ee6 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.h @@ -366,7 +366,8 @@ public: virtual llvm::Value *getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD, llvm::Value *This, - llvm::Type *Ty) = 0; + llvm::Type *Ty, + SourceLocation Loc) = 0; /// Emit the ABI-specific virtual destructor call. virtual llvm::Value * diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp index e77539c..58ef171 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp @@ -1465,6 +1465,8 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI, FuncAttrs.addAttribute("no-frame-pointer-elim-non-leaf"); } + FuncAttrs.addAttribute("disable-tail-calls", + llvm::toStringRef(CodeGenOpts.DisableTailCalls)); FuncAttrs.addAttribute("less-precise-fpmad", llvm::toStringRef(CodeGenOpts.LessPreciseFPMAD)); FuncAttrs.addAttribute("no-infs-fp-math", @@ -1481,24 +1483,62 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI, if (!CodeGenOpts.StackRealignment) FuncAttrs.addAttribute("no-realign-stack"); - // Add target-cpu and target-features work if they differ from the defaults. - std::string &CPU = getTarget().getTargetOpts().CPU; - if (CPU != "") - FuncAttrs.addAttribute("target-cpu", CPU); + // Add target-cpu and target-features attributes to functions. If + // we have a decl for the function and it has a target attribute then + // parse that and add it to the feature set. + StringRef TargetCPU = getTarget().getTargetOpts().CPU; // TODO: Features gets us the features on the command line including // feature dependencies. For canonicalization purposes we might want to - // avoid putting features in the target-features set if we know it'll be one - // of the default features in the backend, e.g. corei7-avx and +avx or figure - // out non-explicit dependencies. - std::vector<std::string> &Features = getTarget().getTargetOpts().Features; + // avoid putting features in the target-features set if we know it'll be + // one of the default features in the backend, e.g. corei7-avx and +avx or + // figure out non-explicit dependencies. + std::vector<std::string> Features(getTarget().getTargetOpts().Features); + + // TODO: The target attribute complicates this further by allowing multiple + // additional features to be tacked on to the feature string for a + // particular function. For now we simply append to the set of features and + // let backend resolution fix them up. + const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl); + if (FD) { + if (const TargetAttr *TD = FD->getAttr<TargetAttr>()) { + StringRef FeaturesStr = TD->getFeatures(); + SmallVector<StringRef, 1> AttrFeatures; + FeaturesStr.split(AttrFeatures, ","); + + // Grab the various features and prepend a "+" to turn on the feature to + // the backend and add them to our existing set of Features. + for (auto &Feature : AttrFeatures) { + // While we're here iterating check for a different target cpu. + if (Feature.startswith("arch=")) + TargetCPU = Feature.split("=").second; + else if (Feature.startswith("tune=")) + // We don't support cpu tuning this way currently. + ; + else if (Feature.startswith("fpmath=")) + // TODO: Support the fpmath option this way. It will require checking + // overall feature validity for the function with the rest of the + // attributes on the function. + ; + else if (Feature.startswith("mno-")) + Features.push_back("-" + Feature.split("-").second.str()); + else + Features.push_back("+" + Feature.str()); + } + } + } + + // Now add the target-cpu and target-features to the function. + if (TargetCPU != "") + FuncAttrs.addAttribute("target-cpu", TargetCPU); if (!Features.empty()) { - std::stringstream S; + std::stringstream TargetFeatures; std::copy(Features.begin(), Features.end(), - std::ostream_iterator<std::string>(S, ",")); + std::ostream_iterator<std::string>(TargetFeatures, ",")); + // The drop_back gets rid of the trailing space. FuncAttrs.addAttribute("target-features", - StringRef(S.str()).drop_back(1)); + StringRef(TargetFeatures.str()).drop_back(1)); } } @@ -2231,11 +2271,10 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) { if (Intrinsic->getIntrinsicID() == llvm::Intrinsic::lifetime_end) { const llvm::Value *CastAddr = Intrinsic->getArgOperand(1); ++II; - if (isa<llvm::BitCastInst>(&*II)) { - if (CastAddr == &*II) { - continue; - } - } + if (II == IE) + break; + if (isa<llvm::BitCastInst>(&*II) && (CastAddr == &*II)) + continue; } } I = &*II; @@ -2571,7 +2610,7 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args, LValue srcLV; // Make an optimistic effort to emit the address as an l-value. - // This can fail if the the argument expression is more complicated. + // This can fail if the argument expression is more complicated. if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(CRE->getSubExpr())) { srcLV = CGF.EmitLValue(lvExpr); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp index cd75da2..1d2b787 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp @@ -370,25 +370,25 @@ namespace { /// A visitor which checks whether an initializer uses 'this' in a /// way which requires the vtable to be properly set. - struct DynamicThisUseChecker : EvaluatedExprVisitor<DynamicThisUseChecker> { - typedef EvaluatedExprVisitor<DynamicThisUseChecker> super; + struct DynamicThisUseChecker : ConstEvaluatedExprVisitor<DynamicThisUseChecker> { + typedef ConstEvaluatedExprVisitor<DynamicThisUseChecker> super; bool UsesThis; - DynamicThisUseChecker(ASTContext &C) : super(C), UsesThis(false) {} + DynamicThisUseChecker(const ASTContext &C) : super(C), UsesThis(false) {} // Black-list all explicit and implicit references to 'this'. // // Do we need to worry about external references to 'this' derived // from arbitrary code? If so, then anything which runs arbitrary // external code might potentially access the vtable. - void VisitCXXThisExpr(CXXThisExpr *E) { UsesThis = true; } + void VisitCXXThisExpr(const CXXThisExpr *E) { UsesThis = true; } }; } static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) { DynamicThisUseChecker Checker(C); - Checker.Visit(const_cast<Expr*>(Init)); + Checker.Visit(Init); return Checker.UsesThis; } @@ -2134,17 +2134,21 @@ LeastDerivedClassWithSameLayout(const CXXRecordDecl *RD) { } void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXMethodDecl *MD, - llvm::Value *VTable) { + llvm::Value *VTable, + CFITypeCheckKind TCK, + SourceLocation Loc) { const CXXRecordDecl *ClassDecl = MD->getParent(); if (!SanOpts.has(SanitizerKind::CFICastStrict)) ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl); - EmitVTablePtrCheck(ClassDecl, VTable); + EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc); } void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, llvm::Value *Derived, - bool MayBeNull) { + bool MayBeNull, + CFITypeCheckKind TCK, + SourceLocation Loc) { if (!getLangOpts().CPlusPlus) return; @@ -2184,7 +2188,7 @@ void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, } llvm::Value *VTable = GetVTablePtr(Derived, Int8PtrTy); - EmitVTablePtrCheck(ClassDecl, VTable); + EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc); if (MayBeNull) { Builder.CreateBr(ContBlock); @@ -2193,11 +2197,15 @@ void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, } void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD, - llvm::Value *VTable) { + llvm::Value *VTable, + CFITypeCheckKind TCK, + SourceLocation Loc) { // FIXME: Add blacklisting scheme. if (RD->isInStdNamespace()) return; + SanitizerScope SanScope(this); + std::string OutName; llvm::raw_string_ostream Out(OutName); CGM.getCXXABI().getMangleContext().mangleCXXVTableBitSet(RD, Out); @@ -2205,20 +2213,34 @@ void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD, llvm::Value *BitSetName = llvm::MetadataAsValue::get( getLLVMContext(), llvm::MDString::get(getLLVMContext(), Out.str())); - llvm::Value *BitSetTest = Builder.CreateCall( - CGM.getIntrinsic(llvm::Intrinsic::bitset_test), - {Builder.CreateBitCast(VTable, CGM.Int8PtrTy), BitSetName}); + llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy); + llvm::Value *BitSetTest = + Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::bitset_test), + {CastedVTable, BitSetName}); - llvm::BasicBlock *ContBlock = createBasicBlock("vtable.check.cont"); - llvm::BasicBlock *TrapBlock = createBasicBlock("vtable.check.trap"); - - Builder.CreateCondBr(BitSetTest, ContBlock, TrapBlock); - - EmitBlock(TrapBlock); - Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::trap), {}); - Builder.CreateUnreachable(); + SanitizerMask M; + switch (TCK) { + case CFITCK_VCall: + M = SanitizerKind::CFIVCall; + break; + case CFITCK_NVCall: + M = SanitizerKind::CFINVCall; + break; + case CFITCK_DerivedCast: + M = SanitizerKind::CFIDerivedCast; + break; + case CFITCK_UnrelatedCast: + M = SanitizerKind::CFIUnrelatedCast; + break; + } - EmitBlock(ContBlock); + llvm::Constant *StaticData[] = { + EmitCheckSourceLocation(Loc), + EmitCheckTypeDescriptor(QualType(RD->getTypeForDecl(), 0)), + llvm::ConstantInt::get(Int8Ty, TCK), + }; + EmitCheck(std::make_pair(BitSetTest, M), "cfi_bad_type", StaticData, + CastedVTable); } // FIXME: Ideally Expr::IgnoreParenNoopCasts should do this, but it doesn't do diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.cpp index 48458db..a20e39f 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.cpp @@ -590,18 +590,29 @@ llvm::DIType *CGDebugInfo::CreateType(const PointerType *Ty, Ty->getPointeeType(), Unit); } +/// \return whether a C++ mangling exists for the type defined by TD. +static bool hasCXXMangling(const TagDecl *TD, llvm::DICompileUnit *TheCU) { + switch (TheCU->getSourceLanguage()) { + case llvm::dwarf::DW_LANG_C_plus_plus: + return true; + case llvm::dwarf::DW_LANG_ObjC_plus_plus: + return isa<CXXRecordDecl>(TD) || isa<EnumDecl>(TD); + default: + return false; + } +} + /// In C++ mode, types have linkage, so we can rely on the ODR and /// on their mangled names, if they're external. static SmallString<256> getUniqueTagTypeName(const TagType *Ty, CodeGenModule &CGM, llvm::DICompileUnit *TheCU) { SmallString<256> FullName; - // FIXME: ODR should apply to ObjC++ exactly the same wasy it does to C++. - // For now, only apply ODR with C++. const TagDecl *TD = Ty->getDecl(); - if (TheCU->getSourceLanguage() != llvm::dwarf::DW_LANG_C_plus_plus || - !TD->isExternallyVisible()) + + if (!hasCXXMangling(TD, TheCU) || !TD->isExternallyVisible()) return FullName; + // Microsoft Mangler does not have support for mangleCXXRTTIName yet. if (CGM.getTarget().getCXXABI().isMicrosoft()) return FullName; diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp index 579a041..07dbce4 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp @@ -864,20 +864,17 @@ llvm::Value *CodeGenFunction::EmitLifetimeStart(uint64_t Size, return nullptr; llvm::Value *SizeV = llvm::ConstantInt::get(Int64Ty, Size); - llvm::Value *Args[] = { - SizeV, - new llvm::BitCastInst(Addr, Int8PtrTy, "", Builder.GetInsertBlock())}; - llvm::CallInst *C = llvm::CallInst::Create(CGM.getLLVMLifetimeStartFn(), Args, - "", Builder.GetInsertBlock()); + Addr = Builder.CreateBitCast(Addr, Int8PtrTy); + llvm::CallInst *C = + Builder.CreateCall(CGM.getLLVMLifetimeStartFn(), {SizeV, Addr}); C->setDoesNotThrow(); return SizeV; } void CodeGenFunction::EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr) { - llvm::Value *Args[] = {Size, new llvm::BitCastInst(Addr, Int8PtrTy, "", - Builder.GetInsertBlock())}; - llvm::CallInst *C = llvm::CallInst::Create(CGM.getLLVMLifetimeEndFn(), Args, - "", Builder.GetInsertBlock()); + Addr = Builder.CreateBitCast(Addr, Int8PtrTy); + llvm::CallInst *C = + Builder.CreateCall(CGM.getLLVMLifetimeEndFn(), {Size, Addr}); C->setDoesNotThrow(); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGDeclCXX.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGDeclCXX.cpp index 06d157b..00d6d5c 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGDeclCXX.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGDeclCXX.cpp @@ -267,12 +267,15 @@ llvm::Function *CodeGenModule::CreateGlobalInitOrDestructFunction( Fn->setDoesNotThrow(); if (!isInSanitizerBlacklist(Fn, Loc)) { - if (getLangOpts().Sanitize.has(SanitizerKind::Address)) + if (getLangOpts().Sanitize.hasOneOf(SanitizerKind::Address | + SanitizerKind::KernelAddress)) Fn->addFnAttr(llvm::Attribute::SanitizeAddress); if (getLangOpts().Sanitize.has(SanitizerKind::Thread)) Fn->addFnAttr(llvm::Attribute::SanitizeThread); if (getLangOpts().Sanitize.has(SanitizerKind::Memory)) Fn->addFnAttr(llvm::Attribute::SanitizeMemory); + if (getLangOpts().Sanitize.has(SanitizerKind::SafeStack)) + Fn->addFnAttr(llvm::Attribute::SafeStack); } return Fn; @@ -421,6 +424,7 @@ CodeGenModule::EmitCXXGlobalInitFunc() { CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, LocalCXXGlobalInits); AddGlobalCtor(Fn, Priority); } + PrioritizedCXXGlobalInits.clear(); } SmallString<128> FileName; @@ -448,7 +452,6 @@ CodeGenModule::EmitCXXGlobalInitFunc() { AddGlobalCtor(Fn); CXXGlobalInits.clear(); - PrioritizedCXXGlobalInits.clear(); } void CodeGenModule::EmitCXXGlobalDtorFunc() { diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGException.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGException.cpp index d9a3f0b..4c85017 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGException.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGException.cpp @@ -698,13 +698,15 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() { const EHPersonality &personality = EHPersonality::get(*this); + if (!CurFn->hasPersonalityFn()) + CurFn->setPersonalityFn(getOpaquePersonalityFn(CGM, personality)); + // Create and configure the landing pad. llvm::BasicBlock *lpad = createBasicBlock("lpad"); EmitBlock(lpad); - llvm::LandingPadInst *LPadInst = - Builder.CreateLandingPad(llvm::StructType::get(Int8PtrTy, Int32Ty, nullptr), - getOpaquePersonalityFn(CGM, personality), 0); + llvm::LandingPadInst *LPadInst = Builder.CreateLandingPad( + llvm::StructType::get(Int8PtrTy, Int32Ty, nullptr), 0); llvm::Value *LPadExn = Builder.CreateExtractValue(LPadInst, 0); Builder.CreateStore(LPadExn, getExceptionSlot()); @@ -1193,9 +1195,12 @@ llvm::BasicBlock *CodeGenFunction::getTerminateLandingPad() { // Tell the backend that this is a landing pad. const EHPersonality &Personality = EHPersonality::get(*this); - llvm::LandingPadInst *LPadInst = - Builder.CreateLandingPad(llvm::StructType::get(Int8PtrTy, Int32Ty, nullptr), - getOpaquePersonalityFn(CGM, Personality), 0); + + if (!CurFn->hasPersonalityFn()) + CurFn->setPersonalityFn(getOpaquePersonalityFn(CGM, Personality)); + + llvm::LandingPadInst *LPadInst = Builder.CreateLandingPad( + llvm::StructType::get(Int8PtrTy, Int32Ty, nullptr), 0); LPadInst->addClause(getCatchAllValue(*this)); llvm::Value *Exn = 0; diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp index 1ed45a3..1a76afa 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp @@ -2300,15 +2300,24 @@ void CodeGenFunction::EmitCheck( llvm::Value *FatalCond = nullptr; llvm::Value *RecoverableCond = nullptr; + llvm::Value *TrapCond = nullptr; for (int i = 0, n = Checked.size(); i < n; ++i) { llvm::Value *Check = Checked[i].first; + // -fsanitize-trap= overrides -fsanitize-recover=. llvm::Value *&Cond = - CGM.getCodeGenOpts().SanitizeRecover.has(Checked[i].second) - ? RecoverableCond - : FatalCond; + CGM.getCodeGenOpts().SanitizeTrap.has(Checked[i].second) + ? TrapCond + : CGM.getCodeGenOpts().SanitizeRecover.has(Checked[i].second) + ? RecoverableCond + : FatalCond; Cond = Cond ? Builder.CreateAnd(Cond, Check) : Check; } + if (TrapCond) + EmitTrapCheck(TrapCond); + if (!FatalCond && !RecoverableCond) + return; + llvm::Value *JointCond; if (FatalCond && RecoverableCond) JointCond = Builder.CreateAnd(FatalCond, RecoverableCond); @@ -2326,15 +2335,6 @@ void CodeGenFunction::EmitCheck( } #endif - if (CGM.getCodeGenOpts().SanitizeUndefinedTrapOnError) { - assert(RecoverKind != CheckRecoverableKind::AlwaysRecoverable && - "Runtime call required for AlwaysRecoverable kind!"); - // Assume that -fsanitize-undefined-trap-on-error overrides - // -fsanitize-recover= options, as we can only print meaningful error - // message and recover if we have a runtime support. - return EmitTrapCheck(JointCond); - } - llvm::BasicBlock *Cont = createBasicBlock("cont"); llvm::BasicBlock *Handlers = createBasicBlock("handler." + CheckName); llvm::Instruction *Branch = Builder.CreateCondBr(JointCond, Cont, Handlers); @@ -3035,7 +3035,8 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { Derived, E->getType()); if (SanOpts.has(SanitizerKind::CFIDerivedCast)) - EmitVTablePtrCheckForCast(E->getType(), Derived, /*MayBeNull=*/false); + EmitVTablePtrCheckForCast(E->getType(), Derived, /*MayBeNull=*/false, + CFITCK_DerivedCast, E->getLocStart()); return MakeAddrLValue(Derived, E->getType()); } @@ -3048,7 +3049,8 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { ConvertType(CE->getTypeAsWritten())); if (SanOpts.has(SanitizerKind::CFIUnrelatedCast)) - EmitVTablePtrCheckForCast(E->getType(), V, /*MayBeNull=*/false); + EmitVTablePtrCheckForCast(E->getType(), V, /*MayBeNull=*/false, + CFITCK_UnrelatedCast, E->getLocStart()); return MakeAddrLValue(V, E->getType()); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprAgg.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprAgg.cpp index 8b1bc69..883b76b 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprAgg.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprAgg.cpp @@ -160,10 +160,12 @@ public: EmitAggLoadOfLValue(E); } + void VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E); void VisitAbstractConditionalOperator(const AbstractConditionalOperator *CO); void VisitChooseExpr(const ChooseExpr *CE); void VisitInitListExpr(InitListExpr *E); void VisitImplicitValueInitExpr(ImplicitValueInitExpr *E); + void VisitNoInitExpr(NoInitExpr *E) { } // Do nothing. void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) { Visit(DAE->getExpr()); } @@ -1056,6 +1058,9 @@ AggExprEmitter::EmitInitializationToLValue(Expr *E, LValue LV) { return; } else if (isa<ImplicitValueInitExpr>(E) || isa<CXXScalarValueInitExpr>(E)) { return EmitNullInitializationToLValue(LV); + } else if (isa<NoInitExpr>(E)) { + // Do nothing. + return; } else if (type->isReferenceType()) { RValue RV = CGF.EmitReferenceBindingToExpr(E); return CGF.EmitStoreThroughLValue(RV, LV); @@ -1276,6 +1281,15 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) { cleanupDominator->eraseFromParent(); } +void AggExprEmitter::VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E) { + AggValueSlot Dest = EnsureSlot(E->getType()); + + LValue DestLV = CGF.MakeAddrLValue(Dest.getAddr(), E->getType(), + Dest.getAlignment()); + EmitInitializationToLValue(E->getBase(), DestLV); + VisitInitListExpr(E->getUpdater()); +} + //===----------------------------------------------------------------------===// // Entry Points into this File //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp index 13dfbb3..f0f706d 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp @@ -254,12 +254,13 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( if (const CXXConstructorDecl *Ctor = dyn_cast<CXXConstructorDecl>(MD)) { Callee = CGM.GetAddrOfFunction(GlobalDecl(Ctor, Ctor_Complete), Ty); } else if (UseVirtualCall) { - Callee = CGM.getCXXABI().getVirtualFunctionPointer(*this, MD, This, Ty); + Callee = CGM.getCXXABI().getVirtualFunctionPointer(*this, MD, This, Ty, + CE->getLocStart()); } else { if (SanOpts.has(SanitizerKind::CFINVCall) && MD->getParent()->isDynamicClass()) { llvm::Value *VTable = GetVTablePtr(This, Int8PtrTy); - EmitVTablePtrCheckForCall(MD, VTable); + EmitVTablePtrCheckForCall(MD, VTable, CFITCK_NVCall, CE->getLocStart()); } if (getLangOpts().AppleKext && MD->isVirtual() && HasQualifier) @@ -958,6 +959,25 @@ void CodeGenFunction::EmitNewArrayInitializer( if (ILE->getNumInits() == 0 && TryMemsetInitialization()) return; + // If we have a struct whose every field is value-initialized, we can + // usually use memset. + if (auto *ILE = dyn_cast<InitListExpr>(Init)) { + if (const RecordType *RType = ILE->getType()->getAs<RecordType>()) { + if (RType->getDecl()->isStruct()) { + unsigned NumFields = 0; + for (auto *Field : RType->getDecl()->fields()) + if (!Field->isUnnamedBitfield()) + ++NumFields; + if (ILE->getNumInits() == NumFields) + for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i) + if (!isa<ImplicitValueInitExpr>(ILE->getInit(i))) + --NumFields; + if (ILE->getNumInits() == NumFields && TryMemsetInitialization()) + return; + } + } + } + // Create the loop blocks. llvm::BasicBlock *EntryBB = Builder.GetInsertBlock(); llvm::BasicBlock *LoopBB = createBasicBlock("new.loop"); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprConstant.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprConstant.cpp index b90b3ab..acfb9b6 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprConstant.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprConstant.cpp @@ -33,6 +33,7 @@ using namespace CodeGen; //===----------------------------------------------------------------------===// namespace { +class ConstExprEmitter; class ConstStructBuilder { CodeGenModule &CGM; CodeGenFunction *CGF; @@ -42,6 +43,10 @@ class ConstStructBuilder { CharUnits LLVMStructAlignment; SmallVector<llvm::Constant *, 32> Elements; public: + static llvm::Constant *BuildStruct(CodeGenModule &CGM, CodeGenFunction *CFG, + ConstExprEmitter *Emitter, + llvm::ConstantStruct *Base, + InitListExpr *Updater); static llvm::Constant *BuildStruct(CodeGenModule &CGM, CodeGenFunction *CGF, InitListExpr *ILE); static llvm::Constant *BuildStruct(CodeGenModule &CGM, CodeGenFunction *CGF, @@ -68,6 +73,8 @@ private: void ConvertStructToPacked(); bool Build(InitListExpr *ILE); + bool Build(ConstExprEmitter *Emitter, llvm::ConstantStruct *Base, + InitListExpr *Updater); void Build(const APValue &Val, const RecordDecl *RD, bool IsPrimaryBase, const CXXRecordDecl *VTableClass, CharUnits BaseOffset); llvm::Constant *Finalize(QualType Ty); @@ -547,6 +554,17 @@ llvm::Constant *ConstStructBuilder::Finalize(QualType Ty) { llvm::Constant *ConstStructBuilder::BuildStruct(CodeGenModule &CGM, CodeGenFunction *CGF, + ConstExprEmitter *Emitter, + llvm::ConstantStruct *Base, + InitListExpr *Updater) { + ConstStructBuilder Builder(CGM, CGF); + if (!Builder.Build(Emitter, Base, Updater)) + return nullptr; + return Builder.Finalize(Updater->getType()); +} + +llvm::Constant *ConstStructBuilder::BuildStruct(CodeGenModule &CGM, + CodeGenFunction *CGF, InitListExpr *ILE) { ConstStructBuilder Builder(CGM, CGF); @@ -818,6 +836,82 @@ public: return nullptr; } + llvm::Constant *EmitDesignatedInitUpdater(llvm::Constant *Base, + InitListExpr *Updater) { + QualType ExprType = Updater->getType(); + + if (ExprType->isArrayType()) { + llvm::ArrayType *AType = cast<llvm::ArrayType>(ConvertType(ExprType)); + llvm::Type *ElemType = AType->getElementType(); + + unsigned NumInitElements = Updater->getNumInits(); + unsigned NumElements = AType->getNumElements(); + + std::vector<llvm::Constant *> Elts; + Elts.reserve(NumElements); + + if (llvm::ConstantDataArray *DataArray = + dyn_cast<llvm::ConstantDataArray>(Base)) + for (unsigned i = 0; i != NumElements; ++i) + Elts.push_back(DataArray->getElementAsConstant(i)); + else if (llvm::ConstantArray *Array = + dyn_cast<llvm::ConstantArray>(Base)) + for (unsigned i = 0; i != NumElements; ++i) + Elts.push_back(Array->getOperand(i)); + else + return nullptr; // FIXME: other array types not implemented + + llvm::Constant *fillC = nullptr; + if (Expr *filler = Updater->getArrayFiller()) + if (!isa<NoInitExpr>(filler)) + fillC = CGM.EmitConstantExpr(filler, filler->getType(), CGF); + bool RewriteType = (fillC && fillC->getType() != ElemType); + + for (unsigned i = 0; i != NumElements; ++i) { + Expr *Init = nullptr; + if (i < NumInitElements) + Init = Updater->getInit(i); + + if (!Init && fillC) + Elts[i] = fillC; + else if (!Init || isa<NoInitExpr>(Init)) + ; // Do nothing. + else if (InitListExpr *ChildILE = dyn_cast<InitListExpr>(Init)) + Elts[i] = EmitDesignatedInitUpdater(Elts[i], ChildILE); + else + Elts[i] = CGM.EmitConstantExpr(Init, Init->getType(), CGF); + + if (!Elts[i]) + return nullptr; + RewriteType |= (Elts[i]->getType() != ElemType); + } + + if (RewriteType) { + std::vector<llvm::Type *> Types; + Types.reserve(NumElements); + for (unsigned i = 0; i != NumElements; ++i) + Types.push_back(Elts[i]->getType()); + llvm::StructType *SType = llvm::StructType::get(AType->getContext(), + Types, true); + return llvm::ConstantStruct::get(SType, Elts); + } + + return llvm::ConstantArray::get(AType, Elts); + } + + if (ExprType->isRecordType()) + return ConstStructBuilder::BuildStruct(CGM, CGF, this, + dyn_cast<llvm::ConstantStruct>(Base), Updater); + + return nullptr; + } + + llvm::Constant *VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E) { + return EmitDesignatedInitUpdater( + CGM.EmitConstantExpr(E->getBase(), E->getType(), CGF), + E->getUpdater()); + } + llvm::Constant *VisitCXXConstructExpr(CXXConstructExpr *E) { if (!E->getConstructor()->isTrivial()) return nullptr; @@ -1003,6 +1097,68 @@ public: } // end anonymous namespace. +bool ConstStructBuilder::Build(ConstExprEmitter *Emitter, + llvm::ConstantStruct *Base, + InitListExpr *Updater) { + assert(Base && "base expression should not be empty"); + + QualType ExprType = Updater->getType(); + RecordDecl *RD = ExprType->getAs<RecordType>()->getDecl(); + const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD); + const llvm::StructLayout *BaseLayout = CGM.getDataLayout().getStructLayout( + Base->getType()); + unsigned FieldNo = -1; + unsigned ElementNo = 0; + + for (FieldDecl *Field : RD->fields()) { + ++FieldNo; + + if (RD->isUnion() && Updater->getInitializedFieldInUnion() != Field) + continue; + + // Skip anonymous bitfields. + if (Field->isUnnamedBitfield()) + continue; + + llvm::Constant *EltInit = Base->getOperand(ElementNo); + + // Bail out if the type of the ConstantStruct does not have the same layout + // as the type of the InitListExpr. + if (CGM.getTypes().ConvertType(Field->getType()) != EltInit->getType() || + Layout.getFieldOffset(ElementNo) != + BaseLayout->getElementOffsetInBits(ElementNo)) + return false; + + // Get the initializer. If we encounter an empty field or a NoInitExpr, + // we use values from the base expression. + Expr *Init = nullptr; + if (ElementNo < Updater->getNumInits()) + Init = Updater->getInit(ElementNo); + + if (!Init || isa<NoInitExpr>(Init)) + ; // Do nothing. + else if (InitListExpr *ChildILE = dyn_cast<InitListExpr>(Init)) + EltInit = Emitter->EmitDesignatedInitUpdater(EltInit, ChildILE); + else + EltInit = CGM.EmitConstantExpr(Init, Field->getType(), CGF); + + ++ElementNo; + + if (!EltInit) + return false; + + if (!Field->isBitField()) + AppendField(Field, Layout.getFieldOffset(FieldNo), EltInit); + else if (llvm::ConstantInt *CI = dyn_cast<llvm::ConstantInt>(EltInit)) + AppendBitField(Field, Layout.getFieldOffset(FieldNo), CI); + else + // Initializing a bitfield with a non-trivial constant? + return false; + } + + return true; +} + llvm::Constant *CodeGenModule::EmitConstantInit(const VarDecl &D, CodeGenFunction *CGF) { // Make a quick check if variable can be default NULL initialized diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprScalar.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprScalar.cpp index 08c81c0..330a0df 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprScalar.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprScalar.cpp @@ -1386,7 +1386,9 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { if (CGF.SanOpts.has(SanitizerKind::CFIUnrelatedCast)) { if (auto PT = DestTy->getAs<PointerType>()) CGF.EmitVTablePtrCheckForCast(PT->getPointeeType(), Src, - /*MayBeNull=*/true); + /*MayBeNull=*/true, + CodeGenFunction::CFITCK_UnrelatedCast, + CE->getLocStart()); } return Builder.CreateBitCast(Src, DstTy); @@ -1420,7 +1422,9 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { if (CGF.SanOpts.has(SanitizerKind::CFIDerivedCast)) CGF.EmitVTablePtrCheckForCast(DestTy->getPointeeType(), Derived, - /*MayBeNull=*/true); + /*MayBeNull=*/true, + CodeGenFunction::CFITCK_DerivedCast, + CE->getLocStart()); return Derived; } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.cpp index 0675544..1163d63 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.cpp @@ -8,6 +8,8 @@ //===----------------------------------------------------------------------===// #include "CGLoopInfo.h" +#include "clang/AST/Attr.h" +#include "clang/Sema/LoopHint.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Constants.h" #include "llvm/IR/InstrTypes.h" @@ -76,7 +78,34 @@ LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs) LoopID = createMetadata(Header->getContext(), Attrs); } -void LoopInfoStack::push(BasicBlock *Header) { +void LoopInfoStack::push(BasicBlock *Header, + ArrayRef<const clang::Attr *> Attrs) { + for (const auto *Attr : Attrs) { + const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr); + + // Skip non loop hint attributes + if (!LH) + continue; + + LoopHintAttr::OptionType Option = LH->getOption(); + LoopHintAttr::LoopHintState State = LH->getState(); + switch (Option) { + case LoopHintAttr::Vectorize: + case LoopHintAttr::Interleave: + if (State == LoopHintAttr::AssumeSafety) { + // Apply "llvm.mem.parallel_loop_access" metadata to load/stores. + setParallel(true); + } + break; + case LoopHintAttr::VectorizeWidth: + case LoopHintAttr::InterleaveCount: + case LoopHintAttr::Unroll: + case LoopHintAttr::UnrollCount: + // Nothing to do here for these loop hints. + break; + } + } + Active.push_back(LoopInfo(Header, StagedAttrs)); // Clear the attributes so nested loops do not inherit them. StagedAttrs.clear(); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.h b/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.h index aee1621..2249937 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.h @@ -15,6 +15,7 @@ #ifndef LLVM_CLANG_LIB_CODEGEN_CGLOOPINFO_H #define LLVM_CLANG_LIB_CODEGEN_CGLOOPINFO_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/Value.h" @@ -27,6 +28,7 @@ class MDNode; } // end namespace llvm namespace clang { +class Attr; namespace CodeGen { /// \brief Attributes that may be specified on loops. @@ -86,7 +88,8 @@ public: /// \brief Begin a new structured loop. The set of staged attributes will be /// applied to the loop and then cleared. - void push(llvm::BasicBlock *Header); + void push(llvm::BasicBlock *Header, + llvm::ArrayRef<const Attr *> Attrs = llvm::None); /// \brief End the current loop. void pop(); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGObjC.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGObjC.cpp index ef9a92d..9981fcc 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGObjC.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGObjC.cpp @@ -2996,13 +2996,9 @@ CodeGenFunction::GenerateObjCAtomicGetterCopyHelperFunction( SmallVector<Expr*, 4> ConstructorArgs; ConstructorArgs.push_back(&SRC); - CXXConstructExpr::arg_iterator A = CXXConstExpr->arg_begin(); - ++A; - - for (CXXConstructExpr::arg_iterator AEnd = CXXConstExpr->arg_end(); - A != AEnd; ++A) - ConstructorArgs.push_back(*A); - + ConstructorArgs.append(std::next(CXXConstExpr->arg_begin()), + CXXConstExpr->arg_end()); + CXXConstructExpr *TheCXXConstructExpr = CXXConstructExpr::Create(C, Ty, SourceLocation(), CXXConstExpr->getConstructor(), diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp index 1238acc..3161af3 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -741,7 +741,7 @@ CGOpenMPRuntime::createRuntimeFunction(OpenMPRTLFunction Function) { break; } case OMPRTL__kmpc_end_ordered: { - // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid); + // Build void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid); llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; llvm::FunctionType *FnTy = llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); @@ -756,6 +756,31 @@ CGOpenMPRuntime::createRuntimeFunction(OpenMPRTLFunction Function) { RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_omp_taskwait"); break; } + case OMPRTL__kmpc_taskgroup: { + // Build void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_taskgroup"); + break; + } + case OMPRTL__kmpc_end_taskgroup: { + // Build void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_taskgroup"); + break; + } + case OMPRTL__kmpc_push_proc_bind: { + // Build void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, + // int proc_bind) + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_proc_bind"); + break; + } } return RTLFn; } @@ -1240,6 +1265,25 @@ void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF, CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield), Args); } +void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF, + const RegionCodeGenTy &TaskgroupOpGen, + SourceLocation Loc) { + // __kmpc_taskgroup(ident_t *, gtid); + // TaskgroupOpGen(); + // __kmpc_end_taskgroup(ident_t *, gtid); + // Prepare arguments and build a call to __kmpc_taskgroup + { + CodeGenFunction::RunCleanupsScope Scope(CGF); + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args); + // Build a call to __kmpc_end_taskgroup + CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>( + NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_taskgroup), + llvm::makeArrayRef(Args)); + emitInlinedDirective(CGF, TaskgroupOpGen); + } +} + static llvm::Value *emitCopyprivateCopyFunction( CodeGenModule &CGM, llvm::Type *ArgsType, ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs, @@ -1600,6 +1644,39 @@ void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF, Args); } +void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF, + OpenMPProcBindClauseKind ProcBind, + SourceLocation Loc) { + // Constants for proc bind value accepted by the runtime. + enum ProcBindTy { + ProcBindFalse = 0, + ProcBindTrue, + ProcBindMaster, + ProcBindClose, + ProcBindSpread, + ProcBindIntel, + ProcBindDefault + } RuntimeProcBind; + switch (ProcBind) { + case OMPC_PROC_BIND_master: + RuntimeProcBind = ProcBindMaster; + break; + case OMPC_PROC_BIND_close: + RuntimeProcBind = ProcBindClose; + break; + case OMPC_PROC_BIND_spread: + RuntimeProcBind = ProcBindSpread; + break; + case OMPC_PROC_BIND_unknown: + llvm_unreachable("Unsupported proc_bind value."); + } + // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind) + llvm::Value *Args[] = { + emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), + llvm::ConstantInt::get(CGM.IntTy, RuntimeProcBind, /*isSigned=*/true)}; + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args); +} + void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>, SourceLocation Loc) { // Build call void __kmpc_flush(ident_t *loc) @@ -2242,7 +2319,7 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps, - bool WithNowait) { + bool WithNowait, bool SimpleReduction) { // Next code should be emitted for reduction: // // static kmp_critical_name lock = { 0 }; @@ -2272,9 +2349,22 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, // break; // default:; // } + // + // if SimpleReduction is true, only the next code is generated: + // ... + // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]); + // ... auto &C = CGM.getContext(); + if (SimpleReduction) { + CodeGenFunction::RunCleanupsScope Scope(CGF); + for (auto *E : ReductionOps) { + CGF.EmitIgnoredExpr(E); + } + return; + } + // 1. Build a list of reduction variables. // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]}; llvm::APInt ArraySize(/*unsigned int numBits=*/32, RHSExprs.size()); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.h b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.h index f5aa4a5..d1efde2 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.h @@ -131,6 +131,13 @@ private: // Call to kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 // global_tid); OMPRTL__kmpc_omp_taskwait, + // Call to void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid); + OMPRTL__kmpc_taskgroup, + // Call to void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid); + OMPRTL__kmpc_end_taskgroup, + // Call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, + // int proc_bind); + OMPRTL__kmpc_push_proc_bind, }; /// \brief Values for bit flags used in the ident_t to describe the fields. @@ -388,6 +395,13 @@ public: /// \brief Emits code for a taskyield directive. virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc); + /// \brief Emit a taskgroup region. + /// \param TaskgroupOpGen Generator for the statement associated with the + /// given taskgroup region. + virtual void emitTaskgroupRegion(CodeGenFunction &CGF, + const RegionCodeGenTy &TaskgroupOpGen, + SourceLocation Loc); + /// \brief Emits a single region. /// \param SingleOpGen Generator for the statement associated with the given /// single region. @@ -401,7 +415,7 @@ public: /// \brief Emit an ordered region. /// \param OrderedOpGen Generator for the statement associated with the given - /// critical region. + /// ordered region. virtual void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc); @@ -504,6 +518,12 @@ public: llvm::Value *NumThreads, SourceLocation Loc); + /// \brief Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 + /// global_tid, int proc_bind) to generate code for 'proc_bind' clause. + virtual void emitProcBindClause(CodeGenFunction &CGF, + OpenMPProcBindClauseKind ProcBind, + SourceLocation Loc); + /// \brief Returns address of the threadprivate variable for the current /// thread. /// \param VD Threadprivate variable. @@ -632,7 +652,7 @@ public: ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps, - bool WithNowait); + bool WithNowait, bool SimpleReduction); /// \brief Emit code for 'taskwait' directive. virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGStmt.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGStmt.cpp index c879750..9286f03 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGStmt.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGStmt.cpp @@ -222,6 +222,9 @@ void CodeGenFunction::EmitStmt(const Stmt *S) { case Stmt::OMPTaskwaitDirectiveClass: EmitOMPTaskwaitDirective(cast<OMPTaskwaitDirective>(*S)); break; + case Stmt::OMPTaskgroupDirectiveClass: + EmitOMPTaskgroupDirective(cast<OMPTaskgroupDirective>(*S)); + break; case Stmt::OMPFlushDirectiveClass: EmitOMPFlushDirective(cast<OMPFlushDirective>(*S)); break; @@ -682,7 +685,7 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S, JumpDest LoopHeader = getJumpDestInCurrentScope("while.cond"); EmitBlock(LoopHeader.getBlock()); - LoopStack.push(LoopHeader.getBlock()); + LoopStack.push(LoopHeader.getBlock(), WhileAttrs); // Create an exit block for when the condition fails, which will // also become the break target. @@ -776,7 +779,7 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S, // Emit the body of the loop. llvm::BasicBlock *LoopBody = createBasicBlock("do.body"); - LoopStack.push(LoopBody); + LoopStack.push(LoopBody, DoAttrs); EmitBlockWithFallThrough(LoopBody, &S); { @@ -842,7 +845,7 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S, llvm::BasicBlock *CondBlock = Continue.getBlock(); EmitBlock(CondBlock); - LoopStack.push(CondBlock); + LoopStack.push(CondBlock, ForAttrs); // If the for loop doesn't have an increment we can just use the // condition as the continue block. Otherwise we'll need to create @@ -940,7 +943,7 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S, llvm::BasicBlock *CondBlock = createBasicBlock("for.cond"); EmitBlock(CondBlock); - LoopStack.push(CondBlock); + LoopStack.push(CondBlock, ForAttrs); // If there are any cleanups between here and the loop-exit scope, // create a block to stage a loop exit along. @@ -1750,6 +1753,16 @@ llvm::Value* CodeGenFunction::EmitAsmInput( const TargetInfo::ConstraintInfo &Info, const Expr *InputExpr, std::string &ConstraintStr) { + // If this can't be a register or memory, i.e., has to be a constant + // (immediate or symbolic), try to emit it as such. + if (!Info.allowsRegister() && !Info.allowsMemory()) { + llvm::APSInt Result; + if (InputExpr->EvaluateAsInt(Result, getContext())) + return llvm::ConstantInt::get(getLLVMContext(), Result); + assert(!Info.requiresImmediateConstant() && + "Required-immediate inlineasm arg isn't constant?"); + } + if (Info.allowsRegister() || !Info.allowsMemory()) if (CodeGenFunction::hasScalarEvaluationKind(InputExpr->getType())) return EmitScalarExpr(InputExpr); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp index 07fc6e9..5e94d56 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp @@ -316,26 +316,32 @@ void CodeGenFunction::EmitOMPLastprivateClauseFinal( // ... // orig_varn = private_orig_varn; // } - auto *ThenBB = createBasicBlock(".omp.lastprivate.then"); - auto *DoneBB = createBasicBlock(".omp.lastprivate.done"); - Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB); - EmitBlock(ThenBB); + llvm::BasicBlock *ThenBB = nullptr; + llvm::BasicBlock *DoneBB = nullptr; + if (IsLastIterCond) { + ThenBB = createBasicBlock(".omp.lastprivate.then"); + DoneBB = createBasicBlock(".omp.lastprivate.done"); + Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB); + EmitBlock(ThenBB); + } llvm::DenseMap<const Decl *, const Expr *> LoopCountersAndUpdates; const Expr *LastIterVal = nullptr; const Expr *IVExpr = nullptr; const Expr *IncExpr = nullptr; if (auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) { - LastIterVal = - cast<VarDecl>(cast<DeclRefExpr>(LoopDirective->getUpperBoundVariable()) - ->getDecl()) - ->getAnyInitializer(); - IVExpr = LoopDirective->getIterationVariable(); - IncExpr = LoopDirective->getInc(); - auto IUpdate = LoopDirective->updates().begin(); - for (auto *E : LoopDirective->counters()) { - auto *D = cast<DeclRefExpr>(E)->getDecl()->getCanonicalDecl(); - LoopCountersAndUpdates[D] = *IUpdate; - ++IUpdate; + if (isOpenMPWorksharingDirective(D.getDirectiveKind())) { + LastIterVal = cast<VarDecl>(cast<DeclRefExpr>( + LoopDirective->getUpperBoundVariable()) + ->getDecl()) + ->getAnyInitializer(); + IVExpr = LoopDirective->getIterationVariable(); + IncExpr = LoopDirective->getInc(); + auto IUpdate = LoopDirective->updates().begin(); + for (auto *E : LoopDirective->counters()) { + auto *D = cast<DeclRefExpr>(E)->getDecl()->getCanonicalDecl(); + LoopCountersAndUpdates[D] = *IUpdate; + ++IUpdate; + } } } { @@ -355,7 +361,7 @@ void CodeGenFunction::EmitOMPLastprivateClauseFinal( // directive, update its value before copyin back to original // variable. if (auto *UpExpr = LoopCountersAndUpdates.lookup(CanonicalVD)) { - if (FirstLCV) { + if (FirstLCV && LastIterVal) { EmitAnyExprToMem(LastIterVal, EmitLValue(IVExpr).getAddress(), IVExpr->getType().getQualifiers(), /*IsInitializer=*/false); @@ -379,7 +385,9 @@ void CodeGenFunction::EmitOMPLastprivateClauseFinal( } } } - EmitBlock(DoneBB, /*IsFinished=*/true); + if (IsLastIterCond) { + EmitBlock(DoneBB, /*IsFinished=*/true); + } } void CodeGenFunction::EmitOMPReductionClauseInit( @@ -435,7 +443,9 @@ void CodeGenFunction::EmitOMPReductionClauseFinal( CGM.getOpenMPRuntime().emitReduction( *this, D.getLocEnd(), LHSExprs, RHSExprs, ReductionOps, D.getSingleClause(OMPC_nowait) || - isOpenMPParallelDirective(D.getDirectiveKind())); + isOpenMPParallelDirective(D.getDirectiveKind()) || + D.getDirectiveKind() == OMPD_simd, + D.getDirectiveKind() == OMPD_simd); } } @@ -454,6 +464,12 @@ static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, CGF.CGM.getOpenMPRuntime().emitNumThreadsClause( CGF, NumThreads, NumThreadsClause->getLocStart()); } + if (auto *C = S.getSingleClause(OMPC_proc_bind)) { + CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); + auto *ProcBindClause = cast<OMPProcBindClause>(C); + CGF.CGM.getOpenMPRuntime().emitProcBindClause( + CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getLocStart()); + } const Expr *IfCond = nullptr; if (auto C = S.getSingleClause(OMPC_if)) { IfCond = cast<OMPIfClause>(C)->getCondition(); @@ -489,15 +505,14 @@ void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) { emitCommonOMPParallelDirective(*this, S, CodeGen); } -void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &S, - bool SeparateIter) { +void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D) { RunCleanupsScope BodyScope(*this); // Update counters values on current iteration. - for (auto I : S.updates()) { + for (auto I : D.updates()) { EmitIgnoredExpr(I); } // Update the linear variables. - for (auto &&I = S.getClausesOfKind(OMPC_linear); I; ++I) { + for (auto &&I = D.getClausesOfKind(OMPC_linear); I; ++I) { auto *C = cast<OMPLinearClause>(*I); for (auto U : C->updates()) { EmitIgnoredExpr(U); @@ -508,16 +523,14 @@ void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &S, auto Continue = getJumpDestInCurrentScope("omp.body.continue"); BreakContinueStack.push_back(BreakContinue(JumpDest(), Continue)); // Emit loop body. - EmitStmt(S.getBody()); + EmitStmt(D.getBody()); // The end (updates/cleanups). EmitBlock(Continue.getBlock()); BreakContinueStack.pop_back(); - if (SeparateIter) { // TODO: Update lastprivates if the SeparateIter flag is true. // This will be implemented in a follow-up OMPLastprivateClause patch, but // result should be still correct without it, as we do not make these // variables private yet. - } } void CodeGenFunction::EmitOMPInnerLoop( @@ -567,70 +580,89 @@ void CodeGenFunction::EmitOMPInnerLoop( EmitBlock(LoopExit.getBlock()); } -void CodeGenFunction::EmitOMPSimdFinal(const OMPLoopDirective &S) { - auto IC = S.counters().begin(); - for (auto F : S.finals()) { - auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl()); - if (LocalDeclMap.lookup(OrigVD)) { +void CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) { + // Emit inits for the linear variables. + for (auto &&I = D.getClausesOfKind(OMPC_linear); I; ++I) { + auto *C = cast<OMPLinearClause>(*I); + for (auto Init : C->inits()) { + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl()); + auto *OrigVD = cast<VarDecl>( + cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())->getDecl()); DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), CapturedStmtInfo->lookup(OrigVD) != nullptr, - (*IC)->getType(), VK_LValue, (*IC)->getExprLoc()); - auto *OrigAddr = EmitLValue(&DRE).getAddress(); - OMPPrivateScope VarScope(*this); - VarScope.addPrivate(OrigVD, - [OrigAddr]() -> llvm::Value *{ return OrigAddr; }); - (void)VarScope.Privatize(); - EmitIgnoredExpr(F); + VD->getInit()->getType(), VK_LValue, + VD->getInit()->getExprLoc()); + AutoVarEmission Emission = EmitAutoVarAlloca(*VD); + EmitExprAsInit(&DRE, VD, + MakeAddrLValue(Emission.getAllocatedAddress(), + VD->getType(), Emission.Alignment), + /*capturedByInit=*/false); + EmitAutoVarCleanups(Emission); } - ++IC; + // Emit the linear steps for the linear clauses. + // If a step is not constant, it is pre-calculated before the loop. + if (auto CS = cast_or_null<BinaryOperator>(C->getCalcStep())) + if (auto SaveRef = cast<DeclRefExpr>(CS->getLHS())) { + EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl())); + // Emit calculation of the linear step. + EmitIgnoredExpr(CS); + } } +} + +static void emitLinearClauseFinal(CodeGenFunction &CGF, + const OMPLoopDirective &D) { // Emit the final values of the linear variables. - for (auto &&I = S.getClausesOfKind(OMPC_linear); I; ++I) { + for (auto &&I = D.getClausesOfKind(OMPC_linear); I; ++I) { auto *C = cast<OMPLinearClause>(*I); auto IC = C->varlist_begin(); for (auto F : C->finals()) { auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl()); DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), - CapturedStmtInfo->lookup(OrigVD) != nullptr, + CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr, (*IC)->getType(), VK_LValue, (*IC)->getExprLoc()); - auto *OrigAddr = EmitLValue(&DRE).getAddress(); - OMPPrivateScope VarScope(*this); + auto *OrigAddr = CGF.EmitLValue(&DRE).getAddress(); + CodeGenFunction::OMPPrivateScope VarScope(CGF); VarScope.addPrivate(OrigVD, [OrigAddr]() -> llvm::Value *{ return OrigAddr; }); (void)VarScope.Privatize(); - EmitIgnoredExpr(F); + CGF.EmitIgnoredExpr(F); ++IC; } } } -static void EmitOMPAlignedClause(CodeGenFunction &CGF, CodeGenModule &CGM, - const OMPAlignedClause &Clause) { - unsigned ClauseAlignment = 0; - if (auto AlignmentExpr = Clause.getAlignment()) { - auto AlignmentCI = - cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr)); - ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue()); - } - for (auto E : Clause.varlists()) { - unsigned Alignment = ClauseAlignment; - if (Alignment == 0) { - // OpenMP [2.8.1, Description] - // If no optional parameter is specified, implementation-defined default - // alignments for SIMD instructions on the target platforms are assumed. - Alignment = CGM.getTargetCodeGenInfo().getOpenMPSimdDefaultAlignment( - E->getType()); +static void emitAlignedClause(CodeGenFunction &CGF, + const OMPExecutableDirective &D) { + for (auto &&I = D.getClausesOfKind(OMPC_aligned); I; ++I) { + auto *Clause = cast<OMPAlignedClause>(*I); + unsigned ClauseAlignment = 0; + if (auto AlignmentExpr = Clause->getAlignment()) { + auto AlignmentCI = + cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr)); + ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue()); } - assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) && - "alignment is not power of 2"); - if (Alignment != 0) { - llvm::Value *PtrValue = CGF.EmitScalarExpr(E); - CGF.EmitAlignmentAssumption(PtrValue, Alignment); + for (auto E : Clause->varlists()) { + unsigned Alignment = ClauseAlignment; + if (Alignment == 0) { + // OpenMP [2.8.1, Description] + // If no optional parameter is specified, implementation-defined default + // alignments for SIMD instructions on the target platforms are assumed. + Alignment = + CGF.CGM.getTargetCodeGenInfo().getOpenMPSimdDefaultAlignment( + E->getType()); + } + assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) && + "alignment is not power of 2"); + if (Alignment != 0) { + llvm::Value *PtrValue = CGF.EmitScalarExpr(E); + CGF.EmitAlignmentAssumption(PtrValue, Alignment); + } } } } -static void EmitPrivateLoopCounters(CodeGenFunction &CGF, +static void emitPrivateLoopCounters(CodeGenFunction &CGF, CodeGenFunction::OMPPrivateScope &LoopScope, ArrayRef<Expr *> Counters) { for (auto *E : Counters) { @@ -647,37 +679,39 @@ static void EmitPrivateLoopCounters(CodeGenFunction &CGF, static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount) { - CodeGenFunction::OMPPrivateScope PreCondScope(CGF); - EmitPrivateLoopCounters(CGF, PreCondScope, S.counters()); - const VarDecl *IVDecl = - cast<VarDecl>(cast<DeclRefExpr>(S.getIterationVariable())->getDecl()); - bool IsRegistered = PreCondScope.addPrivate(IVDecl, [&]() -> llvm::Value *{ - // Emit var without initialization. - auto VarEmission = CGF.EmitAutoVarAlloca(*IVDecl); - CGF.EmitAutoVarCleanups(VarEmission); - return VarEmission.getAllocatedAddress(); - }); - assert(IsRegistered && "counter already registered as private"); - // Silence the warning about unused variable. - (void)IsRegistered; - (void)PreCondScope.Privatize(); - // Initialize internal counter to 0 to calculate initial values of real - // counters. - LValue IV = CGF.EmitLValue(S.getIterationVariable()); - CGF.EmitStoreOfScalar( - llvm::ConstantInt::getNullValue( - IV.getAddress()->getType()->getPointerElementType()), - CGF.EmitLValue(S.getIterationVariable()), /*isInit=*/true); - // Get initial values of real counters. - for (auto I : S.updates()) { - CGF.EmitIgnoredExpr(I); + { + CodeGenFunction::OMPPrivateScope PreCondScope(CGF); + emitPrivateLoopCounters(CGF, PreCondScope, S.counters()); + const VarDecl *IVDecl = + cast<VarDecl>(cast<DeclRefExpr>(S.getIterationVariable())->getDecl()); + bool IsRegistered = PreCondScope.addPrivate(IVDecl, [&]() -> llvm::Value *{ + // Emit var without initialization. + auto VarEmission = CGF.EmitAutoVarAlloca(*IVDecl); + CGF.EmitAutoVarCleanups(VarEmission); + return VarEmission.getAllocatedAddress(); + }); + assert(IsRegistered && "counter already registered as private"); + // Silence the warning about unused variable. + (void)IsRegistered; + (void)PreCondScope.Privatize(); + // Initialize internal counter to 0 to calculate initial values of real + // counters. + LValue IV = CGF.EmitLValue(S.getIterationVariable()); + CGF.EmitStoreOfScalar( + llvm::ConstantInt::getNullValue( + IV.getAddress()->getType()->getPointerElementType()), + CGF.EmitLValue(S.getIterationVariable()), /*isInit=*/true); + // Get initial values of real counters. + for (auto I : S.updates()) { + CGF.EmitIgnoredExpr(I); + } } // Check that loop is executed at least one time. CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount); } static void -EmitPrivateLinearVars(CodeGenFunction &CGF, const OMPExecutableDirective &D, +emitPrivateLinearVars(CodeGenFunction &CGF, const OMPExecutableDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) { for (auto &&I = D.getClausesOfKind(OMPC_linear); I; ++I) { auto *C = cast<OMPLinearClause>(*I); @@ -696,19 +730,50 @@ EmitPrivateLinearVars(CodeGenFunction &CGF, const OMPExecutableDirective &D, } } +static void emitSafelenClause(CodeGenFunction &CGF, + const OMPExecutableDirective &D) { + if (auto *C = + cast_or_null<OMPSafelenClause>(D.getSingleClause(OMPC_safelen))) { + RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(), + /*ignoreResult=*/true); + llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal()); + CGF.LoopStack.setVectorizerWidth(Val->getZExtValue()); + // In presence of finite 'safelen', it may be unsafe to mark all + // the memory instructions parallel, because loop-carried + // dependences of 'safelen' iterations are possible. + CGF.LoopStack.setParallel(false); + } +} + +void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D) { + // Walk clauses and process safelen/lastprivate. + LoopStack.setParallel(); + LoopStack.setVectorizerEnable(true); + emitSafelenClause(*this, D); +} + +void CodeGenFunction::EmitOMPSimdFinal(const OMPLoopDirective &D) { + auto IC = D.counters().begin(); + for (auto F : D.finals()) { + auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl()); + if (LocalDeclMap.lookup(OrigVD) || CapturedStmtInfo->lookup(OrigVD)) { + DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), + CapturedStmtInfo->lookup(OrigVD) != nullptr, + (*IC)->getType(), VK_LValue, (*IC)->getExprLoc()); + auto *OrigAddr = EmitLValue(&DRE).getAddress(); + OMPPrivateScope VarScope(*this); + VarScope.addPrivate(OrigVD, + [OrigAddr]() -> llvm::Value *{ return OrigAddr; }); + (void)VarScope.Privatize(); + EmitIgnoredExpr(F); + } + ++IC; + } + emitLinearClauseFinal(*this, D); +} + void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF) { - // Pragma 'simd' code depends on presence of 'lastprivate'. - // If present, we have to separate last iteration of the loop: - // - // if (PreCond) { - // for (IV in 0..LastIteration-1) BODY; - // BODY with updates of lastprivate vars; - // <Final counter/linear vars updates>; - // } - // - // otherwise (when there's no lastprivate): - // // if (PreCond) { // for (IV in 0..LastIteration) BODY; // <Final counter/linear vars updates>; @@ -731,43 +796,6 @@ void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) { CGF.EmitBlock(ThenBlock); CGF.incrementProfileCounter(&S); } - // Walk clauses and process safelen/lastprivate. - bool SeparateIter = false; - CGF.LoopStack.setParallel(); - CGF.LoopStack.setVectorizerEnable(true); - for (auto C : S.clauses()) { - switch (C->getClauseKind()) { - case OMPC_safelen: { - RValue Len = CGF.EmitAnyExpr(cast<OMPSafelenClause>(C)->getSafelen(), - AggValueSlot::ignored(), true); - llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal()); - CGF.LoopStack.setVectorizerWidth(Val->getZExtValue()); - // In presence of finite 'safelen', it may be unsafe to mark all - // the memory instructions parallel, because loop-carried - // dependences of 'safelen' iterations are possible. - CGF.LoopStack.setParallel(false); - break; - } - case OMPC_aligned: - EmitOMPAlignedClause(CGF, CGF.CGM, cast<OMPAlignedClause>(*C)); - break; - case OMPC_lastprivate: - SeparateIter = true; - break; - default: - // Not handled yet - ; - } - } - - // Emit inits for the linear variables. - for (auto &&I = S.getClausesOfKind(OMPC_linear); I; ++I) { - auto *C = cast<OMPLinearClause>(*I); - for (auto Init : C->inits()) { - auto *D = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl()); - CGF.EmitVarDecl(*D); - } - } // Emit the loop iteration variable. const Expr *IVExpr = S.getIterationVariable(); @@ -784,34 +812,31 @@ void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) { CGF.EmitIgnoredExpr(S.getCalcLastIteration()); } - // Emit the linear steps for the linear clauses. - // If a step is not constant, it is pre-calculated before the loop. - for (auto &&I = S.getClausesOfKind(OMPC_linear); I; ++I) { - auto *C = cast<OMPLinearClause>(*I); - if (auto CS = cast_or_null<BinaryOperator>(C->getCalcStep())) - if (auto SaveRef = cast<DeclRefExpr>(CS->getLHS())) { - CGF.EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl())); - // Emit calculation of the linear step. - CGF.EmitIgnoredExpr(CS); - } - } + CGF.EmitOMPSimdInit(S); + emitAlignedClause(CGF, S); + CGF.EmitOMPLinearClauseInit(S); + bool HasLastprivateClause; { OMPPrivateScope LoopScope(CGF); - EmitPrivateLoopCounters(CGF, LoopScope, S.counters()); - EmitPrivateLinearVars(CGF, S, LoopScope); + emitPrivateLoopCounters(CGF, LoopScope, S.counters()); + emitPrivateLinearVars(CGF, S, LoopScope); CGF.EmitOMPPrivateClause(S, LoopScope); + CGF.EmitOMPReductionClauseInit(S, LoopScope); + HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); (void)LoopScope.Privatize(); CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), - S.getCond(SeparateIter), S.getInc(), + S.getCond(), S.getInc(), [&S](CodeGenFunction &CGF) { CGF.EmitOMPLoopBody(S); CGF.EmitStopPoint(&S); }, [](CodeGenFunction &) {}); - if (SeparateIter) { - CGF.EmitOMPLoopBody(S, /*SeparateIter=*/true); + // Emit final copy of the lastprivate variables at the end of loops. + if (HasLastprivateClause) { + CGF.EmitOMPLastprivateClauseFinal(S); } + CGF.EmitOMPReductionClauseFinal(S); } CGF.EmitOMPSimdFinal(S); // Emit: if (PreCond) - end. @@ -912,7 +937,7 @@ void CodeGenFunction::EmitOMPForOuterLoop(OpenMPScheduleClauseKind ScheduleKind, // IV = LB EmitIgnoredExpr(S.getInit()); // IV < UB - BoolCondVal = EvaluateExprAsBool(S.getCond(false)); + BoolCondVal = EvaluateExprAsBool(S.getCond()); } else { BoolCondVal = RT.emitForNext(*this, S.getLocStart(), IVSize, IVSigned, IL, LB, UB, ST); @@ -941,14 +966,19 @@ void CodeGenFunction::EmitOMPForOuterLoop(OpenMPScheduleClauseKind ScheduleKind, auto Continue = getJumpDestInCurrentScope("omp.dispatch.inc"); BreakContinueStack.push_back(BreakContinue(LoopExit, Continue)); + // Generate !llvm.loop.parallel metadata for loads and stores for loops + // with dynamic/guided scheduling and without ordered clause. + if (!isOpenMPSimdDirective(S.getDirectiveKind())) { + LoopStack.setParallel((ScheduleKind == OMPC_SCHEDULE_dynamic || + ScheduleKind == OMPC_SCHEDULE_guided) && + !Ordered); + } else { + EmitOMPSimdInit(S); + } + SourceLocation Loc = S.getLocStart(); - // Generate !llvm.loop.parallel metadata for loads and stores for loops with - // dynamic/guided scheduling and without ordered clause. - LoopStack.setParallel((ScheduleKind == OMPC_SCHEDULE_dynamic || - ScheduleKind == OMPC_SCHEDULE_guided) && - !Ordered); EmitOMPInnerLoop( - S, LoopScope.requiresCleanups(), S.getCond(/*SeparateIter=*/false), + S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(), [&S](CodeGenFunction &CGF) { CGF.EmitOMPLoopBody(S); @@ -1055,6 +1085,9 @@ bool CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) { EmitBlock(ThenBlock); incrementProfileCounter(&S); } + + emitAlignedClause(*this, S); + EmitOMPLinearClauseInit(S); // Emit 'then' code. { // Emit helper vars inits. @@ -1077,7 +1110,8 @@ bool CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) { EmitOMPPrivateClause(S, LoopScope); HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope); EmitOMPReductionClauseInit(S, LoopScope); - EmitPrivateLoopCounters(*this, LoopScope, S.counters()); + emitPrivateLoopCounters(*this, LoopScope, S.counters()); + emitPrivateLinearVars(*this, S, LoopScope); (void)LoopScope.Privatize(); // Detect the loop schedule kind and chunk. @@ -1093,6 +1127,9 @@ bool CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) { if (RT.isStaticNonchunked(ScheduleKind, /* Chunked */ Chunk != nullptr) && !Ordered) { + if (isOpenMPSimdDirective(S.getDirectiveKind())) { + EmitOMPSimdInit(S); + } // OpenMP [2.7.1, Loop Construct, Description, table 2-1] // When no chunk_size is specified, the iteration space is divided into // chunks that are approximately equal in size, and at most one chunk is @@ -1106,8 +1143,8 @@ bool CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) { // IV = LB; EmitIgnoredExpr(S.getInit()); // while (idx <= UB) { BODY; ++idx; } - EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), - S.getCond(/*SeparateIter=*/false), S.getInc(), + EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(), + S.getInc(), [&S](CodeGenFunction &CGF) { CGF.EmitOMPLoopBody(S); CGF.EmitStopPoint(&S); @@ -1128,6 +1165,9 @@ bool CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) { EmitOMPLastprivateClauseFinal( S, Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart()))); } + if (isOpenMPSimdDirective(S.getDirectiveKind())) { + EmitOMPSimdFinal(S); + } // We're now done with the loop, so jump to the continuation block. if (ContBlock) { EmitBranch(ContBlock); @@ -1151,8 +1191,18 @@ void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) { } } -void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &) { - llvm_unreachable("CodeGen for 'omp for simd' is not supported yet."); +void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) { + LexicalScope Scope(*this, S.getSourceRange()); + bool HasLastprivates = false; + auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF) { + HasLastprivates = CGF.EmitOMPWorksharingLoop(S); + }; + CGM.getOpenMPRuntime().emitInlinedDirective(*this, CodeGen); + + // Emit an implicit barrier at the end. + if (!S.getSingleClause(OMPC_nowait) || HasLastprivates) { + CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for); + } } static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, @@ -1407,8 +1457,20 @@ void CodeGenFunction::EmitOMPParallelForDirective( } void CodeGenFunction::EmitOMPParallelForSimdDirective( - const OMPParallelForSimdDirective &) { - llvm_unreachable("CodeGen for 'omp parallel for simd' is not supported yet."); + const OMPParallelForSimdDirective &S) { + // Emit directive as a combined directive that consists of two implicit + // directives: 'parallel' with 'for' directive. + LexicalScope Scope(*this, S.getSourceRange()); + (void)emitScheduleClause(*this, S, /*OuterRegion=*/true); + auto &&CodeGen = [&S](CodeGenFunction &CGF) { + CGF.EmitOMPWorksharingLoop(S); + // Emit implicit barrier at the end of parallel region, but this barrier + // is at the end of 'for' directive, so emit it as the implicit barrier for + // this 'for' directive. + CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), + OMPD_parallel); + }; + emitCommonOMPParallelDirective(*this, S, CodeGen); } void CodeGenFunction::EmitOMPParallelSectionsDirective( @@ -1556,6 +1618,16 @@ void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) { CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getLocStart()); } +void CodeGenFunction::EmitOMPTaskgroupDirective( + const OMPTaskgroupDirective &S) { + LexicalScope Scope(*this, S.getSourceRange()); + auto &&CodeGen = [&S](CodeGenFunction &CGF) { + CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + CGF.EnsureInsertPoint(); + }; + CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getLocStart()); +} + void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) { CGM.getOpenMPRuntime().emitFlush(*this, [&]() -> ArrayRef<const Expr *> { if (auto C = S.getSingleClause(/*K*/ OMPC_flush)) { diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.cpp index 57370a6..17db401 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.cpp @@ -848,7 +848,8 @@ void CodeGenModule::EmitVTableBitSetEntries(llvm::GlobalVariable *VTable, !LangOpts.Sanitize.has(SanitizerKind::CFIUnrelatedCast)) return; - llvm::Metadata *VTableMD = llvm::ConstantAsMetadata::get(VTable); + CharUnits PointerWidth = + Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(0)); std::vector<llvm::MDTuple *> BitsetEntries; // Create a bit set entry for each address point. @@ -857,23 +858,8 @@ void CodeGenModule::EmitVTableBitSetEntries(llvm::GlobalVariable *VTable, if (AP.first.getBase()->isInStdNamespace()) continue; - std::string OutName; - llvm::raw_string_ostream Out(OutName); - getCXXABI().getMangleContext().mangleCXXVTableBitSet(AP.first.getBase(), - Out); - - CharUnits PointerWidth = - Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(0)); - uint64_t AddrPointOffset = AP.second * PointerWidth.getQuantity(); - - llvm::Metadata *BitsetOps[] = { - llvm::MDString::get(getLLVMContext(), Out.str()), - VTableMD, - llvm::ConstantAsMetadata::get( - llvm::ConstantInt::get(Int64Ty, AddrPointOffset))}; - llvm::MDTuple *BitsetEntry = - llvm::MDTuple::get(getLLVMContext(), BitsetOps); - BitsetEntries.push_back(BitsetEntry); + BitsetEntries.push_back(CreateVTableBitSetEntry( + VTable, PointerWidth * AP.second, AP.first.getBase())); } // Sort the bit set entries for determinism. diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp index 7e82fcc..54e6b73 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp @@ -649,14 +649,14 @@ CodeGenAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { return nullptr; } - ErrorOr<llvm::Module *> ModuleOrErr = + ErrorOr<std::unique_ptr<llvm::Module>> ModuleOrErr = getLazyBitcodeModule(std::move(*BCBuf), *VMContext); if (std::error_code EC = ModuleOrErr.getError()) { CI.getDiagnostics().Report(diag::err_cannot_open_file) << LinkBCFile << EC.message(); return nullptr; } - LinkModuleToUse = ModuleOrErr.get(); + LinkModuleToUse = ModuleOrErr.get().release(); } CoverageSourceInfo *CoverageInfo = nullptr; diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.cpp index 01da750..bece41e 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.cpp @@ -615,12 +615,14 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, } // Apply sanitizer attributes to the function. - if (SanOpts.has(SanitizerKind::Address)) + if (SanOpts.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress)) Fn->addFnAttr(llvm::Attribute::SanitizeAddress); if (SanOpts.has(SanitizerKind::Thread)) Fn->addFnAttr(llvm::Attribute::SanitizeThread); if (SanOpts.has(SanitizerKind::Memory)) Fn->addFnAttr(llvm::Attribute::SanitizeMemory); + if (SanOpts.has(SanitizerKind::SafeStack)) + Fn->addFnAttr(llvm::Attribute::SafeStack); // Pass inline keyword to optimizer if it appears explicitly on any // declaration. Also, in the case of -fno-inline attach NoInline diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h index 469022d..45475d1 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h @@ -1311,19 +1311,29 @@ public: /// to by This. llvm::Value *GetVTablePtr(llvm::Value *This, llvm::Type *Ty); + enum CFITypeCheckKind { + CFITCK_VCall, + CFITCK_NVCall, + CFITCK_DerivedCast, + CFITCK_UnrelatedCast, + }; + /// \brief Derived is the presumed address of an object of type T after a /// cast. If T is a polymorphic class type, emit a check that the virtual /// table for Derived belongs to a class derived from T. void EmitVTablePtrCheckForCast(QualType T, llvm::Value *Derived, - bool MayBeNull); + bool MayBeNull, CFITypeCheckKind TCK, + SourceLocation Loc); /// EmitVTablePtrCheckForCall - Virtual method MD is being called via VTable. /// If vptr CFI is enabled, emit a check that VTable is valid. - void EmitVTablePtrCheckForCall(const CXXMethodDecl *MD, llvm::Value *VTable); + void EmitVTablePtrCheckForCall(const CXXMethodDecl *MD, llvm::Value *VTable, + CFITypeCheckKind TCK, SourceLocation Loc); /// EmitVTablePtrCheck - Emit a check that VTable is a valid virtual table for /// RD using llvm.bitset.test. - void EmitVTablePtrCheck(const CXXRecordDecl *RD, llvm::Value *VTable); + void EmitVTablePtrCheck(const CXXRecordDecl *RD, llvm::Value *VTable, + CFITypeCheckKind TCK, SourceLocation Loc); /// CanDevirtualizeMemberFunctionCalls - Checks whether virtual calls on given /// expr can be devirtualized. @@ -2130,9 +2140,9 @@ public: /// \param D Directive that has at least one 'lastprivate' directives. /// \param IsLastIterCond Boolean condition that must be set to 'i1 true' if /// it is the last iteration of the loop code in associated directive, or to - /// 'i1 false' otherwise. + /// 'i1 false' otherwise. If this item is nullptr, no final check is required. void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, - llvm::Value *IsLastIterCond); + llvm::Value *IsLastIterCond = nullptr); /// \brief Emit initial code for reduction variables. Creates reduction copies /// and initializes them with the values according to OpenMP standard. /// @@ -2147,6 +2157,11 @@ public: /// /// \param D Directive that has at least one 'reduction' directives. void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D); + /// \brief Emit initial code for linear variables. Creates private copies + /// and initializes them with the values according to OpenMP standard. + /// + /// \param D Directive (possibly) with the 'linear' clause. + void EmitOMPLinearClauseInit(const OMPLoopDirective &D); void EmitOMPParallelDirective(const OMPParallelDirective &S); void EmitOMPSimdDirective(const OMPSimdDirective &S); @@ -2164,6 +2179,7 @@ public: void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S); void EmitOMPBarrierDirective(const OMPBarrierDirective &S); void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S); + void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S); void EmitOMPFlushDirective(const OMPFlushDirective &S); void EmitOMPOrderedDirective(const OMPOrderedDirective &S); void EmitOMPAtomicDirective(const OMPAtomicDirective &S); @@ -2189,9 +2205,9 @@ public: private: /// Helpers for the OpenMP loop directives. - void EmitOMPLoopBody(const OMPLoopDirective &Directive, - bool SeparateIter = false); - void EmitOMPSimdFinal(const OMPLoopDirective &S); + void EmitOMPLoopBody(const OMPLoopDirective &D); + void EmitOMPSimdInit(const OMPLoopDirective &D); + void EmitOMPSimdFinal(const OMPLoopDirective &D); /// \brief Emit code for the worksharing loop-based directive. /// \return true, if this construct has any lastprivate clause, false - /// otherwise. @@ -2568,7 +2584,7 @@ public: llvm::Value *BuildVector(ArrayRef<llvm::Value*> Ops); llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E); llvm::Value *EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E); - llvm::Value *EmitR600BuiltinExpr(unsigned BuiltinID, const CallExpr *E); + llvm::Value *EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E); llvm::Value *EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E); llvm::Value *EmitObjCProtocolExpr(const ObjCProtocolExpr *E); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp index 9496831..2dd5414 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp @@ -205,11 +205,9 @@ void CodeGenModule::addReplacement(StringRef Name, llvm::Constant *C) { } void CodeGenModule::applyReplacements() { - for (ReplacementsTy::iterator I = Replacements.begin(), - E = Replacements.end(); - I != E; ++I) { - StringRef MangledName = I->first(); - llvm::Constant *Replacement = I->second; + for (auto &I : Replacements) { + StringRef MangledName = I.first(); + llvm::Constant *Replacement = I.second; llvm::GlobalValue *Entry = GetGlobalValue(MangledName); if (!Entry) continue; @@ -261,9 +259,7 @@ void CodeGenModule::checkAliases() { // and aliases during codegen. bool Error = false; DiagnosticsEngine &Diags = getDiags(); - for (std::vector<GlobalDecl>::iterator I = Aliases.begin(), - E = Aliases.end(); I != E; ++I) { - const GlobalDecl &GD = *I; + for (const GlobalDecl &GD : Aliases) { const auto *D = cast<ValueDecl>(GD.getDecl()); const AliasAttr *AA = D->getAttr<AliasAttr>(); StringRef MangledName = getMangledName(GD); @@ -310,9 +306,7 @@ void CodeGenModule::checkAliases() { if (!Error) return; - for (std::vector<GlobalDecl>::iterator I = Aliases.begin(), - E = Aliases.end(); I != E; ++I) { - const GlobalDecl &GD = *I; + for (const GlobalDecl &GD : Aliases) { StringRef MangledName = getMangledName(GD); llvm::GlobalValue *Entry = GetGlobalValue(MangledName); auto *Alias = cast<llvm::GlobalAlias>(Entry); @@ -637,15 +631,14 @@ void CodeGenModule::EmitCtorList(const CtorList &Fns, const char *GlobalName) { Int32Ty, llvm::PointerType::getUnqual(CtorFTy), VoidPtrTy, nullptr); // Construct the constructor and destructor arrays. - SmallVector<llvm::Constant*, 8> Ctors; - for (CtorList::const_iterator I = Fns.begin(), E = Fns.end(); I != E; ++I) { + SmallVector<llvm::Constant *, 8> Ctors; + for (const auto &I : Fns) { llvm::Constant *S[] = { - llvm::ConstantInt::get(Int32Ty, I->Priority, false), - llvm::ConstantExpr::getBitCast(I->Initializer, CtorPFTy), - (I->AssociatedData - ? llvm::ConstantExpr::getBitCast(I->AssociatedData, VoidPtrTy) - : llvm::Constant::getNullValue(VoidPtrTy)) - }; + llvm::ConstantInt::get(Int32Ty, I.Priority, false), + llvm::ConstantExpr::getBitCast(I.Initializer, CtorPFTy), + (I.AssociatedData + ? llvm::ConstantExpr::getBitCast(I.AssociatedData, VoidPtrTy) + : llvm::Constant::getNullValue(VoidPtrTy))}; Ctors.push_back(llvm::ConstantStruct::get(CtorStructTy, S)); } @@ -1029,12 +1022,9 @@ void CodeGenModule::EmitModuleLinkOptions() { SmallVector<clang::Module *, 16> Stack; // Seed the stack with imported modules. - for (llvm::SetVector<clang::Module *>::iterator M = ImportedModules.begin(), - MEnd = ImportedModules.end(); - M != MEnd; ++M) { - if (Visited.insert(*M).second) - Stack.push_back(*M); - } + for (Module *M : ImportedModules) + if (Visited.insert(M).second) + Stack.push_back(M); // Find all of the modules to import, making a little effort to prune // non-leaf modules. @@ -1070,12 +1060,9 @@ void CodeGenModule::EmitModuleLinkOptions() { // to linker options inserted by things like #pragma comment(). SmallVector<llvm::Metadata *, 16> MetadataArgs; Visited.clear(); - for (llvm::SetVector<clang::Module *>::iterator M = LinkModules.begin(), - MEnd = LinkModules.end(); - M != MEnd; ++M) { - if (Visited.insert(*M).second) - addLinkOptionsPostorder(*this, *M, MetadataArgs, Visited); - } + for (Module *M : LinkModules) + if (Visited.insert(M).second) + addLinkOptionsPostorder(*this, M, MetadataArgs, Visited); std::reverse(MetadataArgs.begin(), MetadataArgs.end()); LinkerOptionsMetadata.append(MetadataArgs.begin(), MetadataArgs.end()); @@ -1231,8 +1218,9 @@ bool CodeGenModule::isInSanitizerBlacklist(llvm::Function *Fn, bool CodeGenModule::isInSanitizerBlacklist(llvm::GlobalVariable *GV, SourceLocation Loc, QualType Ty, StringRef Category) const { - // For now globals can be blacklisted only in ASan. - if (!LangOpts.Sanitize.has(SanitizerKind::Address)) + // For now globals can be blacklisted only in ASan and KASan. + if (!LangOpts.Sanitize.hasOneOf( + SanitizerKind::Address | SanitizerKind::KernelAddress)) return false; const auto &SanitizerBL = getContext().getSanitizerBlacklist(); if (SanitizerBL.isBlacklistedGlobal(GV->getName(), Category)) @@ -3510,11 +3498,9 @@ static void EmitGlobalDeclMetadata(CodeGenModule &CGM, /// to such functions with an unmangled name from inline assembly within the /// same translation unit. void CodeGenModule::EmitStaticExternCAliases() { - for (StaticExternCMap::iterator I = StaticExternCValues.begin(), - E = StaticExternCValues.end(); - I != E; ++I) { - IdentifierInfo *Name = I->first; - llvm::GlobalValue *Val = I->second; + for (auto &I : StaticExternCValues) { + IdentifierInfo *Name = I.first; + llvm::GlobalValue *Val = I.second; if (Val && !getModule().getNamedValue(Name->getName())) addUsedGlobal(llvm::GlobalAlias::create(Name->getName(), Val)); } @@ -3674,3 +3660,17 @@ void CodeGenModule::EmitOMPThreadPrivateDecl(const OMPThreadPrivateDecl *D) { CXXGlobalInits.push_back(InitFunction); } } + +llvm::MDTuple *CodeGenModule::CreateVTableBitSetEntry( + llvm::GlobalVariable *VTable, CharUnits Offset, const CXXRecordDecl *RD) { + std::string OutName; + llvm::raw_string_ostream Out(OutName); + getCXXABI().getMangleContext().mangleCXXVTableBitSet(RD, Out); + + llvm::Metadata *BitsetOps[] = { + llvm::MDString::get(getLLVMContext(), Out.str()), + llvm::ConstantAsMetadata::get(VTable), + llvm::ConstantAsMetadata::get( + llvm::ConstantInt::get(Int64Ty, Offset.getQuantity()))}; + return llvm::MDTuple::get(getLLVMContext(), BitsetOps); +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h index edde426..8e671fa 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h @@ -1115,6 +1115,11 @@ public: void EmitVTableBitSetEntries(llvm::GlobalVariable *VTable, const VTableLayout &VTLayout); + /// Create a bitset entry for the given vtable. + llvm::MDTuple *CreateVTableBitSetEntry(llvm::GlobalVariable *VTable, + CharUnits Offset, + const CXXRecordDecl *RD); + /// \breif Get the declaration of std::terminate for the platform. llvm::Constant *getTerminateFn(); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CoverageMappingGen.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CoverageMappingGen.cpp index 024a45d..9ad5d14 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CoverageMappingGen.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CoverageMappingGen.cpp @@ -806,6 +806,9 @@ struct CounterCoverageMappingBuilder void VisitIfStmt(const IfStmt *S) { extendRegion(S); + // Extend into the condition before we propagate through it below - this is + // needed to handle macros that generate the "if" but not the condition. + extendRegion(S->getCond()); Counter ParentCount = getRegion().getCounter(); Counter ThenCount = getRegionCounter(S); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp b/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp index 0a1a4ce..3f5ad5d 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp @@ -204,7 +204,8 @@ public: llvm::Value *getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD, llvm::Value *This, - llvm::Type *Ty) override; + llvm::Type *Ty, + SourceLocation Loc) override; llvm::Value *EmitVirtualDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, @@ -1439,13 +1440,15 @@ llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, llvm::Value *ItaniumCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD, llvm::Value *This, - llvm::Type *Ty) { + llvm::Type *Ty, + SourceLocation Loc) { GD = GD.getCanonicalDecl(); Ty = Ty->getPointerTo()->getPointerTo(); llvm::Value *VTable = CGF.GetVTablePtr(This, Ty); if (CGF.SanOpts.has(SanitizerKind::CFIVCall)) - CGF.EmitVTablePtrCheckForCall(cast<CXXMethodDecl>(GD.getDecl()), VTable); + CGF.EmitVTablePtrCheckForCall(cast<CXXMethodDecl>(GD.getDecl()), VTable, + CodeGenFunction::CFITCK_VCall, Loc); uint64_t VTableIndex = CGM.getItaniumVTableContext().getMethodVTableIndex(GD); llvm::Value *VFuncPtr = @@ -1463,7 +1466,8 @@ llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall( Dtor, getFromDtorType(DtorType)); llvm::Type *Ty = CGF.CGM.getTypes().GetFunctionType(*FInfo); llvm::Value *Callee = - getVirtualFunctionPointer(CGF, GlobalDecl(Dtor, DtorType), This, Ty); + getVirtualFunctionPointer(CGF, GlobalDecl(Dtor, DtorType), This, Ty, + CE ? CE->getLocStart() : SourceLocation()); CGF.EmitCXXMemberOrOperatorCall(Dtor, Callee, ReturnValueSlot(), This, /*ImplicitParam=*/nullptr, QualType(), CE); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp index de30883..679516b 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp @@ -205,6 +205,9 @@ public: CXXDtorType Type, bool ForVirtualBase, bool Delegating, llvm::Value *This) override; + void emitVTableBitSetEntries(VPtrInfo *Info, const CXXRecordDecl *RD, + llvm::GlobalVariable *VTable); + void emitVTableDefinitions(CodeGenVTables &CGVT, const CXXRecordDecl *RD) override; @@ -221,8 +224,8 @@ public: CharUnits VPtrOffset) override; llvm::Value *getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD, - llvm::Value *This, - llvm::Type *Ty) override; + llvm::Value *This, llvm::Type *Ty, + SourceLocation Loc) override; llvm::Value *EmitVirtualDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, @@ -1401,6 +1404,58 @@ void MicrosoftCXXABI::EmitDestructorCall(CodeGenFunction &CGF, getFromDtorType(Type)); } +void MicrosoftCXXABI::emitVTableBitSetEntries(VPtrInfo *Info, + const CXXRecordDecl *RD, + llvm::GlobalVariable *VTable) { + if (!getContext().getLangOpts().Sanitize.has(SanitizerKind::CFIVCall) && + !getContext().getLangOpts().Sanitize.has(SanitizerKind::CFINVCall) && + !getContext().getLangOpts().Sanitize.has(SanitizerKind::CFIDerivedCast) && + !getContext().getLangOpts().Sanitize.has(SanitizerKind::CFIUnrelatedCast)) + return; + + llvm::NamedMDNode *BitsetsMD = + CGM.getModule().getOrInsertNamedMetadata("llvm.bitsets"); + CharUnits PointerWidth = getContext().toCharUnitsFromBits( + getContext().getTargetInfo().getPointerWidth(0)); + + // FIXME: Add blacklisting scheme. + + if (Info->PathToBaseWithVPtr.empty()) { + BitsetsMD->addOperand( + CGM.CreateVTableBitSetEntry(VTable, PointerWidth, RD)); + return; + } + + // Add a bitset entry for the least derived base belonging to this vftable. + BitsetsMD->addOperand(CGM.CreateVTableBitSetEntry( + VTable, PointerWidth, Info->PathToBaseWithVPtr.back())); + + // Add a bitset entry for each derived class that is laid out at the same + // offset as the least derived base. + for (unsigned I = Info->PathToBaseWithVPtr.size() - 1; I != 0; --I) { + const CXXRecordDecl *DerivedRD = Info->PathToBaseWithVPtr[I - 1]; + const CXXRecordDecl *BaseRD = Info->PathToBaseWithVPtr[I]; + + const ASTRecordLayout &Layout = + getContext().getASTRecordLayout(DerivedRD); + CharUnits Offset; + auto VBI = Layout.getVBaseOffsetsMap().find(BaseRD); + if (VBI == Layout.getVBaseOffsetsMap().end()) + Offset = Layout.getBaseClassOffset(BaseRD); + else + Offset = VBI->second.VBaseOffset; + if (!Offset.isZero()) + return; + BitsetsMD->addOperand( + CGM.CreateVTableBitSetEntry(VTable, PointerWidth, DerivedRD)); + } + + // Finally do the same for the most derived class. + if (Info->FullOffsetInMDC.isZero()) + BitsetsMD->addOperand( + CGM.CreateVTableBitSetEntry(VTable, PointerWidth, RD)); +} + void MicrosoftCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, const CXXRecordDecl *RD) { MicrosoftVTableContext &VFTContext = CGM.getMicrosoftVTableContext(); @@ -1423,6 +1478,8 @@ void MicrosoftCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, VTLayout.getNumVTableThunks(), RTTI); VTable->setInitializer(Init); + + emitVTableBitSetEntries(Info, RD, VTable); } } @@ -1585,10 +1642,54 @@ llvm::GlobalVariable *MicrosoftCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, return VTable; } +// Compute the identity of the most derived class whose virtual table is located +// at the given offset into RD. +static const CXXRecordDecl *getClassAtVTableLocation(ASTContext &Ctx, + const CXXRecordDecl *RD, + CharUnits Offset) { + if (Offset.isZero()) + return RD; + + const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD); + const CXXRecordDecl *MaxBase = nullptr; + CharUnits MaxBaseOffset; + for (auto &&B : RD->bases()) { + const CXXRecordDecl *Base = B.getType()->getAsCXXRecordDecl(); + CharUnits BaseOffset = Layout.getBaseClassOffset(Base); + if (BaseOffset <= Offset && BaseOffset > MaxBaseOffset) { + MaxBase = Base; + MaxBaseOffset = BaseOffset; + } + } + for (auto &&B : RD->vbases()) { + const CXXRecordDecl *Base = B.getType()->getAsCXXRecordDecl(); + CharUnits BaseOffset = Layout.getVBaseClassOffset(Base); + if (BaseOffset <= Offset && BaseOffset > MaxBaseOffset) { + MaxBase = Base; + MaxBaseOffset = BaseOffset; + } + } + assert(MaxBase); + return getClassAtVTableLocation(Ctx, MaxBase, Offset - MaxBaseOffset); +} + +// Compute the identity of the most derived class whose virtual table is located +// at the MethodVFTableLocation ML. +static const CXXRecordDecl * +getClassAtVTableLocation(ASTContext &Ctx, GlobalDecl GD, + MicrosoftVTableContext::MethodVFTableLocation &ML) { + const CXXRecordDecl *RD = ML.VBase; + if (!RD) + RD = cast<CXXMethodDecl>(GD.getDecl())->getParent(); + + return getClassAtVTableLocation(Ctx, RD, ML.VFPtrOffset); +} + llvm::Value *MicrosoftCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD, llvm::Value *This, - llvm::Type *Ty) { + llvm::Type *Ty, + SourceLocation Loc) { GD = GD.getCanonicalDecl(); CGBuilderTy &Builder = CGF.Builder; @@ -1599,6 +1700,10 @@ llvm::Value *MicrosoftCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, MicrosoftVTableContext::MethodVFTableLocation ML = CGM.getMicrosoftVTableContext().getMethodVFTableLocation(GD); + if (CGF.SanOpts.has(SanitizerKind::CFIVCall)) + CGF.EmitVTablePtrCheck(getClassAtVTableLocation(getContext(), GD, ML), + VTable, CodeGenFunction::CFITCK_VCall, Loc); + llvm::Value *VFuncPtr = Builder.CreateConstInBoundsGEP1_64(VTable, ML.Index, "vfn"); return Builder.CreateLoad(VFuncPtr); @@ -1616,7 +1721,8 @@ llvm::Value *MicrosoftCXXABI::EmitVirtualDestructorCall( const CGFunctionInfo *FInfo = &CGM.getTypes().arrangeCXXStructorDeclaration( Dtor, StructorType::Deleting); llvm::Type *Ty = CGF.CGM.getTypes().GetFunctionType(*FInfo); - llvm::Value *Callee = getVirtualFunctionPointer(CGF, GD, This, Ty); + llvm::Value *Callee = getVirtualFunctionPointer( + CGF, GD, This, Ty, CE ? CE->getLocStart() : SourceLocation()); ASTContext &Context = getContext(); llvm::Value *ImplicitParam = llvm::ConstantInt::get( @@ -2351,7 +2457,7 @@ MicrosoftCXXABI::EmitFullMemberPointer(llvm::Constant *FirstField, if (MSInheritanceAttr::hasVBPtrOffsetField(Inheritance)) { CharUnits Offs = CharUnits::Zero(); - if (RD->getNumVBases()) + if (VBTableIndex && RD->getNumVBases()) Offs = getContext().getASTRecordLayout(RD).getVBPtrOffset(); fields.push_back(llvm::ConstantInt::get(CGM.IntTy, Offs.getQuantity())); } @@ -2423,25 +2529,15 @@ MicrosoftCXXABI::BuildMemberPointer(const CXXRecordDecl *RD, FirstField = CGM.GetAddrOfFunction(MD, Ty); FirstField = llvm::ConstantExpr::getBitCast(FirstField, CGM.VoidPtrTy); } else { - if (!CGM.getTypes().isFuncTypeConvertible( - MD->getType()->castAs<FunctionType>())) { - CGM.ErrorUnsupported(MD, "pointer to virtual member function with " - "incomplete return or parameter type"); - FirstField = llvm::Constant::getNullValue(CGM.VoidPtrTy); - } else if (FPT->getCallConv() == CC_X86FastCall) { - CGM.ErrorUnsupported(MD, "pointer to fastcall virtual member function"); - FirstField = llvm::Constant::getNullValue(CGM.VoidPtrTy); - } else { - auto &VTableContext = CGM.getMicrosoftVTableContext(); - MicrosoftVTableContext::MethodVFTableLocation ML = - VTableContext.getMethodVFTableLocation(MD); - llvm::Function *Thunk = EmitVirtualMemPtrThunk(MD, ML); - FirstField = llvm::ConstantExpr::getBitCast(Thunk, CGM.VoidPtrTy); - // Include the vfptr adjustment if the method is in a non-primary vftable. - NonVirtualBaseAdjustment += ML.VFPtrOffset; - if (ML.VBase) - VBTableIndex = VTableContext.getVBTableIndex(RD, ML.VBase) * 4; - } + auto &VTableContext = CGM.getMicrosoftVTableContext(); + MicrosoftVTableContext::MethodVFTableLocation ML = + VTableContext.getMethodVFTableLocation(MD); + llvm::Function *Thunk = EmitVirtualMemPtrThunk(MD, ML); + FirstField = llvm::ConstantExpr::getBitCast(Thunk, CGM.VoidPtrTy); + // Include the vfptr adjustment if the method is in a non-primary vftable. + NonVirtualBaseAdjustment += ML.VFPtrOffset; + if (ML.VBase) + VBTableIndex = VTableContext.getVBTableIndex(RD, ML.VBase) * 4; } // The rest of the fields are common with data member pointers. diff --git a/contrib/llvm/tools/clang/lib/CodeGen/SanitizerMetadata.cpp b/contrib/llvm/tools/clang/lib/CodeGen/SanitizerMetadata.cpp index 7c38b28..2a338ba 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/SanitizerMetadata.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/SanitizerMetadata.cpp @@ -25,7 +25,8 @@ void SanitizerMetadata::reportGlobalToASan(llvm::GlobalVariable *GV, SourceLocation Loc, StringRef Name, QualType Ty, bool IsDynInit, bool IsBlacklisted) { - if (!CGM.getLangOpts().Sanitize.has(SanitizerKind::Address)) + if (!CGM.getLangOpts().Sanitize.hasOneOf(SanitizerKind::Address | + SanitizerKind::KernelAddress)) return; IsDynInit &= !CGM.isInSanitizerBlacklist(GV, Loc, Ty, "init"); IsBlacklisted |= CGM.isInSanitizerBlacklist(GV, Loc, Ty); @@ -56,7 +57,8 @@ void SanitizerMetadata::reportGlobalToASan(llvm::GlobalVariable *GV, void SanitizerMetadata::reportGlobalToASan(llvm::GlobalVariable *GV, const VarDecl &D, bool IsDynInit) { - if (!CGM.getLangOpts().Sanitize.has(SanitizerKind::Address)) + if (!CGM.getLangOpts().Sanitize.hasOneOf(SanitizerKind::Address | + SanitizerKind::KernelAddress)) return; std::string QualName; llvm::raw_string_ostream OS(QualName); @@ -67,7 +69,8 @@ void SanitizerMetadata::reportGlobalToASan(llvm::GlobalVariable *GV, void SanitizerMetadata::disableSanitizerForGlobal(llvm::GlobalVariable *GV) { // For now, just make sure the global is not modified by the ASan // instrumentation. - if (CGM.getLangOpts().Sanitize.has(SanitizerKind::Address)) + if (CGM.getLangOpts().Sanitize.hasOneOf(SanitizerKind::Address | + SanitizerKind::KernelAddress)) reportGlobalToASan(GV, SourceLocation(), "", QualType(), false, true); } |
