diff options
Diffstat (limited to 'sys/x86')
-rw-r--r-- | sys/x86/include/specialreg.h | 2 | ||||
-rw-r--r-- | sys/x86/include/x86_var.h | 2 | ||||
-rw-r--r-- | sys/x86/x86/cpu_machdep.c | 195 |
3 files changed, 199 insertions, 0 deletions
diff --git a/sys/x86/include/specialreg.h b/sys/x86/include/specialreg.h index bdd87ef..e17068f 100644 --- a/sys/x86/include/specialreg.h +++ b/sys/x86/include/specialreg.h @@ -385,6 +385,7 @@ /* * CPUID instruction 7 Structured Extended Features, leaf 0 edx info */ +#define CPUID_STDEXT3_MD_CLEAR 0x00000400 #define CPUID_STDEXT3_IBPB 0x04000000 #define CPUID_STDEXT3_STIBP 0x08000000 #define CPUID_STDEXT3_L1D_FLUSH 0x10000000 @@ -395,6 +396,7 @@ #define IA32_ARCH_CAP_RDCL_NO 0x00000001 #define IA32_ARCH_CAP_IBRS_ALL 0x00000002 #define IA32_ARCH_CAP_SSBD_NO 0x00000004 +#define IA32_ARCH_CAP_MDS_NO 0x00000020 /* * CPUID manufacturers identifiers diff --git a/sys/x86/include/x86_var.h b/sys/x86/include/x86_var.h index 8dd28f3..c17fe5c 100644 --- a/sys/x86/include/x86_var.h +++ b/sys/x86/include/x86_var.h @@ -83,6 +83,7 @@ extern int use_xsave; extern uint64_t xsave_mask; extern int pti; extern int hw_ibrs_active; +extern int hw_mds_disable; extern int hw_ssb_active; struct pcb; @@ -134,6 +135,7 @@ int isa_nmi(int cd); void handle_ibrs_entry(void); void handle_ibrs_exit(void); void hw_ibrs_recalculate(void); +void hw_mds_recalculate(void); void hw_ssb_recalculate(bool all_cpus); void nmi_call_kdb(u_int cpu, u_int type, struct trapframe *frame); void nmi_call_kdb_smp(u_int type, struct trapframe *frame); diff --git a/sys/x86/x86/cpu_machdep.c b/sys/x86/x86/cpu_machdep.c index 47d1d67..0604054 100644 --- a/sys/x86/x86/cpu_machdep.c +++ b/sys/x86/x86/cpu_machdep.c @@ -938,3 +938,198 @@ SYSCTL_PROC(_hw, OID_AUTO, spec_store_bypass_disable, CTLTYPE_INT | hw_ssb_disable_handler, "I", "Speculative Store Bypass Disable (0 - off, 1 - on, 2 - auto"); +int hw_mds_disable; + +/* + * Handler for Microarchitectural Data Sampling issues. Really not a + * pointer to C function: on amd64 the code must not change any CPU + * architectural state except possibly %rflags. Also, it is always + * called with interrupts disabled. + */ +void (*mds_handler)(void); +void mds_handler_void(void); +void mds_handler_verw(void); +void mds_handler_ivb(void); +void mds_handler_bdw(void); +void mds_handler_skl_sse(void); +void mds_handler_skl_avx(void); +void mds_handler_skl_avx512(void); +void mds_handler_silvermont(void); + +static int +sysctl_hw_mds_disable_state_handler(SYSCTL_HANDLER_ARGS) +{ + const char *state; + + if (mds_handler == mds_handler_void) + state = "inactive"; + else if (mds_handler == mds_handler_verw) + state = "VERW"; + else if (mds_handler == mds_handler_ivb) + state = "software IvyBridge"; + else if (mds_handler == mds_handler_bdw) + state = "software Broadwell"; + else if (mds_handler == mds_handler_skl_sse) + state = "software Skylake SSE"; + else if (mds_handler == mds_handler_skl_avx) + state = "software Skylake AVX"; + else if (mds_handler == mds_handler_skl_avx512) + state = "software Skylake AVX512"; + else if (mds_handler == mds_handler_silvermont) + state = "software Silvermont"; + else + state = "unknown"; + return (SYSCTL_OUT(req, state, strlen(state))); +} + +SYSCTL_PROC(_hw, OID_AUTO, mds_disable_state, + CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, + sysctl_hw_mds_disable_state_handler, "A", + "Microarchitectural Data Sampling Mitigation state"); + +_Static_assert(__offsetof(struct pcpu, pc_mds_tmp) % 64 == 0, "MDS AVX512"); + +void +hw_mds_recalculate(void) +{ + struct pcpu *pc; + vm_offset_t b64; + u_long xcr0; + int i; + + /* + * Allow user to force VERW variant even if MD_CLEAR is not + * reported. For instance, hypervisor might unknowingly + * filter the cap out. + * For the similar reasons, and for testing, allow to enable + * mitigation even for RDCL_NO or MDS_NO caps. + */ + if (cpu_vendor_id != CPU_VENDOR_INTEL || hw_mds_disable == 0 || + ((cpu_ia32_arch_caps & (IA32_ARCH_CAP_RDCL_NO | + IA32_ARCH_CAP_MDS_NO)) != 0 && hw_mds_disable == 3)) { + mds_handler = mds_handler_void; + } else if (((cpu_stdext_feature3 & CPUID_STDEXT3_MD_CLEAR) != 0 && + hw_mds_disable == 3) || hw_mds_disable == 1) { + mds_handler = mds_handler_verw; + } else if (CPUID_TO_FAMILY(cpu_id) == 0x6 && + (CPUID_TO_MODEL(cpu_id) == 0x2e || CPUID_TO_MODEL(cpu_id) == 0x1e || + CPUID_TO_MODEL(cpu_id) == 0x1f || CPUID_TO_MODEL(cpu_id) == 0x1a || + CPUID_TO_MODEL(cpu_id) == 0x2f || CPUID_TO_MODEL(cpu_id) == 0x25 || + CPUID_TO_MODEL(cpu_id) == 0x2c || CPUID_TO_MODEL(cpu_id) == 0x2d || + CPUID_TO_MODEL(cpu_id) == 0x2a || CPUID_TO_MODEL(cpu_id) == 0x3e || + CPUID_TO_MODEL(cpu_id) == 0x3a) && + (hw_mds_disable == 2 || hw_mds_disable == 3)) { + /* + * Nehalem, SandyBridge, IvyBridge + */ + CPU_FOREACH(i) { + pc = pcpu_find(i); + if (pc->pc_mds_buf == NULL) { + pc->pc_mds_buf = malloc(672, M_TEMP, + M_WAITOK); + bzero(pc->pc_mds_buf, 16); + } + } + mds_handler = mds_handler_ivb; + } else if (CPUID_TO_FAMILY(cpu_id) == 0x6 && + (CPUID_TO_MODEL(cpu_id) == 0x3f || CPUID_TO_MODEL(cpu_id) == 0x3c || + CPUID_TO_MODEL(cpu_id) == 0x45 || CPUID_TO_MODEL(cpu_id) == 0x46 || + CPUID_TO_MODEL(cpu_id) == 0x56 || CPUID_TO_MODEL(cpu_id) == 0x4f || + CPUID_TO_MODEL(cpu_id) == 0x47 || CPUID_TO_MODEL(cpu_id) == 0x3d) && + (hw_mds_disable == 2 || hw_mds_disable == 3)) { + /* + * Haswell, Broadwell + */ + CPU_FOREACH(i) { + pc = pcpu_find(i); + if (pc->pc_mds_buf == NULL) { + pc->pc_mds_buf = malloc(1536, M_TEMP, + M_WAITOK); + bzero(pc->pc_mds_buf, 16); + } + } + mds_handler = mds_handler_bdw; + } else if (CPUID_TO_FAMILY(cpu_id) == 0x6 && + ((CPUID_TO_MODEL(cpu_id) == 0x55 && (cpu_id & + CPUID_STEPPING) <= 5) || + CPUID_TO_MODEL(cpu_id) == 0x4e || CPUID_TO_MODEL(cpu_id) == 0x5e || + (CPUID_TO_MODEL(cpu_id) == 0x8e && (cpu_id & + CPUID_STEPPING) <= 0xb) || + (CPUID_TO_MODEL(cpu_id) == 0x9e && (cpu_id & + CPUID_STEPPING) <= 0xc)) && + (hw_mds_disable == 2 || hw_mds_disable == 3)) { + /* + * Skylake, KabyLake, CoffeeLake, WhiskeyLake, + * CascadeLake + */ + CPU_FOREACH(i) { + pc = pcpu_find(i); + if (pc->pc_mds_buf == NULL) { + pc->pc_mds_buf = malloc(6 * 1024, + M_TEMP, M_WAITOK); + b64 = (vm_offset_t)malloc(64 + 63, + M_TEMP, M_WAITOK); + pc->pc_mds_buf64 = (void *)roundup2(b64, 64); + bzero(pc->pc_mds_buf64, 64); + } + } + xcr0 = rxcr(0); + if ((xcr0 & XFEATURE_ENABLED_ZMM_HI256) != 0 && + (cpu_stdext_feature2 & CPUID_STDEXT_AVX512DQ) != 0) + mds_handler = mds_handler_skl_avx512; + else if ((xcr0 & XFEATURE_ENABLED_AVX) != 0 && + (cpu_feature2 & CPUID2_AVX) != 0) + mds_handler = mds_handler_skl_avx; + else + mds_handler = mds_handler_skl_sse; + } else if (CPUID_TO_FAMILY(cpu_id) == 0x6 && + ((CPUID_TO_MODEL(cpu_id) == 0x37 || + CPUID_TO_MODEL(cpu_id) == 0x4a || + CPUID_TO_MODEL(cpu_id) == 0x4c || + CPUID_TO_MODEL(cpu_id) == 0x4d || + CPUID_TO_MODEL(cpu_id) == 0x5a || + CPUID_TO_MODEL(cpu_id) == 0x5d || + CPUID_TO_MODEL(cpu_id) == 0x6e || + CPUID_TO_MODEL(cpu_id) == 0x65 || + CPUID_TO_MODEL(cpu_id) == 0x75 || + CPUID_TO_MODEL(cpu_id) == 0x1c || + CPUID_TO_MODEL(cpu_id) == 0x26 || + CPUID_TO_MODEL(cpu_id) == 0x27 || + CPUID_TO_MODEL(cpu_id) == 0x35 || + CPUID_TO_MODEL(cpu_id) == 0x36 || + CPUID_TO_MODEL(cpu_id) == 0x7a))) { + /* Silvermont, Airmont */ + CPU_FOREACH(i) { + pc = pcpu_find(i); + if (pc->pc_mds_buf == NULL) + pc->pc_mds_buf = malloc(256, M_TEMP, M_WAITOK); + } + mds_handler = mds_handler_silvermont; + } else { + hw_mds_disable = 0; + mds_handler = mds_handler_void; + } +} + +static int +sysctl_mds_disable_handler(SYSCTL_HANDLER_ARGS) +{ + int error, val; + + val = hw_mds_disable; + error = sysctl_handle_int(oidp, &val, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + if (val < 0 || val > 3) + return (EINVAL); + hw_mds_disable = val; + hw_mds_recalculate(); + return (0); +} + +SYSCTL_PROC(_hw, OID_AUTO, mds_disable, CTLTYPE_INT | + CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, NULL, 0, + sysctl_mds_disable_handler, "I", + "Microarchitectural Data Sampling Mitigation " + "(0 - off, 1 - on VERW, 2 - on SW, 3 - on AUTO"); + |