diff options
Diffstat (limited to 'sys/amd64/include')
| -rw-r--r-- | sys/amd64/include/asmacros.h | 194 | ||||
| -rw-r--r-- | sys/amd64/include/frame.h | 48 | ||||
| -rw-r--r-- | sys/amd64/include/intr_machdep.h | 2 | ||||
| -rw-r--r-- | sys/amd64/include/md_var.h | 7 | ||||
| -rw-r--r-- | sys/amd64/include/pcb.h | 2 | ||||
| -rw-r--r-- | sys/amd64/include/pcpu.h | 9 | ||||
| -rw-r--r-- | sys/amd64/include/pmap.h | 12 | ||||
| -rw-r--r-- | sys/amd64/include/smp.h | 28 |
8 files changed, 70 insertions, 232 deletions
diff --git a/sys/amd64/include/asmacros.h b/sys/amd64/include/asmacros.h index cd7acd8..d463bfe 100644 --- a/sys/amd64/include/asmacros.h +++ b/sys/amd64/include/asmacros.h @@ -1,15 +1,7 @@ -/* -*- mode: asm -*- */ /*- * Copyright (c) 1993 The Regents of the University of California. * All rights reserved. * - * Copyright (c) 2018 The FreeBSD Foundation - * All rights reserved. - * - * Portions of this software were developed by - * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from - * the FreeBSD Foundation. - * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: @@ -152,135 +144,75 @@ #ifdef LOCORE /* - * Access per-CPU data. - */ -#define PCPU(member) %gs:PC_ ## member -#define PCPU_ADDR(member, reg) \ - movq %gs:PC_PRVSPACE, reg ; \ - addq $PC_ ## member, reg - -/* * Convenience macro for declaring interrupt entry points. */ #define IDTVEC(name) ALIGN_TEXT; .globl __CONCAT(X,name); \ .type __CONCAT(X,name),@function; __CONCAT(X,name): - .macro SAVE_SEGS - movw %fs,TF_FS(%rsp) - movw %gs,TF_GS(%rsp) - movw %es,TF_ES(%rsp) - movw %ds,TF_DS(%rsp) - .endm - - .macro MOVE_STACKS qw - .L.offset=0 - .rept \qw - movq .L.offset(%rsp),%rdx - movq %rdx,.L.offset(%rax) - .L.offset=.L.offset+8 - .endr - .endm - - .macro PTI_UUENTRY has_err - movq PCPU(KCR3),%rax - movq %rax,%cr3 - movq PCPU(RSP0),%rax - subq $PTI_SIZE,%rax - MOVE_STACKS ((PTI_SIZE / 8) - 1 + \has_err) - movq %rax,%rsp - popq %rdx - popq %rax - .endm - - .macro PTI_UENTRY has_err - swapgs - pushq %rax - pushq %rdx - PTI_UUENTRY \has_err - .endm - - .macro PTI_ENTRY name, cont, has_err=0 - ALIGN_TEXT - .globl X\name\()_pti - .type X\name\()_pti,@function -X\name\()_pti: - /* %rax, %rdx and possibly err not yet pushed */ - testb $SEL_RPL_MASK,PTI_CS-(2+1-\has_err)*8(%rsp) - jz \cont - PTI_UENTRY \has_err - swapgs - jmp \cont - .endm - - .macro PTI_INTRENTRY vec_name - SUPERALIGN_TEXT - .globl X\vec_name\()_pti - .type X\vec_name\()_pti,@function -X\vec_name\()_pti: - testb $SEL_RPL_MASK,PTI_CS-3*8(%rsp) /* err, %rax, %rdx not pushed */ - jz \vec_name\()_u - PTI_UENTRY has_err=0 - jmp \vec_name\()_u - .endm - - .macro INTR_PUSH_FRAME vec_name - SUPERALIGN_TEXT - .globl X\vec_name - .type X\vec_name,@function -X\vec_name: - testb $SEL_RPL_MASK,PTI_CS-3*8(%rsp) /* come from kernel? */ - jz \vec_name\()_u /* Yes, dont swapgs again */ - swapgs -\vec_name\()_u: - subq $TF_RIP,%rsp /* skip dummy tf_err and tf_trapno */ - movq %rdi,TF_RDI(%rsp) - movq %rsi,TF_RSI(%rsp) - movq %rdx,TF_RDX(%rsp) - movq %rcx,TF_RCX(%rsp) - movq %r8,TF_R8(%rsp) - movq %r9,TF_R9(%rsp) - movq %rax,TF_RAX(%rsp) - movq %rbx,TF_RBX(%rsp) - movq %rbp,TF_RBP(%rsp) - movq %r10,TF_R10(%rsp) - movq %r11,TF_R11(%rsp) - movq %r12,TF_R12(%rsp) - movq %r13,TF_R13(%rsp) - movq %r14,TF_R14(%rsp) - movq %r15,TF_R15(%rsp) - SAVE_SEGS - movl $TF_HASSEGS,TF_FLAGS(%rsp) - cld - testb $SEL_RPL_MASK,TF_CS(%rsp) /* come from kernel ? */ - jz 1f /* yes, leave PCB_FULL_IRET alone */ - movq PCPU(CURPCB),%r8 - andl $~PCB_FULL_IRET,PCB_FLAGS(%r8) -1: - .endm - - .macro INTR_HANDLER vec_name - .text - PTI_INTRENTRY \vec_name - INTR_PUSH_FRAME \vec_name - .endm +/* + * Macros to create and destroy a trap frame. + */ +#define PUSH_FRAME \ + subq $TF_RIP,%rsp ; /* skip dummy tf_err and tf_trapno */ \ + testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \ + jz 1f ; /* Yes, dont swapgs again */ \ + swapgs ; \ +1: movq %rdi,TF_RDI(%rsp) ; \ + movq %rsi,TF_RSI(%rsp) ; \ + movq %rdx,TF_RDX(%rsp) ; \ + movq %rcx,TF_RCX(%rsp) ; \ + movq %r8,TF_R8(%rsp) ; \ + movq %r9,TF_R9(%rsp) ; \ + movq %rax,TF_RAX(%rsp) ; \ + movq %rbx,TF_RBX(%rsp) ; \ + movq %rbp,TF_RBP(%rsp) ; \ + movq %r10,TF_R10(%rsp) ; \ + movq %r11,TF_R11(%rsp) ; \ + movq %r12,TF_R12(%rsp) ; \ + movq %r13,TF_R13(%rsp) ; \ + movq %r14,TF_R14(%rsp) ; \ + movq %r15,TF_R15(%rsp) ; \ + movw %fs,TF_FS(%rsp) ; \ + movw %gs,TF_GS(%rsp) ; \ + movw %es,TF_ES(%rsp) ; \ + movw %ds,TF_DS(%rsp) ; \ + movl $TF_HASSEGS,TF_FLAGS(%rsp) ; \ + cld ; \ + testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel ? */ \ + jz 2f ; /* yes, leave PCB_FULL_IRET alone */ \ + movq PCPU(CURPCB),%r8 ; \ + andl $~PCB_FULL_IRET,PCB_FLAGS(%r8) ; \ +2: + +#define POP_FRAME \ + movq TF_RDI(%rsp),%rdi ; \ + movq TF_RSI(%rsp),%rsi ; \ + movq TF_RDX(%rsp),%rdx ; \ + movq TF_RCX(%rsp),%rcx ; \ + movq TF_R8(%rsp),%r8 ; \ + movq TF_R9(%rsp),%r9 ; \ + movq TF_RAX(%rsp),%rax ; \ + movq TF_RBX(%rsp),%rbx ; \ + movq TF_RBP(%rsp),%rbp ; \ + movq TF_R10(%rsp),%r10 ; \ + movq TF_R11(%rsp),%r11 ; \ + movq TF_R12(%rsp),%r12 ; \ + movq TF_R13(%rsp),%r13 ; \ + movq TF_R14(%rsp),%r14 ; \ + movq TF_R15(%rsp),%r15 ; \ + testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \ + jz 1f ; /* keep kernel GS.base */ \ + cli ; \ + swapgs ; \ +1: addq $TF_RIP,%rsp /* skip over tf_err, tf_trapno */ - .macro RESTORE_REGS - movq TF_RDI(%rsp),%rdi - movq TF_RSI(%rsp),%rsi - movq TF_RDX(%rsp),%rdx - movq TF_RCX(%rsp),%rcx - movq TF_R8(%rsp),%r8 - movq TF_R9(%rsp),%r9 - movq TF_RAX(%rsp),%rax - movq TF_RBX(%rsp),%rbx - movq TF_RBP(%rsp),%rbp - movq TF_R10(%rsp),%r10 - movq TF_R11(%rsp),%r11 - movq TF_R12(%rsp),%r12 - movq TF_R13(%rsp),%r13 - movq TF_R14(%rsp),%r14 - movq TF_R15(%rsp),%r15 - .endm +/* + * Access per-CPU data. + */ +#define PCPU(member) %gs:PC_ ## member +#define PCPU_ADDR(member, reg) \ + movq %gs:PC_PRVSPACE, reg ; \ + addq $PC_ ## member, reg #endif /* LOCORE */ diff --git a/sys/amd64/include/frame.h b/sys/amd64/include/frame.h index f0a6fcf..0953be7 100644 --- a/sys/amd64/include/frame.h +++ b/sys/amd64/include/frame.h @@ -1,50 +1,6 @@ /*- - * SPDX-License-Identifier: BSD-2-Clause-FreeBSD - * - * Copyright (c) 2018 The FreeBSD Foundation - * All rights reserved. - * - * This software was developed by Konstantin Belousov <kib@FreeBSD.org> - * under sponsorship from the FreeBSD Foundation. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ + * This file is in the public domain. */ - -#ifndef _AMD64_FRAME_H -#define _AMD64_FRAME_H +/* $FreeBSD$ */ #include <x86/frame.h> - -struct pti_frame { - register_t pti_rdx; - register_t pti_rax; - register_t pti_err; - register_t pti_rip; - register_t pti_cs; - register_t pti_rflags; - register_t pti_rsp; - register_t pti_ss; -}; - -#endif diff --git a/sys/amd64/include/intr_machdep.h b/sys/amd64/include/intr_machdep.h index 29c20b6..e7320e6 100644 --- a/sys/amd64/include/intr_machdep.h +++ b/sys/amd64/include/intr_machdep.h @@ -136,7 +136,7 @@ struct trapframe; /* * The following data structure holds per-cpu data, and is placed just - * above the top of the space used for the NMI and MC# stacks. + * above the top of the space used for the NMI stack. */ struct nmi_pcpu { register_t np_pcpu; diff --git a/sys/amd64/include/md_var.h b/sys/amd64/include/md_var.h index f4d6e60..7a84631 100644 --- a/sys/amd64/include/md_var.h +++ b/sys/amd64/include/md_var.h @@ -36,13 +36,6 @@ extern uint64_t *vm_page_dump; extern int hw_lower_amd64_sharedpage; -extern int hw_ibrs_disable; - -/* - * The file "conf/ldscript.amd64" defines the symbol "kernphys". Its - * value is the physical address at which the kernel is loaded. - */ -extern char kernphys[]; struct savefpu; struct sysentvec; diff --git a/sys/amd64/include/pcb.h b/sys/amd64/include/pcb.h index 09aea36..8d0dce9 100644 --- a/sys/amd64/include/pcb.h +++ b/sys/amd64/include/pcb.h @@ -90,7 +90,7 @@ struct pcb { /* copyin/out fault recovery */ caddr_t pcb_onfault; - uint64_t pcb_saved_ucr3; + uint64_t pcb_pad0; /* local tss, with i/o bitmap; NULL for common */ struct amd64tss *pcb_tssp; diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h index e40c521..a4f4e1d 100644 --- a/sys/amd64/include/pcpu.h +++ b/sys/amd64/include/pcpu.h @@ -33,7 +33,6 @@ #error "sys/cdefs.h is a prerequisite for this file" #endif -#define PC_PTI_STACK_SZ 16 /* * The SMP parts are setup in pmap.c and locore.s for the BSP, and * mp_machdep.c sets up the data for the AP's to "see" when they awake. @@ -47,12 +46,8 @@ struct pmap *pc_curpmap; \ struct amd64tss *pc_tssp; /* TSS segment active on CPU */ \ struct amd64tss *pc_commontssp;/* Common TSS for the CPU */ \ - uint64_t pc_kcr3; \ - uint64_t pc_ucr3; \ - uint64_t pc_saved_ucr3; \ register_t pc_rsp0; \ register_t pc_scratch_rsp; /* User %rsp in syscall */ \ - register_t pc_scratch_rax; \ u_int pc_apic_id; \ u_int pc_acpi_id; /* ACPI CPU id */ \ /* Pointer to the CPU %fs descriptor */ \ @@ -66,14 +61,12 @@ uint64_t pc_pm_save_cnt; \ u_int pc_cmci_mask; /* MCx banks for CMCI */ \ uint64_t pc_dbreg[16]; /* ddb debugging regs */ \ - uint64_t pc_pti_stack[PC_PTI_STACK_SZ]; \ int pc_dbreg_cmd; /* ddb debugging reg cmd */ \ u_int pc_vcpu_id; /* Xen vCPU ID */ \ uint32_t pc_pcid_next; \ uint32_t pc_pcid_gen; \ uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \ - uint32_t pc_ibpb_set; \ - char __pad[96] /* be divisor of PAGE_SIZE \ + char __pad[145] /* be divisor of PAGE_SIZE \ after cache alignment */ #define PC_DBREG_CMD_NONE 0 diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h index acf0301..a0b8ee3 100644 --- a/sys/amd64/include/pmap.h +++ b/sys/amd64/include/pmap.h @@ -223,10 +223,6 @@ #define PMAP_PCID_NONE 0xffffffff #define PMAP_PCID_KERN 0 #define PMAP_PCID_OVERMAX 0x1000 -#define PMAP_PCID_OVERMAX_KERN 0x800 -#define PMAP_PCID_USER_PT 0x800 - -#define PMAP_NO_CR3 (~0UL) #ifndef LOCORE @@ -317,9 +313,7 @@ struct pmap_pcids { struct pmap { struct mtx pm_mtx; pml4_entry_t *pm_pml4; /* KVA of level 4 page table */ - pml4_entry_t *pm_pml4u; /* KVA of user l4 page table */ uint64_t pm_cr3; - uint64_t pm_ucr3; TAILQ_HEAD(,pv_chunk) pm_pvchunk; /* list of mappings in pmap */ cpuset_t pm_active; /* active on cpus */ enum pmap_type pm_type; /* regular or nested tables */ @@ -425,12 +419,6 @@ void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva, void pmap_get_mapping(pmap_t pmap, vm_offset_t va, uint64_t *ptr, int *num); boolean_t pmap_map_io_transient(vm_page_t *, vm_offset_t *, int, boolean_t); void pmap_unmap_io_transient(vm_page_t *, vm_offset_t *, int, boolean_t); -void pmap_pti_add_kva(vm_offset_t sva, vm_offset_t eva, bool exec); -void pmap_pti_remove_kva(vm_offset_t sva, vm_offset_t eva); -void pmap_pti_pcid_invalidate(uint64_t ucr3, uint64_t kcr3); -void pmap_pti_pcid_invlpg(uint64_t ucr3, uint64_t kcr3, vm_offset_t va); -void pmap_pti_pcid_invlrng(uint64_t ucr3, uint64_t kcr3, vm_offset_t sva, - vm_offset_t eva); #endif /* _KERNEL */ /* Return various clipped indexes for a given VA */ diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h index 64135bc..d97c730 100644 --- a/sys/amd64/include/smp.h +++ b/sys/amd64/include/smp.h @@ -28,36 +28,12 @@ extern u_int32_t mptramp_pagetables; /* IPI handlers */ inthand_t - IDTVEC(justreturn), /* interrupt CPU with minimum overhead */ - IDTVEC(justreturn1_pti), - IDTVEC(invltlb_pti), - IDTVEC(invltlb_pcid_pti), IDTVEC(invltlb_pcid), /* TLB shootdowns - global, pcid */ - IDTVEC(invltlb_invpcid_pti_pti), - IDTVEC(invltlb_invpcid_nopti), - IDTVEC(invlpg_pti), - IDTVEC(invlpg_invpcid_pti), - IDTVEC(invlpg_invpcid), - IDTVEC(invlpg_pcid_pti), - IDTVEC(invlpg_pcid), - IDTVEC(invlrng_pti), - IDTVEC(invlrng_invpcid_pti), - IDTVEC(invlrng_invpcid), - IDTVEC(invlrng_pcid_pti), - IDTVEC(invlrng_pcid), - IDTVEC(invlcache_pti), - IDTVEC(ipi_intr_bitmap_handler_pti), - IDTVEC(cpustop_pti), - IDTVEC(cpususpend_pti), - IDTVEC(rendezvous_pti); + IDTVEC(invltlb_invpcid),/* TLB shootdowns - global, invpcid */ + IDTVEC(justreturn); /* interrupt CPU with minimum overhead */ void invltlb_pcid_handler(void); void invltlb_invpcid_handler(void); -void invltlb_invpcid_pti_handler(void); -void invlpg_invpcid_handler(void); -void invlpg_pcid_handler(void); -void invlrng_invpcid_handler(void); -void invlrng_pcid_handler(void); int native_start_all_aps(void); #endif /* !LOCORE */ |
