diff options
| author | peter <peter@FreeBSD.org> | 2003-05-01 01:05:25 +0000 |
|---|---|---|
| committer | peter <peter@FreeBSD.org> | 2003-05-01 01:05:25 +0000 |
| commit | 45949ccde13fb04ed597a5aef80b678ba16bcab7 (patch) | |
| tree | dd665cefeba0e426ad2b212b76851de96f1ad18b /sys/amd64/include | |
| parent | 1fd7bc609e9a5249f7b7b558405e1c39aca79796 (diff) | |
| download | FreeBSD-src-45949ccde13fb04ed597a5aef80b678ba16bcab7.zip FreeBSD-src-45949ccde13fb04ed597a5aef80b678ba16bcab7.tar.gz | |
Commit MD parts of a loosely functional AMD64 port. This is based on
a heavily stripped down FreeBSD/i386 (brutally stripped down actually) to
attempt to get a stable base to start from. There is a lot missing still.
Worth noting:
- The kernel runs at 1GB in order to cheat with the pmap code. pmap uses
a variation of the PAE code in order to avoid having to worry about 4
levels of page tables yet.
- It boots in 64 bit "long mode" with a tiny trampoline embedded in the
i386 loader. This simplifies locore.s greatly.
- There are still quite a few fragments of i386-specific code that have
not been translated yet, and some that I cheated and wrote dumb C
versions of (bcopy etc).
- It has both int 0x80 for syscalls (but using registers for argument
passing, as is native on the amd64 ABI), and the 'syscall' instruction
for syscalls. int 0x80 preserves all registers, 'syscall' does not.
- I have tried to minimize looking at the NetBSD code, except in a couple
of places (eg: to find which register they use to replace the trashed
%rcx register in the syscall instruction). As a result, there is not a
lot of similarity. I did look at NetBSD a few times while debugging to
get some ideas about what I might have done wrong in my first attempt.
Diffstat (limited to 'sys/amd64/include')
42 files changed, 719 insertions, 1642 deletions
diff --git a/sys/amd64/include/_inttypes.h b/sys/amd64/include/_inttypes.h index 7da589d..e6b2536 100644 --- a/sys/amd64/include/_inttypes.h +++ b/sys/amd64/include/_inttypes.h @@ -49,172 +49,172 @@ #define PRId8 "d" /* int8_t */ #define PRId16 "d" /* int16_t */ #define PRId32 "d" /* int32_t */ -#define PRId64 "lld" /* int64_t */ +#define PRId64 "ld" /* int64_t */ #define PRIdLEAST8 "d" /* int_least8_t */ #define PRIdLEAST16 "d" /* int_least16_t */ #define PRIdLEAST32 "d" /* int_least32_t */ -#define PRIdLEAST64 "lld" /* int_least64_t */ +#define PRIdLEAST64 "ld" /* int_least64_t */ #define PRIdFAST8 "d" /* int_fast8_t */ #define PRIdFAST16 "d" /* int_fast16_t */ #define PRIdFAST32 "d" /* int_fast32_t */ -#define PRIdFAST64 "lld" /* int_fast64_t */ +#define PRIdFAST64 "ld" /* int_fast64_t */ #define PRIdMAX "jd" /* intmax_t */ -#define PRIdPTR "d" /* intptr_t */ +#define PRIdPTR "ld" /* intptr_t */ #define PRIi8 "i" /* int8_t */ #define PRIi16 "i" /* int16_t */ #define PRIi32 "i" /* int32_t */ -#define PRIi64 "lli" /* int64_t */ +#define PRIi64 "li" /* int64_t */ #define PRIiLEAST8 "i" /* int_least8_t */ #define PRIiLEAST16 "i" /* int_least16_t */ #define PRIiLEAST32 "i" /* int_least32_t */ -#define PRIiLEAST64 "lli" /* int_least64_t */ +#define PRIiLEAST64 "li" /* int_least64_t */ #define PRIiFAST8 "i" /* int_fast8_t */ #define PRIiFAST16 "i" /* int_fast16_t */ #define PRIiFAST32 "i" /* int_fast32_t */ -#define PRIiFAST64 "lli" /* int_fast64_t */ +#define PRIiFAST64 "li" /* int_fast64_t */ #define PRIiMAX "ji" /* intmax_t */ -#define PRIiPTR "i" /* intptr_t */ +#define PRIiPTR "li" /* intptr_t */ /* fprintf(3) macros for unsigned integers. */ #define PRIo8 "o" /* uint8_t */ #define PRIo16 "o" /* uint16_t */ #define PRIo32 "o" /* uint32_t */ -#define PRIo64 "llo" /* uint64_t */ +#define PRIo64 "lo" /* uint64_t */ #define PRIoLEAST8 "o" /* uint_least8_t */ #define PRIoLEAST16 "o" /* uint_least16_t */ #define PRIoLEAST32 "o" /* uint_least32_t */ -#define PRIoLEAST64 "llo" /* uint_least64_t */ +#define PRIoLEAST64 "lo" /* uint_least64_t */ #define PRIoFAST8 "o" /* uint_fast8_t */ #define PRIoFAST16 "o" /* uint_fast16_t */ #define PRIoFAST32 "o" /* uint_fast32_t */ -#define PRIoFAST64 "llo" /* uint_fast64_t */ +#define PRIoFAST64 "lo" /* uint_fast64_t */ #define PRIoMAX "jo" /* uintmax_t */ -#define PRIoPTR "o" /* uintptr_t */ +#define PRIoPTR "lo" /* uintptr_t */ #define PRIu8 "u" /* uint8_t */ #define PRIu16 "u" /* uint16_t */ #define PRIu32 "u" /* uint32_t */ -#define PRIu64 "llu" /* uint64_t */ +#define PRIu64 "lu" /* uint64_t */ #define PRIuLEAST8 "u" /* uint_least8_t */ #define PRIuLEAST16 "u" /* uint_least16_t */ #define PRIuLEAST32 "u" /* uint_least32_t */ -#define PRIuLEAST64 "llu" /* uint_least64_t */ +#define PRIuLEAST64 "lu" /* uint_least64_t */ #define PRIuFAST8 "u" /* uint_fast8_t */ #define PRIuFAST16 "u" /* uint_fast16_t */ #define PRIuFAST32 "u" /* uint_fast32_t */ -#define PRIuFAST64 "llu" /* uint_fast64_t */ +#define PRIuFAST64 "lu" /* uint_fast64_t */ #define PRIuMAX "ju" /* uintmax_t */ -#define PRIuPTR "u" /* uintptr_t */ +#define PRIuPTR "lu" /* uintptr_t */ #define PRIx8 "x" /* uint8_t */ #define PRIx16 "x" /* uint16_t */ #define PRIx32 "x" /* uint32_t */ -#define PRIx64 "llx" /* uint64_t */ +#define PRIx64 "lx" /* uint64_t */ #define PRIxLEAST8 "x" /* uint_least8_t */ #define PRIxLEAST16 "x" /* uint_least16_t */ #define PRIxLEAST32 "x" /* uint_least32_t */ -#define PRIxLEAST64 "llx" /* uint_least64_t */ +#define PRIxLEAST64 "lx" /* uint_least64_t */ #define PRIxFAST8 "x" /* uint_fast8_t */ #define PRIxFAST16 "x" /* uint_fast16_t */ #define PRIxFAST32 "x" /* uint_fast32_t */ -#define PRIxFAST64 "llx" /* uint_fast64_t */ +#define PRIxFAST64 "lx" /* uint_fast64_t */ #define PRIxMAX "jx" /* uintmax_t */ -#define PRIxPTR "x" /* uintptr_t */ +#define PRIxPTR "lx" /* uintptr_t */ #define PRIX8 "X" /* uint8_t */ #define PRIX16 "X" /* uint16_t */ #define PRIX32 "X" /* uint32_t */ -#define PRIX64 "llX" /* uint64_t */ +#define PRIX64 "lX" /* uint64_t */ #define PRIXLEAST8 "X" /* uint_least8_t */ #define PRIXLEAST16 "X" /* uint_least16_t */ #define PRIXLEAST32 "X" /* uint_least32_t */ -#define PRIXLEAST64 "llX" /* uint_least64_t */ +#define PRIXLEAST64 "lX" /* uint_least64_t */ #define PRIXFAST8 "X" /* uint_fast8_t */ #define PRIXFAST16 "X" /* uint_fast16_t */ #define PRIXFAST32 "X" /* uint_fast32_t */ -#define PRIXFAST64 "llX" /* uint_fast64_t */ +#define PRIXFAST64 "lX" /* uint_fast64_t */ #define PRIXMAX "jX" /* uintmax_t */ -#define PRIXPTR "X" /* uintptr_t */ +#define PRIXPTR "lX" /* uintptr_t */ /* fscanf(3) macros for signed integers. */ #define SCNd8 "hhd" /* int8_t */ #define SCNd16 "hd" /* int16_t */ #define SCNd32 "d" /* int32_t */ -#define SCNd64 "lld" /* int64_t */ +#define SCNd64 "ld" /* int64_t */ #define SCNdLEAST8 "hhd" /* int_least8_t */ #define SCNdLEAST16 "hd" /* int_least16_t */ #define SCNdLEAST32 "d" /* int_least32_t */ -#define SCNdLEAST64 "lld" /* int_least64_t */ +#define SCNdLEAST64 "ld" /* int_least64_t */ #define SCNdFAST8 "d" /* int_fast8_t */ #define SCNdFAST16 "d" /* int_fast16_t */ #define SCNdFAST32 "d" /* int_fast32_t */ -#define SCNdFAST64 "lld" /* int_fast64_t */ +#define SCNdFAST64 "ld" /* int_fast64_t */ #define SCNdMAX "jd" /* intmax_t */ -#define SCNdPTR "d" /* intptr_t */ +#define SCNdPTR "ld" /* intptr_t */ #define SCNi8 "hhi" /* int8_t */ #define SCNi16 "hi" /* int16_t */ #define SCNi32 "i" /* int32_t */ -#define SCNi64 "lli" /* int64_t */ +#define SCNi64 "li" /* int64_t */ #define SCNiLEAST8 "hhi" /* int_least8_t */ #define SCNiLEAST16 "hi" /* int_least16_t */ #define SCNiLEAST32 "i" /* int_least32_t */ -#define SCNiLEAST64 "lli" /* int_least64_t */ +#define SCNiLEAST64 "li" /* int_least64_t */ #define SCNiFAST8 "i" /* int_fast8_t */ #define SCNiFAST16 "i" /* int_fast16_t */ #define SCNiFAST32 "i" /* int_fast32_t */ -#define SCNiFAST64 "lli" /* int_fast64_t */ +#define SCNiFAST64 "li" /* int_fast64_t */ #define SCNiMAX "ji" /* intmax_t */ -#define SCNiPTR "i" /* intptr_t */ +#define SCNiPTR "li" /* intptr_t */ /* fscanf(3) macros for unsigned integers. */ #define SCNo8 "hho" /* uint8_t */ #define SCNo16 "ho" /* uint16_t */ #define SCNo32 "o" /* uint32_t */ -#define SCNo64 "llo" /* uint64_t */ +#define SCNo64 "lo" /* uint64_t */ #define SCNoLEAST8 "hho" /* uint_least8_t */ #define SCNoLEAST16 "ho" /* uint_least16_t */ #define SCNoLEAST32 "o" /* uint_least32_t */ -#define SCNoLEAST64 "llo" /* uint_least64_t */ +#define SCNoLEAST64 "lo" /* uint_least64_t */ #define SCNoFAST8 "o" /* uint_fast8_t */ #define SCNoFAST16 "o" /* uint_fast16_t */ #define SCNoFAST32 "o" /* uint_fast32_t */ -#define SCNoFAST64 "llo" /* uint_fast64_t */ +#define SCNoFAST64 "lo" /* uint_fast64_t */ #define SCNoMAX "jo" /* uintmax_t */ -#define SCNoPTR "o" /* uintptr_t */ +#define SCNoPTR "lo" /* uintptr_t */ #define SCNu8 "hhu" /* uint8_t */ #define SCNu16 "hu" /* uint16_t */ #define SCNu32 "u" /* uint32_t */ -#define SCNu64 "llu" /* uint64_t */ +#define SCNu64 "lu" /* uint64_t */ #define SCNuLEAST8 "hhu" /* uint_least8_t */ #define SCNuLEAST16 "hu" /* uint_least16_t */ #define SCNuLEAST32 "u" /* uint_least32_t */ -#define SCNuLEAST64 "llu" /* uint_least64_t */ +#define SCNuLEAST64 "lu" /* uint_least64_t */ #define SCNuFAST8 "u" /* uint_fast8_t */ #define SCNuFAST16 "u" /* uint_fast16_t */ #define SCNuFAST32 "u" /* uint_fast32_t */ -#define SCNuFAST64 "llu" /* uint_fast64_t */ +#define SCNuFAST64 "lu" /* uint_fast64_t */ #define SCNuMAX "ju" /* uintmax_t */ -#define SCNuPTR "u" /* uintptr_t */ +#define SCNuPTR "lu" /* uintptr_t */ #define SCNx8 "hhx" /* uint8_t */ #define SCNx16 "hx" /* uint16_t */ #define SCNx32 "x" /* uint32_t */ -#define SCNx64 "llx" /* uint64_t */ +#define SCNx64 "lx" /* uint64_t */ #define SCNxLEAST8 "hhx" /* uint_least8_t */ #define SCNxLEAST16 "hx" /* uint_least16_t */ #define SCNxLEAST32 "x" /* uint_least32_t */ -#define SCNxLEAST64 "llx" /* uint_least64_t */ +#define SCNxLEAST64 "lx" /* uint_least64_t */ #define SCNxFAST8 "x" /* uint_fast8_t */ #define SCNxFAST16 "x" /* uint_fast16_t */ #define SCNxFAST32 "x" /* uint_fast32_t */ -#define SCNxFAST64 "llx" /* uint_fast64_t */ +#define SCNxFAST64 "lx" /* uint_fast64_t */ #define SCNxMAX "jx" /* uintmax_t */ -#define SCNxPTR "x" /* uintptr_t */ +#define SCNxPTR "lx" /* uintptr_t */ #endif /* !_MACHINE_INTTYPES_H_ */ diff --git a/sys/amd64/include/_types.h b/sys/amd64/include/_types.h index 69b6a0d..f61842c 100644 --- a/sys/amd64/include/_types.h +++ b/sys/amd64/include/_types.h @@ -56,6 +56,8 @@ typedef unsigned long __uint64_t; */ typedef __int32_t __clock_t; /* clock()... */ typedef __int64_t __critical_t; +typedef double __double_t; +typedef double __float_t; typedef __int64_t __intfptr_t; typedef __int64_t __intmax_t; typedef __int64_t __intptr_t; @@ -72,7 +74,7 @@ typedef __int64_t __register_t; typedef __int64_t __segsz_t; /* segment size (in pages) */ typedef __uint64_t __size_t; /* sizeof() */ typedef __int64_t __ssize_t; /* byte count or error */ -typedef __int32_t __time_t; /* time()... */ +typedef __int64_t __time_t; /* time()... */ typedef __uint64_t __uintfptr_t; typedef __uint64_t __uintmax_t; typedef __uint64_t __uintptr_t; diff --git a/sys/amd64/include/asm.h b/sys/amd64/include/asm.h index 30008a2..8ea4122 100644 --- a/sys/amd64/include/asm.h +++ b/sys/amd64/include/asm.h @@ -43,19 +43,9 @@ #include <sys/cdefs.h> #ifdef PIC -#define PIC_PROLOGUE \ - pushl %ebx; \ - call 1f; \ -1: \ - popl %ebx; \ - addl $_GLOBAL_OFFSET_TABLE_+[.-1b],%ebx -#define PIC_EPILOGUE \ - popl %ebx #define PIC_PLT(x) x@PLT -#define PIC_GOT(x) x@GOT(%ebx) +#define PIC_GOT(x) x@GOTPCREL(%rip) #else -#define PIC_PROLOGUE -#define PIC_EPILOGUE #define PIC_PLT(x) x #define PIC_GOT(x) x #endif @@ -78,14 +68,14 @@ #ifdef PROF #define ALTENTRY(x) _ENTRY(x); \ - pushl %ebp; movl %esp,%ebp; \ + pushl %rbp; movl %rsp,%rbp; \ call PIC_PLT(HIDENAME(mcount)); \ - popl %ebp; \ + popl %rbp; \ jmp 9f #define ENTRY(x) _ENTRY(x); \ - pushl %ebp; movl %esp,%ebp; \ + pushl %rbp; movl %rsp,%rbp; \ call PIC_PLT(HIDENAME(mcount)); \ - popl %ebp; \ + popl %rbp; \ 9: #else #define ALTENTRY(x) _ENTRY(x) @@ -117,42 +107,38 @@ #define ARCH_DISPATCH(x) \ _START_ENTRY; \ .globl CNAME(x); .type CNAME(x),@function; CNAME(x): ; \ - PIC_PROLOGUE; \ - movl PIC_GOT(AVECNAME(x)),%eax; \ - PIC_EPILOGUE; \ - jmpl *(%eax) + movq PIC_GOT(AVECNAME(x)),%rax; \ + jmpq *(%rax) #define ARCH_SELECT(x) _START_ENTRY; \ .type ASELNAME(x),@function; \ ASELNAME(x): \ - PIC_PROLOGUE; \ call PIC_PLT(CNAME(__get_hw_float)); \ - testl %eax,%eax; \ - movl PIC_GOT(ANAME(x)),%eax; \ + testq %rax,%rax; \ + movq PIC_GOT(ANAME(x)),%rax; \ jne 8f; \ - movl PIC_GOT(GNAME(x)),%eax; \ + movq PIC_GOT(GNAME(x)),%rax; \ 8: \ - movl PIC_GOT(AVECNAME(x)),%edx; \ - movl %eax,(%edx); \ - PIC_EPILOGUE; \ - jmpl *%eax + movq PIC_GOT(AVECNAME(x)),%rdx; \ + movq %rax,(%rdx); \ + jmpq *%rax #else /* !PIC */ #define ARCH_DISPATCH(x) \ _START_ENTRY; \ .globl CNAME(x); .type CNAME(x),@function; CNAME(x): ; \ - jmpl *AVECNAME(x) + jmpw *AVECNAME(x) #define ARCH_SELECT(x) _START_ENTRY; \ .type ASELNAME(x),@function; \ ASELNAME(x): \ call CNAME(__get_hw_float); \ - testl %eax,%eax; \ - movl $ANAME(x),%eax; \ + testw %rax,%rax; \ + movw $ANAME(x),%rax; \ jne 8f; \ - movl $GNAME(x),%eax; \ + movw $GNAME(x),%rax; \ 8: \ - movl %eax,AVECNAME(x); \ - jmpl *%eax + movw %rax,AVECNAME(x); \ + jmpw *%rax #endif /* PIC */ #define ARCH_VECTOR(x) .data; .p2align 2; \ diff --git a/sys/amd64/include/asmacros.h b/sys/amd64/include/asmacros.h index 07035ca..9893e5f 100644 --- a/sys/amd64/include/asmacros.h +++ b/sys/amd64/include/asmacros.h @@ -50,11 +50,11 @@ #define CNAME(csym) csym #define HIDENAME(asmsym) .asmsym -#define ALIGN_DATA .p2align 2 /* 4 byte alignment, zero filled */ +#define ALIGN_DATA .p2align 3 /* 8 byte alignment, zero filled */ #ifdef GPROF #define ALIGN_TEXT .p2align 4,0x90 /* 16-byte alignment, nop filled */ #else -#define ALIGN_TEXT .p2align 2,0x90 /* 4-byte alignment, nop filled */ +#define ALIGN_TEXT .p2align 4,0x90 /* 16-byte alignment, nop filled */ #endif #define SUPERALIGN_TEXT .p2align 4,0x90 /* 16-byte alignment, nop filled */ @@ -64,9 +64,9 @@ #define NON_GPROF_RET .byte 0xc3 /* opcode for `ret' */ #ifdef LOCORE -#define PCPU(member) %fs:PC_ ## member -#define PCPU_ADDR(member, reg) movl %fs:PC_PRVSPACE,reg; \ - addl $PC_ ## member,reg +#define PCPU(member) %gs:PC_ ## member +#define PCPU_ADDR(member, reg) movq %gs:PC_PRVSPACE,reg; \ + addq $PC_ ## member,reg #endif #ifdef GPROF @@ -115,7 +115,7 @@ #define CROSSJUMPTARGET(label) \ ALIGN_TEXT; __CONCAT(to,label): ; MCOUNT; jmp label #define ENTRY(name) GEN_ENTRY(name) ; 9: ; MCOUNT -#define FAKE_MCOUNT(caller) pushl caller ; call __mcount ; popl %ecx +#define FAKE_MCOUNT(caller) pushq caller ; call __mcount ; popl %ecx #define MCOUNT call __mcount #define MCOUNT_LABEL(name) GEN_ENTRY(name) ; nop ; ALIGN_TEXT #define MEXITCOUNT call HIDENAME(mexitcount) diff --git a/sys/amd64/include/atomic.h b/sys/amd64/include/atomic.h index 919bd5a..0c4d9b0 100644 --- a/sys/amd64/include/atomic.h +++ b/sys/amd64/include/atomic.h @@ -68,6 +68,7 @@ void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v) int atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src); +int atomic_cmpset_long(volatile u_long *dst, u_long exp, u_long src); #define ATOMIC_STORE_LOAD(TYPE, LOP, SOP) \ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p); \ @@ -81,7 +82,7 @@ void atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) * For userland, assume the SMP case and use lock prefixes so that * the binaries will run on both types of systems. */ -#if defined(SMP) || !defined(_KERNEL) +#if !defined(_KERNEL) #define MPLOCKED lock ; #else #define MPLOCKED @@ -117,84 +118,49 @@ extern void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v) #if defined(__GNUC__) -#if defined(I386_CPU) || defined(CPU_DISABLE_CMPXCHG) - static __inline int atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src) { int res = exp; - __asm __volatile( - " pushfl ; " - " cli ; " - " cmpl %0,%2 ; " - " jne 1f ; " - " movl %1,%2 ; " - "1: " - " sete %%al; " + __asm __volatile ( + " " __XSTRING(MPLOCKED) " " + " cmpxchgl %1,%2 ; " + " setz %%al ; " " movzbl %%al,%0 ; " - " popfl ; " + "1: " "# atomic_cmpset_int" : "+a" (res) /* 0 (result) */ : "r" (src), /* 1 */ "m" (*(dst)) /* 2 */ - : "memory"); + : "memory"); return (res); } -#else /* defined(I386_CPU) */ - static __inline int -atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src) +atomic_cmpset_long(volatile u_long *dst, u_long exp, u_long src) { - int res = exp; + long res = exp; __asm __volatile ( " " __XSTRING(MPLOCKED) " " - " cmpxchgl %1,%2 ; " + " cmpxchgq %1,%2 ; " " setz %%al ; " - " movzbl %%al,%0 ; " + " movzbq %%al,%0 ; " "1: " - "# atomic_cmpset_int" - : "+a" (res) /* 0 (result) */ + "# atomic_cmpset_long" + : "+a" (res) /* 0 (result) %rax, XXX check */ : "r" (src), /* 1 */ "m" (*(dst)) /* 2 */ : "memory"); return (res); } - -#endif /* defined(I386_CPU) */ - #endif /* defined(__GNUC__) */ #if defined(__GNUC__) -#if defined(I386_CPU) - -/* - * We assume that a = b will do atomic loads and stores. - * - * XXX: This is _NOT_ safe on a P6 or higher because it does not guarantee - * memory ordering. These should only be used on a 386. - */ -#define ATOMIC_STORE_LOAD(TYPE, LOP, SOP) \ -static __inline u_##TYPE \ -atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ -{ \ - return (*p); \ -} \ - \ -static __inline void \ -atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ -{ \ - *p = v; \ - __asm __volatile("" : : : "memory"); \ -} - -#else /* !defined(I386_CPU) */ - #define ATOMIC_STORE_LOAD(TYPE, LOP, SOP) \ static __inline u_##TYPE \ atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ @@ -221,11 +187,10 @@ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ : : "memory"); \ } -#endif /* defined(I386_CPU) */ - #else /* !defined(__GNUC__) */ extern int atomic_cmpset_int(volatile u_int *, u_int, u_int); +extern int atomic_cmpset_long(volatile u_long *, u_long, u_long); #define ATOMIC_STORE_LOAD(TYPE, LOP, SOP) \ extern u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p); \ @@ -250,15 +215,15 @@ ATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v); ATOMIC_ASM(add, int, "addl %1,%0", "ir", v); ATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v); -ATOMIC_ASM(set, long, "orl %1,%0", "ir", v); -ATOMIC_ASM(clear, long, "andl %1,%0", "ir", ~v); -ATOMIC_ASM(add, long, "addl %1,%0", "ir", v); -ATOMIC_ASM(subtract, long, "subl %1,%0", "ir", v); +ATOMIC_ASM(set, long, "orq %1,%0", "ir", v); +ATOMIC_ASM(clear, long, "andq %1,%0", "ir", ~v); +ATOMIC_ASM(add, long, "addq %1,%0", "ir", v); +ATOMIC_ASM(subtract, long, "subq %1,%0", "ir", v); ATOMIC_STORE_LOAD(char, "cmpxchgb %b0,%1", "xchgb %b1,%0"); ATOMIC_STORE_LOAD(short,"cmpxchgw %w0,%1", "xchgw %w1,%0"); ATOMIC_STORE_LOAD(int, "cmpxchgl %0,%1", "xchgl %1,%0"); -ATOMIC_STORE_LOAD(long, "cmpxchgl %0,%1", "xchgl %1,%0"); +ATOMIC_STORE_LOAD(long, "cmpxchgq %0,%1", "xchgq %1,%0"); #undef ATOMIC_ASM #undef ATOMIC_STORE_LOAD @@ -300,9 +265,6 @@ ATOMIC_STORE_LOAD(long, "cmpxchgl %0,%1", "xchgl %1,%0"); #define atomic_add_rel_long atomic_add_long #define atomic_subtract_acq_long atomic_subtract_long #define atomic_subtract_rel_long atomic_subtract_long -#define atomic_cmpset_long atomic_cmpset_int -#define atomic_cmpset_acq_long atomic_cmpset_acq_int -#define atomic_cmpset_rel_long atomic_cmpset_rel_int #define atomic_cmpset_acq_ptr atomic_cmpset_ptr #define atomic_cmpset_rel_ptr atomic_cmpset_ptr @@ -361,39 +323,39 @@ static __inline int atomic_cmpset_ptr(volatile void *dst, void *exp, void *src) { - return (atomic_cmpset_int((volatile u_int *)dst, (u_int)exp, - (u_int)src)); + return (atomic_cmpset_long((volatile u_long *)dst, + (u_long)exp, (u_long)src)); } static __inline void * atomic_load_acq_ptr(volatile void *p) { - return (void *)atomic_load_acq_int((volatile u_int *)p); + return (void *)atomic_load_acq_long((volatile u_long *)p); } static __inline void atomic_store_rel_ptr(volatile void *p, void *v) { - atomic_store_rel_int((volatile u_int *)p, (u_int)v); + atomic_store_rel_long((volatile u_long *)p, (u_long)v); } #define ATOMIC_PTR(NAME) \ static __inline void \ atomic_##NAME##_ptr(volatile void *p, uintptr_t v) \ { \ - atomic_##NAME##_int((volatile u_int *)p, v); \ + atomic_##NAME##_long((volatile u_long *)p, v); \ } \ \ static __inline void \ atomic_##NAME##_acq_ptr(volatile void *p, uintptr_t v) \ { \ - atomic_##NAME##_acq_int((volatile u_int *)p, v);\ + atomic_##NAME##_acq_long((volatile u_long *)p, v);\ } \ \ static __inline void \ atomic_##NAME##_rel_ptr(volatile void *p, uintptr_t v) \ { \ - atomic_##NAME##_rel_int((volatile u_int *)p, v);\ + atomic_##NAME##_rel_long((volatile u_long *)p, v);\ } ATOMIC_PTR(set) @@ -426,8 +388,8 @@ atomic_readandclear_long(volatile u_long *addr) u_long result; __asm __volatile ( - " xorl %0,%0 ; " - " xchgl %1,%0 ; " + " xorq %0,%0 ; " + " xchgq %1,%0 ; " "# atomic_readandclear_int" : "=&r" (result) /* 0 (result) */ : "m" (*addr)); /* 1 (addr) */ diff --git a/sys/amd64/include/bus.h b/sys/amd64/include/bus.h index a33569f..9aa9336 100644 --- a/sys/amd64/include/bus.h +++ b/sys/amd64/include/bus.h @@ -34,17 +34,11 @@ #ifndef _I386_BUS_H_ #define _I386_BUS_H_ -#ifdef PC98 -/* NEC PC-98 */ -#include <machine/bus_pc98.h> -#else -/* IBM-PC */ #include <machine/bus_at386.h> -#endif #include <machine/bus_dma.h> /* - * Stream accesses are the same as normal accesses on i386/pc98; there are no + * Stream accesses are the same as normal accesses on i386; there are no * supported bus systems with an endianess different from the host one. */ #define bus_space_read_stream_1(t, h, o) bus_space_read_1((t), (h), (o)) diff --git a/sys/amd64/include/bus_amd64.h b/sys/amd64/include/bus_amd64.h index e1f6e8e..d8ae035 100644 --- a/sys/amd64/include/bus_amd64.h +++ b/sys/amd64/include/bus_amd64.h @@ -92,31 +92,23 @@ /* * Bus address and size types */ -#ifdef PAE typedef uint64_t bus_addr_t; -#else -typedef uint32_t bus_addr_t; -#endif -typedef uint32_t bus_size_t; +typedef uint64_t bus_size_t; #define BUS_SPACE_MAXSIZE_24BIT 0xFFFFFF #define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFF #define BUS_SPACE_MAXSIZE 0xFFFFFFFF #define BUS_SPACE_MAXADDR_24BIT 0xFFFFFF #define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFF -#ifdef PAE #define BUS_SPACE_MAXADDR 0xFFFFFFFFFFFFFFFFULL -#else -#define BUS_SPACE_MAXADDR 0xFFFFFFFF -#endif #define BUS_SPACE_UNRESTRICTED (~0) /* * Access methods for bus resources and address space. */ -typedef int bus_space_tag_t; -typedef u_int bus_space_handle_t; +typedef uint64_t bus_space_tag_t; +typedef uint64_t bus_space_handle_t; /* * Map a region of device bus space into CPU virtual address space. @@ -1215,7 +1207,7 @@ bus_space_barrier(bus_space_tag_t tag __unused, bus_space_handle_t bsh __unused, { #ifdef __GNUC__ if (flags & BUS_SPACE_BARRIER_READ) - __asm __volatile("lock; addl $0,0(%%esp)" : : : "memory"); + __asm __volatile("lock; addl $0,0(%%rsp)" : : : "memory"); else __asm __volatile("" : : : "memory"); #endif diff --git a/sys/amd64/include/bus_at386.h b/sys/amd64/include/bus_at386.h index e1f6e8e..d8ae035 100644 --- a/sys/amd64/include/bus_at386.h +++ b/sys/amd64/include/bus_at386.h @@ -92,31 +92,23 @@ /* * Bus address and size types */ -#ifdef PAE typedef uint64_t bus_addr_t; -#else -typedef uint32_t bus_addr_t; -#endif -typedef uint32_t bus_size_t; +typedef uint64_t bus_size_t; #define BUS_SPACE_MAXSIZE_24BIT 0xFFFFFF #define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFF #define BUS_SPACE_MAXSIZE 0xFFFFFFFF #define BUS_SPACE_MAXADDR_24BIT 0xFFFFFF #define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFF -#ifdef PAE #define BUS_SPACE_MAXADDR 0xFFFFFFFFFFFFFFFFULL -#else -#define BUS_SPACE_MAXADDR 0xFFFFFFFF -#endif #define BUS_SPACE_UNRESTRICTED (~0) /* * Access methods for bus resources and address space. */ -typedef int bus_space_tag_t; -typedef u_int bus_space_handle_t; +typedef uint64_t bus_space_tag_t; +typedef uint64_t bus_space_handle_t; /* * Map a region of device bus space into CPU virtual address space. @@ -1215,7 +1207,7 @@ bus_space_barrier(bus_space_tag_t tag __unused, bus_space_handle_t bsh __unused, { #ifdef __GNUC__ if (flags & BUS_SPACE_BARRIER_READ) - __asm __volatile("lock; addl $0,0(%%esp)" : : : "memory"); + __asm __volatile("lock; addl $0,0(%%rsp)" : : : "memory"); else __asm __volatile("" : : : "memory"); #endif diff --git a/sys/amd64/include/clock.h b/sys/amd64/include/clock.h index 5ae4fb2..511780f 100644 --- a/sys/amd64/include/clock.h +++ b/sys/amd64/include/clock.h @@ -24,9 +24,6 @@ extern int timer0_max_count; extern uint64_t tsc_freq; extern int tsc_is_broken; extern int wall_cmos_clock; -#ifdef APIC_IO -extern int apic_8254_intr; -#endif /* * Driver to clock driver interface. @@ -37,14 +34,8 @@ int acquire_timer0(int rate, void (*function)(struct clockframe *frame)); int acquire_timer2(int mode); int release_timer0(void); int release_timer2(void); -#ifndef PC98 int rtcin(int val); -#else -int acquire_timer1(int mode); -int release_timer1(void); -#endif int sysbeep(int pitch, int period); -void timer_restore(void); void init_TSC(void); #endif /* _KERNEL */ diff --git a/sys/amd64/include/cpu.h b/sys/amd64/include/cpu.h index d4e72bb..e9b7fff 100644 --- a/sys/amd64/include/cpu.h +++ b/sys/amd64/include/cpu.h @@ -55,16 +55,16 @@ #define cpu_exec(p) /* nothing */ #define cpu_swapin(p) /* nothing */ -#define cpu_getstack(td) ((td)->td_frame->tf_esp) -#define cpu_setstack(td, ap) ((td)->td_frame->tf_esp = (ap)) +#define cpu_getstack(td) ((td)->td_frame->tf_rsp) +#define cpu_setstack(td, ap) ((td)->td_frame->tf_rsp = (ap)) #define TRAPF_USERMODE(framep) \ - ((ISPL((framep)->tf_cs) == SEL_UPL) || ((framep)->tf_eflags & PSL_VM)) -#define TRAPF_PC(framep) ((framep)->tf_eip) + (ISPL((framep)->tf_cs) == SEL_UPL) +#define TRAPF_PC(framep) ((framep)->tf_rip) #define CLKF_USERMODE(framep) \ - ((ISPL((framep)->cf_cs) == SEL_UPL) || ((framep)->cf_eflags & PSL_VM)) -#define CLKF_PC(framep) ((framep)->cf_eip) + (ISPL((framep)->cf_cs) == SEL_UPL) +#define CLKF_PC(framep) ((framep)->cf_rip) /* * CTL_MACHDEP definitions. @@ -72,19 +72,9 @@ #define CPU_CONSDEV 1 /* dev_t: console terminal device */ #define CPU_ADJKERNTZ 2 /* int: timezone offset (seconds) */ #define CPU_DISRTCSET 3 /* int: disable resettodr() call */ -#define CPU_BOOTINFO 4 /* struct: bootinfo */ #define CPU_WALLCLOCK 5 /* int: indicates wall CMOS clock */ #define CPU_MAXID 6 /* number of valid machdep ids */ -#define CTL_MACHDEP_NAMES { \ - { 0, 0 }, \ - { "console_device", CTLTYPE_STRUCT }, \ - { "adjkerntz", CTLTYPE_INT }, \ - { "disable_rtc_set", CTLTYPE_INT }, \ - { "bootinfo", CTLTYPE_STRUCT }, \ - { "wall_cmos_clock", CTLTYPE_INT }, \ -} - #ifdef _KERNEL extern char btext[]; extern char etext[]; @@ -99,14 +89,7 @@ void fork_trampoline(void); static __inline u_int64_t get_cyclecount(void) { -#if defined(I386_CPU) || defined(I486_CPU) - struct bintime bt; - if (!tsc_present) { - binuptime(&bt); - return (bt.frac ^ bt.sec); - } -#endif return (rdtsc()); } diff --git a/sys/amd64/include/cpufunc.h b/sys/amd64/include/cpufunc.h index c7f9ace..1e1df33 100644 --- a/sys/amd64/include/cpufunc.h +++ b/sys/amd64/include/cpufunc.h @@ -52,10 +52,12 @@ __BEGIN_DECLS #define readb(va) (*(volatile u_int8_t *) (va)) #define readw(va) (*(volatile u_int16_t *) (va)) #define readl(va) (*(volatile u_int32_t *) (va)) +#define readq(va) (*(volatile u_int64_t *) (va)) #define writeb(va, d) (*(volatile u_int8_t *) (va) = (d)) #define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) #define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) +#define writeq(va, d) (*(volatile u_int64_t *) (va) = (d)) #ifdef __GNUC__ @@ -310,40 +312,40 @@ ia32_pause(void) __asm __volatile("pause"); } -static __inline u_int -read_eflags(void) +static __inline u_long +read_rflags(void) { - u_int ef; + u_long rf; - __asm __volatile("pushfl; popl %0" : "=r" (ef)); - return (ef); + __asm __volatile("pushfq; popq %0" : "=r" (rf)); + return (rf); } static __inline u_int64_t rdmsr(u_int msr) { - u_int64_t rv; + u_int32_t low, high; - __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); - return (rv); + __asm __volatile("rdmsr" : "=a" (low), "=d" (high) : "c" (msr)); + return (low | ((u_int64_t)high << 32)); } static __inline u_int64_t rdpmc(u_int pmc) { - u_int64_t rv; + u_int32_t low, high; - __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); - return (rv); + __asm __volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (pmc)); + return (low | ((u_int64_t)high << 32)); } static __inline u_int64_t rdtsc(void) { - u_int64_t rv; + u_int32_t low, high; - __asm __volatile("rdtsc" : "=A" (rv)); - return (rv); + __asm __volatile("rdtsc" : "=a" (low), "=d" (high)); + return (low | ((u_int64_t)high << 32)); } static __inline void @@ -353,70 +355,74 @@ wbinvd(void) } static __inline void -write_eflags(u_int ef) +write_rflags(u_long rf) { - __asm __volatile("pushl %0; popfl" : : "r" (ef)); + __asm __volatile("pushq %0; popfq" : : "r" (rf)); } static __inline void wrmsr(u_int msr, u_int64_t newval) { - __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); + u_int32_t low, high; + + low = newval; + high = newval >> 32; + __asm __volatile("wrmsr" : : "a" (low), "d" (high), "c" (msr)); } static __inline void -load_cr0(u_int data) +load_cr0(u_long data) { - __asm __volatile("movl %0,%%cr0" : : "r" (data)); + __asm __volatile("movq %0,%%cr0" : : "r" (data)); } -static __inline u_int +static __inline u_long rcr0(void) { - u_int data; + u_long data; - __asm __volatile("movl %%cr0,%0" : "=r" (data)); + __asm __volatile("movq %%cr0,%0" : "=r" (data)); return (data); } -static __inline u_int +static __inline u_long rcr2(void) { - u_int data; + u_long data; - __asm __volatile("movl %%cr2,%0" : "=r" (data)); + __asm __volatile("movq %%cr2,%0" : "=r" (data)); return (data); } static __inline void -load_cr3(u_int data) +load_cr3(u_long data) { - __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); + __asm __volatile("movq %0,%%cr3" : : "r" (data) : "memory"); } -static __inline u_int +static __inline u_long rcr3(void) { - u_int data; + u_long data; - __asm __volatile("movl %%cr3,%0" : "=r" (data)); + __asm __volatile("movq %%cr3,%0" : "=r" (data)); return (data); } static __inline void -load_cr4(u_int data) +load_cr4(u_long data) { - __asm __volatile("movl %0,%%cr4" : : "r" (data)); + __asm __volatile("movq %0,%%cr4" : : "r" (data)); } -static __inline u_int +static __inline u_long rcr4(void) { - u_int data; + u_long data; - __asm __volatile("movl %%cr4,%0" : "=r" (data)); + __asm __volatile("movq %%cr4,%0" : "=r" (data)); return (data); } @@ -435,12 +441,13 @@ invltlb(void) * Only works on 486+ CPUs (i386 does not have PG_G). */ static __inline void -invlpg(u_int addr) +invlpg(u_long addr) { __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); } +/* XXX these are replaced with rdmsr/wrmsr */ static __inline u_int rfs(void) { @@ -490,132 +497,20 @@ ltr(u_short sel) __asm __volatile("ltr %0" : : "r" (sel)); } -static __inline u_int -rdr0(void) -{ - u_int data; - __asm __volatile("movl %%dr0,%0" : "=r" (data)); - return (data); -} - -static __inline void -load_dr0(u_int dr0) -{ - __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); -} - -static __inline u_int -rdr1(void) -{ - u_int data; - __asm __volatile("movl %%dr1,%0" : "=r" (data)); - return (data); -} - -static __inline void -load_dr1(u_int dr1) -{ - __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); -} - -static __inline u_int -rdr2(void) -{ - u_int data; - __asm __volatile("movl %%dr2,%0" : "=r" (data)); - return (data); -} - -static __inline void -load_dr2(u_int dr2) -{ - __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); -} - -static __inline u_int -rdr3(void) -{ - u_int data; - __asm __volatile("movl %%dr3,%0" : "=r" (data)); - return (data); -} - -static __inline void -load_dr3(u_int dr3) -{ - __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); -} - -static __inline u_int -rdr4(void) -{ - u_int data; - __asm __volatile("movl %%dr4,%0" : "=r" (data)); - return (data); -} - -static __inline void -load_dr4(u_int dr4) -{ - __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); -} - -static __inline u_int -rdr5(void) -{ - u_int data; - __asm __volatile("movl %%dr5,%0" : "=r" (data)); - return (data); -} - -static __inline void -load_dr5(u_int dr5) -{ - __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); -} - -static __inline u_int -rdr6(void) -{ - u_int data; - __asm __volatile("movl %%dr6,%0" : "=r" (data)); - return (data); -} - -static __inline void -load_dr6(u_int dr6) -{ - __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); -} - -static __inline u_int -rdr7(void) -{ - u_int data; - __asm __volatile("movl %%dr7,%0" : "=r" (data)); - return (data); -} - -static __inline void -load_dr7(u_int dr7) -{ - __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); -} - static __inline register_t intr_disable(void) { - register_t eflags; + register_t rflags; - eflags = read_eflags(); + rflags = read_rflags(); disable_intr(); - return (eflags); + return (rflags); } static __inline void -intr_restore(register_t eflags) +intr_restore(register_t rflags) { - write_eflags(eflags); + write_rflags(rflags); } #else /* !__GNUC__ */ @@ -623,8 +518,8 @@ intr_restore(register_t eflags) int breakpoint(void); u_int bsfl(u_int mask); u_int bsrl(u_int mask); -void cpu_invlpg(u_int addr); -void cpu_invlpg_range(u_int start, u_int end); +void cpu_invlpg(u_long addr); +void cpu_invlpg_range(u_long start, u_long end); void disable_intr(void); void do_cpuid(u_int ax, u_int *p); void enable_intr(void); @@ -664,28 +559,13 @@ u_int rgs(void); u_int64_t rdmsr(u_int msr); u_int64_t rdpmc(u_int pmc); u_int64_t rdtsc(void); -u_int read_eflags(void); +u_int read_rflags(void); void wbinvd(void); -void write_eflags(u_int ef); +void write_rflags(u_int rf); void wrmsr(u_int msr, u_int64_t newval); -u_int rdr0(void); -void load_dr0(u_int dr0); -u_int rdr1(void); -void load_dr1(u_int dr1); -u_int rdr2(void); -void load_dr2(u_int dr2); -u_int rdr3(void); -void load_dr3(u_int dr3); -u_int rdr4(void); -void load_dr4(u_int dr4); -u_int rdr5(void); -void load_dr5(u_int dr5); -u_int rdr6(void); -void load_dr6(u_int dr6); -u_int rdr7(void); void load_dr7(u_int dr7); register_t intr_disable(void); -void intr_restore(register_t ef); +void intr_restore(register_t rf); #endif /* __GNUC__ */ diff --git a/sys/amd64/include/cputypes.h b/sys/amd64/include/cputypes.h index 585df67..2048458 100644 --- a/sys/amd64/include/cputypes.h +++ b/sys/amd64/include/cputypes.h @@ -33,32 +33,15 @@ /* * Classes of processor. */ -#define CPUCLASS_286 0 -#define CPUCLASS_386 1 -#define CPUCLASS_486 2 -#define CPUCLASS_586 3 -#define CPUCLASS_686 4 +#define CPUCLASS_X86 0 /* X86 */ +#define CPUCLASS_K8 1 /* K8 AMD64 class */ /* * Kinds of processor. */ -#define CPU_286 0 /* Intel 80286 */ -#define CPU_386SX 1 /* Intel 80386SX */ -#define CPU_386 2 /* Intel 80386DX */ -#define CPU_486SX 3 /* Intel 80486SX */ -#define CPU_486 4 /* Intel 80486DX */ -#define CPU_586 5 /* Intel P.....m (I hate lawyers; it's TM) */ -#define CPU_486DLC 6 /* Cyrix 486DLC */ -#define CPU_686 7 /* Pentium Pro */ -#define CPU_M1SC 8 /* Cyrix M1sc (aka 5x86) */ -#define CPU_M1 9 /* Cyrix M1 (aka 6x86) */ -#define CPU_BLUE 10 /* IBM BlueLighting CPU */ -#define CPU_M2 11 /* Cyrix M2 (aka enhanced 6x86 with MMX */ -#define CPU_NX586 12 /* NexGen (now AMD) 586 */ -#define CPU_CY486DX 13 /* Cyrix 486S/DX/DX2/DX4 */ -#define CPU_PII 14 /* Intel Pentium II */ -#define CPU_PIII 15 /* Intel Pentium III */ -#define CPU_P4 16 /* Intel Pentium 4 */ +#define CPU_X86 0 /* Intel */ +#define CPU_CLAWHAMMER 1 /* AMD Clawhammer */ +#define CPU_SLEDGEHAMMER 2 /* AMD Sledgehammer */ #ifndef LOCORE struct cpu_nameclass { diff --git a/sys/amd64/include/critical.h b/sys/amd64/include/critical.h index 7cc7ff2..dc5119c 100644 --- a/sys/amd64/include/critical.h +++ b/sys/amd64/include/critical.h @@ -23,7 +23,6 @@ __BEGIN_DECLS /* * Prototypes - see <arch>/<arch>/critical.c */ -void cpu_unpend(void); void cpu_critical_fork_exit(void); void cpu_thread_link(struct thread *td); @@ -34,12 +33,15 @@ void cpu_thread_link(struct thread *td); * * This routine is called from critical_enter() on the 0->1 transition * of td_critnest, prior to it being incremented to 1. - * - * If new-style critical section handling we do not have to do anything. - * However, as a side effect any interrupts occuring while td_critnest - * is non-zero will be deferred. */ -#define cpu_critical_enter() +static __inline void +cpu_critical_enter(void) +{ + struct thread *td; + + td = curthread; + td->td_md.md_savecrit = intr_disable(); +} /* * cpu_critical_exit: @@ -47,27 +49,14 @@ void cpu_thread_link(struct thread *td); * This routine is called from critical_exit() on a 1->0 transition * of td_critnest, after it has been decremented to 0. We are * exiting the last critical section. - * - * Note that the td->critnest (1->0) transition interrupt race against - * our int_pending/unpend() check below is handled by the interrupt - * code for us, so we do not have to do anything fancy. */ static __inline void cpu_critical_exit(void) { - /* - * We may have to schedule pending interrupts. Create - * conditions similar to an interrupt context and call - * unpend(). - * - * note: we do this even if we are in an interrupt - * nesting level. Deep nesting is protected by - * critical_*() and if we conditionalized it then we - * would have to check int_pending again whenever - * we decrement td_intr_nesting_level to 0. - */ - if (PCPU_GET(int_pending)) - cpu_unpend(); + struct thread *td; + + td = curthread; + intr_restore(td->td_md.md_savecrit); } #else /* !__GNUC__ */ diff --git a/sys/amd64/include/db_machdep.h b/sys/amd64/include/db_machdep.h index ba84fc6..5a2e65d 100644 --- a/sys/amd64/include/db_machdep.h +++ b/sys/amd64/include/db_machdep.h @@ -36,13 +36,13 @@ #define i386_saved_state trapframe typedef vm_offset_t db_addr_t; /* address - unsigned */ -typedef int db_expr_t; /* expression - signed */ +typedef long db_expr_t; /* expression - signed */ typedef struct i386_saved_state db_regs_t; extern db_regs_t ddb_regs; /* register state */ #define DDB_REGS (&ddb_regs) -#define PC_REGS(regs) ((db_addr_t)(regs)->tf_eip) +#define PC_REGS(regs) ((db_addr_t)(regs)->tf_rip) #define BKPT_INST 0xcc /* breakpoint instruction */ #define BKPT_SIZE (1) /* size of breakpoint inst */ diff --git a/sys/amd64/include/elf.h b/sys/amd64/include/elf.h index 46d512c..3883d49 100644 --- a/sys/amd64/include/elf.h +++ b/sys/amd64/include/elf.h @@ -30,17 +30,17 @@ #define _MACHINE_ELF_H_ 1 /* - * ELF definitions for the i386 architecture. + * ELF definitions for the AMD64 architecture. */ -#include <sys/elf32.h> /* Definitions common to all 32 bit architectures. */ +#include <sys/elf64.h> /* Definitions common to all 64 bit architectures. */ -#define __ELF_WORD_SIZE 32 /* Used by <sys/elf_generic.h> */ +#define __ELF_WORD_SIZE 64 /* Used by <sys/elf_generic.h> */ #include <sys/elf_generic.h> -#define ELF_ARCH EM_386 +#define ELF_ARCH EM_X86_64 -#define ELF_MACHINE_OK(x) ((x) == EM_386 || (x) == EM_486) +#define ELF_MACHINE_OK(x) ((x) == EM_X86_64) /* * Auxiliary vector entries for passing information to the interpreter. @@ -50,13 +50,13 @@ */ typedef struct { /* Auxiliary vector entry on initial stack */ - int a_type; /* Entry type. */ + long a_type; /* Entry type. */ union { long a_val; /* Integer value. */ void *a_ptr; /* Address. */ void (*a_fcn)(void); /* Function pointer (not used). */ } a_un; -} Elf32_Auxinfo; +} Elf64_Auxinfo; __ElfType(Auxinfo); @@ -98,24 +98,29 @@ __ElfType(Auxinfo); * Relocation types. */ -#define R_386_NONE 0 /* No relocation. */ -#define R_386_32 1 /* Add symbol value. */ -#define R_386_PC32 2 /* Add PC-relative symbol value. */ -#define R_386_GOT32 3 /* Add PC-relative GOT offset. */ -#define R_386_PLT32 4 /* Add PC-relative PLT offset. */ -#define R_386_COPY 5 /* Copy data from shared object. */ -#define R_386_GLOB_DAT 6 /* Set GOT entry to data address. */ -#define R_386_JMP_SLOT 7 /* Set GOT entry to code address. */ -#define R_386_RELATIVE 8 /* Add load address of shared object. */ -#define R_386_GOTOFF 9 /* Add GOT-relative symbol address. */ -#define R_386_GOTPC 10 /* Add PC-relative GOT table address. */ - -#define R_386_COUNT 11 /* Count of defined relocation types. */ +#define R_X86_64_NONE 0 /* No relocation. */ +#define R_X86_64_64 1 /* Add 64 bit symbol value. */ +#define R_X86_64_PC32 2 /* PC-relative 32 bit signed sym value. */ +#define R_X86_64_GOT32 3 /* PC-relative 32 bit GOT offset. */ +#define R_X86_64_PLT32 4 /* PC-relative 32 bit PLT offset. */ +#define R_X86_64_COPY 5 /* Copy data from shared object. */ +#define R_X86_64_GLOB_DAT 6 /* Set GOT entry to data address. */ +#define R_X86_64_JMP_SLOT 7 /* Set GOT entry to code address. */ +#define R_X86_64_RELATIVE 8 /* Add load address of shared object. */ +#define R_X86_64_GOTPCREL 9 /* Add 32 bit signed pcrel offset to GOT. */ +#define R_X86_64_32 10 /* Add 32 bit zero extended symbol value */ +#define R_X86_64_32S 11 /* Add 32 bit sign extended symbol value */ +#define R_X86_64_16 12 /* Add 16 bit zero extended symbol value */ +#define R_X86_64_PC16 13 /* Add 16 bit signed extended pc relative symbol value */ +#define R_X86_64_8 14 /* Add 8 bit zero extended symbol value */ +#define R_X86_64_PC8 15 /* Add 8 bit signed extended pc relative symbol value */ + +#define R_X86_64_COUNT 16 /* Count of defined relocation types. */ /* Define "machine" characteristics */ -#define ELF_TARG_CLASS ELFCLASS32 +#define ELF_TARG_CLASS ELFCLASS64 #define ELF_TARG_DATA ELFDATA2LSB -#define ELF_TARG_MACH EM_386 +#define ELF_TARG_MACH EM_X86_64 #define ELF_TARG_VER 1 #ifdef _KERNEL diff --git a/sys/amd64/include/endian.h b/sys/amd64/include/endian.h index 723a5f3..1766866 100644 --- a/sys/amd64/include/endian.h +++ b/sys/amd64/include/endian.h @@ -74,25 +74,26 @@ __extension__ ({ register __uint32_t __X = (x); \ __asm ("rorl $16, %0" : "+r" (__X)); \ __X; }) -#if defined(_KERNEL) #define __byte_swap_int(x) \ __extension__ ({ register __uint32_t __X = (x); \ __asm ("bswap %0" : "+r" (__X)); \ __X; }) -#endif + +#define __byte_swap_long(x) \ +__extension__ ({ register __uint64_t __X = (x); \ + __asm ("bswap %0" : "+r" (__X)); \ + __X; }) #define __byte_swap_word(x) \ __extension__ ({ register __uint16_t __X = (x); \ - __asm ("xchgb %h0, %b0" : "+q" (__X)); \ + __asm ("xchgb %h0, %b0" : "+Q" (__X)); \ __X; }) static __inline __uint64_t __bswap64(__uint64_t _x) { - return ((_x >> 56) | ((_x >> 40) & 0xff00) | ((_x >> 24) & 0xff0000) | - ((_x >> 8) & 0xff000000) | ((_x << 8) & ((__uint64_t)0xff << 32)) | - ((_x << 24) & ((__uint64_t)0xff << 40)) | - ((_x << 40) & ((__uint64_t)0xff << 48)) | ((_x << 56))); + + return (__byte_swap_long(_x)); } static __inline __uint32_t diff --git a/sys/amd64/include/fpu.h b/sys/amd64/include/fpu.h index 8e0f9bc..db1fe40 100644 --- a/sys/amd64/include/fpu.h +++ b/sys/amd64/include/fpu.h @@ -45,81 +45,38 @@ #ifndef _MACHINE_NPX_H_ #define _MACHINE_NPX_H_ -/* Environment information of floating point unit */ -struct env87 { - long en_cw; /* control word (16bits) */ - long en_sw; /* status word (16bits) */ - long en_tw; /* tag word (16bits) */ - long en_fip; /* floating point instruction pointer */ - u_short en_fcs; /* floating code segment selector */ - u_short en_opcode; /* opcode last executed (11 bits ) */ - long en_foo; /* floating operand offset */ - long en_fos; /* floating operand segment selector */ -}; - -/* Contents of each floating point accumulator */ +/* Contents of each x87 floating point accumulator */ struct fpacc87 { -#ifdef dontdef /* too unportable */ - u_long fp_mantlo; /* mantissa low (31:0) */ - u_long fp_manthi; /* mantissa high (63:32) */ - int fp_exp:15; /* exponent */ - int fp_sgn:1; /* mantissa sign */ -#else u_char fp_bytes[10]; -#endif }; -/* Floating point context */ -struct save87 { - struct env87 sv_env; /* floating point control/status */ - struct fpacc87 sv_ac[8]; /* accumulator contents, 0-7 */ - u_char sv_pad0[4]; /* padding for (now unused) saved status word */ - /* - * Bogus padding for emulators. Emulators should use their own - * struct and arrange to store into this struct (ending here) - * before it is inspected for ptracing or for core dumps. Some - * emulators overwrite the whole struct. We have no good way of - * knowing how much padding to leave. Leave just enough for the - * GPL emulator's i387_union (176 bytes total). - */ - u_char sv_pad[64]; /* padding; used by emulators */ +/* Contents of each SSE extended accumulator */ +struct xmmacc { + u_char xmm_bytes[16]; }; struct envxmm { u_int16_t en_cw; /* control word (16bits) */ u_int16_t en_sw; /* status word (16bits) */ - u_int16_t en_tw; /* tag word (16bits) */ + u_int8_t en_tw; /* tag word (8bits) */ + u_int8_t en_zero; u_int16_t en_opcode; /* opcode last executed (11 bits ) */ - u_int32_t en_fip; /* floating point instruction pointer */ - u_int16_t en_fcs; /* floating code segment selector */ - u_int16_t en_pad0; /* padding */ - u_int32_t en_foo; /* floating operand offset */ - u_int16_t en_fos; /* floating operand segment selector */ - u_int16_t en_pad1; /* padding */ + u_int64_t en_rip; /* floating point instruction pointer */ + u_int64_t en_rdp; /* floating operand pointer */ u_int32_t en_mxcsr; /* SSE sontorol/status register */ - u_int32_t en_pad2; /* padding */ + u_int32_t en_mxcsr_mask; /* valid bits in mxcsr */ }; -/* Contents of each SSE extended accumulator */ -struct xmmacc { - u_char xmm_bytes[16]; -}; - -struct savexmm { +struct savefpu { struct envxmm sv_env; struct { struct fpacc87 fp_acc; u_char fp_pad[6]; /* padding */ } sv_fp[8]; - struct xmmacc sv_xmm[8]; - u_char sv_pad[224]; + struct xmmacc sv_xmm[16]; + u_char sv_pad[96]; } __aligned(16); -union savefpu { - struct save87 sv_87; - struct savexmm sv_xmm; -}; - /* * The hardware default control word for i387's and later coprocessors is * 0x37F, giving: @@ -144,10 +101,10 @@ int npxdna(void); void npxdrop(void); void npxexit(struct thread *td); int npxformat(void); -int npxgetregs(struct thread *td, union savefpu *addr); +int npxgetregs(struct thread *td, struct savefpu *addr); void npxinit(u_short control); -void npxsave(union savefpu *addr); -void npxsetregs(struct thread *td, union savefpu *addr); +void npxsave(struct savefpu *addr); +void npxsetregs(struct thread *td, struct savefpu *addr); int npxtrap(void); #endif diff --git a/sys/amd64/include/frame.h b/sys/amd64/include/frame.h index c32afc4..8f96528 100644 --- a/sys/amd64/include/frame.h +++ b/sys/amd64/include/frame.h @@ -46,116 +46,93 @@ /* * Exception/Trap Stack Frame + * + * The ordering of this is specifically so that we can take first 6 + * the syscall arguments directly from the beginning of the frame. */ struct trapframe { - int tf_fs; - int tf_es; - int tf_ds; - int tf_edi; - int tf_esi; - int tf_ebp; - int tf_isp; - int tf_ebx; - int tf_edx; - int tf_ecx; - int tf_eax; - int tf_trapno; - /* below portion defined in 386 hardware */ - int tf_err; - int tf_eip; - int tf_cs; - int tf_eflags; - /* below only when crossing rings (e.g. user to kernel) */ - int tf_esp; - int tf_ss; -}; - -/* Superset of trap frame, for traps from virtual-8086 mode */ - -struct trapframe_vm86 { - int tf_fs; - int tf_es; - int tf_ds; - int tf_edi; - int tf_esi; - int tf_ebp; - int tf_isp; - int tf_ebx; - int tf_edx; - int tf_ecx; - int tf_eax; - int tf_trapno; - /* below portion defined in 386 hardware */ - int tf_err; - int tf_eip; - int tf_cs; - int tf_eflags; - /* below only when crossing rings (e.g. user to kernel) */ - int tf_esp; - int tf_ss; - /* below only when switching out of VM86 mode */ - int tf_vm86_es; - int tf_vm86_ds; - int tf_vm86_fs; - int tf_vm86_gs; + register_t tf_rdi; + register_t tf_rsi; + register_t tf_rdx; + register_t tf_rcx; + register_t tf_r8; + register_t tf_r9; + register_t tf_rax; + register_t tf_rbx; + register_t tf_rbp; + register_t tf_r10; + register_t tf_r11; + register_t tf_r12; + register_t tf_r13; + register_t tf_r14; + register_t tf_r15; + register_t tf_trapno; + /* below portion defined in hardware */ + register_t tf_err; + register_t tf_rip; + register_t tf_cs; + register_t tf_rflags; + register_t tf_rsp; + register_t tf_ss; }; /* Interrupt stack frame */ struct intrframe { - int if_vec; - int if_fs; - int if_es; - int if_ds; - int if_edi; - int if_esi; - int if_ebp; - int :32; - int if_ebx; - int if_edx; - int if_ecx; - int if_eax; - int :32; /* for compat with trap frame - trapno */ - int :32; /* for compat with trap frame - err */ - /* below portion defined in 386 hardware */ - int if_eip; - int if_cs; - int if_eflags; - /* below only when crossing rings (e.g. user to kernel) */ - int if_esp; - int if_ss; + register_t if_rdi; + register_t if_rsi; + register_t if_rdx; + register_t if_rcx; + register_t if_r8; + register_t if_r9; + register_t if_rax; + register_t if_rbx; + register_t if_rbp; + register_t if_r10; + register_t if_r11; + register_t if_r12; + register_t if_r13; + register_t if_r14; + register_t if_r15; + register_t :64; /* compat with trap frame - trapno */ + register_t :64; /* compat with trap frame - err */ + /* below portion defined in hardware */ + register_t if_rip; + register_t if_cs; + register_t if_rflags; + register_t if_rsp; + register_t if_ss; }; /* frame of clock (same as interrupt frame) */ struct clockframe { - int cf_vec; - int cf_fs; - int cf_es; - int cf_ds; - int cf_edi; - int cf_esi; - int cf_ebp; - int :32; - int cf_ebx; - int cf_edx; - int cf_ecx; - int cf_eax; - int :32; /* for compat with trap frame - trapno */ - int :32; /* for compat with trap frame - err */ - /* below portion defined in 386 hardware */ - int cf_eip; - int cf_cs; - int cf_eflags; - /* below only when crossing rings (e.g. user to kernel) */ - int cf_esp; - int cf_ss; + register_t cf_rdi; + register_t cf_rsi; + register_t cf_rdx; + register_t cf_rcx; + register_t cf_r8; + register_t cf_r9; + register_t cf_rax; + register_t cf_rbx; + register_t cf_rbp; + register_t cf_r10; + register_t cf_r11; + register_t cf_r12; + register_t cf_r13; + register_t cf_r14; + register_t cf_r15; + register_t :64; /* compat with trap frame - trapno */ + register_t :64; /* compat with trap frame - err */ + /* below portion defined in hardware */ + register_t cf_rip; + register_t cf_cs; + register_t cf_rflags; + register_t cf_rsp; + register_t cf_ss; }; int kdb_trap(int, int, struct trapframe *); -extern int (*pmath_emulate)(struct trapframe *); - -#define INTR_TO_TRAPFRAME(frame) ((struct trapframe *)&(frame)->if_fs) #endif /* _MACHINE_FRAME_H_ */ diff --git a/sys/amd64/include/kse.h b/sys/amd64/include/kse.h index feaf535..c96e4f5 100644 --- a/sys/amd64/include/kse.h +++ b/sys/amd64/include/kse.h @@ -36,14 +36,7 @@ #include <machine/ucontext.h> union kse_td_ctx { -#ifdef _KERNEL - struct { - int tf_onstack; - int tf_gs; - struct trapframe tf_tf; - } tfrm; -#endif - mcontext_t mcontext; + int teh_stuff; }; #endif /* MACHINE_KSE_H */ diff --git a/sys/amd64/include/md_var.h b/sys/amd64/include/md_var.h index bf3eca4..c5f0e458 100644 --- a/sys/amd64/include/md_var.h +++ b/sys/amd64/include/md_var.h @@ -36,11 +36,6 @@ * Miscellaneous machine-dependent declarations. */ -extern void (*bcopy_vector)(const void *from, void *to, size_t len); -extern void (*bzero_vector)(void *buf, size_t len); -extern int (*copyin_vector)(const void *udaddr, void *kaddr, size_t len); -extern int (*copyout_vector)(const void *kaddr, void *udaddr, size_t len); - extern long Maxmem; extern u_int atdevbase; /* offset in virtual memory of ISA io mem */ extern int busdma_swi_pending; @@ -51,37 +46,10 @@ extern u_int cpu_high; extern u_int cpu_id; extern u_int cpu_procinfo; extern char cpu_vendor[]; -extern u_int cyrix_did; extern uint16_t *elan_mmcr; extern char kstack[]; -#ifdef PC98 -extern int need_pre_dma_flush; -extern int need_post_dma_flush; -#endif extern char sigcode[]; extern int szsigcode; -#ifdef COMPAT_FREEBSD4 -extern int szfreebsd4_sigcode; -#endif -#ifdef COMPAT_43 -extern int szosigcode; -#endif -#ifdef SWTCH_OPTIM_STATS -extern int stupid_switch; -extern int swtch_optim_stats; -extern int tlb_flush_count; -extern int lazy_flush_count; -extern int lazy_flush_fixup; -#ifdef SMP -extern int lazy_flush_smpfixup; -extern int lazy_flush_smpipi; -extern int lazy_flush_smpbadcr3; -extern int lazy_flush_smpmiss; -#endif -#endif -#ifdef LAZY_SWITCH -extern int lazy_flush_enable; -#endif typedef void alias_for_inthand_t(u_int cs, u_int ef, u_int esp, u_int ss); struct thread; @@ -94,28 +62,13 @@ void busdma_swi(void); void cpu_halt(void); void cpu_reset(void); void cpu_setregs(void); -void cpu_switch_load_gs(void) __asm(__STRING(cpu_switch_load_gs)); void doreti_iret(void) __asm(__STRING(doreti_iret)); void doreti_iret_fault(void) __asm(__STRING(doreti_iret_fault)); -void doreti_popl_ds(void) __asm(__STRING(doreti_popl_ds)); -void doreti_popl_ds_fault(void) __asm(__STRING(doreti_popl_ds_fault)); -void doreti_popl_es(void) __asm(__STRING(doreti_popl_es)); -void doreti_popl_es_fault(void) __asm(__STRING(doreti_popl_es_fault)); -void doreti_popl_fs(void) __asm(__STRING(doreti_popl_fs)); -void doreti_popl_fs_fault(void) __asm(__STRING(doreti_popl_fs_fault)); void enable_sse(void); void fillw(int /*u_short*/ pat, void *base, size_t cnt); -void i486_bzero(void *buf, size_t len); -void i586_bcopy(const void *from, void *to, size_t len); -void i586_bzero(void *buf, size_t len); -int i586_copyin(const void *udaddr, void *kaddr, size_t len); -int i586_copyout(const void *kaddr, void *udaddr, size_t len); -void i686_pagezero(void *addr); -void init_AMD_Elan_sc520(void); +void pagezero(void *addr); int is_physical_memory(vm_offset_t addr); -vm_paddr_t kvtop(void *addr); -void setidt(int idx, alias_for_inthand_t *func, int typ, int dpl, int selec); +void setidt(int idx, alias_for_inthand_t *func, int typ, int dpl, int ist); void swi_vm(void *); -int user_dbreg_trap(void); #endif /* !_MACHINE_MD_VAR_H_ */ diff --git a/sys/amd64/include/mutex.h b/sys/amd64/include/mutex.h index 87d47e5..eb13945 100644 --- a/sys/amd64/include/mutex.h +++ b/sys/amd64/include/mutex.h @@ -56,20 +56,20 @@ extern struct mtx clock_lock; * locks) in the near future, however. */ #define MTX_LOCK_SPIN(lck, flags) \ - pushl $0 ; \ - pushl $0 ; \ - pushl $flags ; \ - pushl $lck ; \ + pushq $0 ; \ + pushq $0 ; \ + pushq $flags ; \ + pushq $lck ; \ call _mtx_lock_spin_flags ; \ - addl $0x10, %esp ; \ + addq $0x20, %rsp ; \ #define MTX_UNLOCK_SPIN(lck) \ - pushl $0 ; \ - pushl $0 ; \ - pushl $0 ; \ - pushl $lck ; \ + pushq $0 ; \ + pushq $0 ; \ + pushq $0 ; \ + pushq $lck ; \ call _mtx_unlock_spin_flags ; \ - addl $0x10, %esp ; \ + addq $0x20, %rsp ; \ #endif /* !LOCORE */ #endif /* __MACHINE_MUTEX_H */ diff --git a/sys/amd64/include/npx.h b/sys/amd64/include/npx.h index 8e0f9bc..db1fe40 100644 --- a/sys/amd64/include/npx.h +++ b/sys/amd64/include/npx.h @@ -45,81 +45,38 @@ #ifndef _MACHINE_NPX_H_ #define _MACHINE_NPX_H_ -/* Environment information of floating point unit */ -struct env87 { - long en_cw; /* control word (16bits) */ - long en_sw; /* status word (16bits) */ - long en_tw; /* tag word (16bits) */ - long en_fip; /* floating point instruction pointer */ - u_short en_fcs; /* floating code segment selector */ - u_short en_opcode; /* opcode last executed (11 bits ) */ - long en_foo; /* floating operand offset */ - long en_fos; /* floating operand segment selector */ -}; - -/* Contents of each floating point accumulator */ +/* Contents of each x87 floating point accumulator */ struct fpacc87 { -#ifdef dontdef /* too unportable */ - u_long fp_mantlo; /* mantissa low (31:0) */ - u_long fp_manthi; /* mantissa high (63:32) */ - int fp_exp:15; /* exponent */ - int fp_sgn:1; /* mantissa sign */ -#else u_char fp_bytes[10]; -#endif }; -/* Floating point context */ -struct save87 { - struct env87 sv_env; /* floating point control/status */ - struct fpacc87 sv_ac[8]; /* accumulator contents, 0-7 */ - u_char sv_pad0[4]; /* padding for (now unused) saved status word */ - /* - * Bogus padding for emulators. Emulators should use their own - * struct and arrange to store into this struct (ending here) - * before it is inspected for ptracing or for core dumps. Some - * emulators overwrite the whole struct. We have no good way of - * knowing how much padding to leave. Leave just enough for the - * GPL emulator's i387_union (176 bytes total). - */ - u_char sv_pad[64]; /* padding; used by emulators */ +/* Contents of each SSE extended accumulator */ +struct xmmacc { + u_char xmm_bytes[16]; }; struct envxmm { u_int16_t en_cw; /* control word (16bits) */ u_int16_t en_sw; /* status word (16bits) */ - u_int16_t en_tw; /* tag word (16bits) */ + u_int8_t en_tw; /* tag word (8bits) */ + u_int8_t en_zero; u_int16_t en_opcode; /* opcode last executed (11 bits ) */ - u_int32_t en_fip; /* floating point instruction pointer */ - u_int16_t en_fcs; /* floating code segment selector */ - u_int16_t en_pad0; /* padding */ - u_int32_t en_foo; /* floating operand offset */ - u_int16_t en_fos; /* floating operand segment selector */ - u_int16_t en_pad1; /* padding */ + u_int64_t en_rip; /* floating point instruction pointer */ + u_int64_t en_rdp; /* floating operand pointer */ u_int32_t en_mxcsr; /* SSE sontorol/status register */ - u_int32_t en_pad2; /* padding */ + u_int32_t en_mxcsr_mask; /* valid bits in mxcsr */ }; -/* Contents of each SSE extended accumulator */ -struct xmmacc { - u_char xmm_bytes[16]; -}; - -struct savexmm { +struct savefpu { struct envxmm sv_env; struct { struct fpacc87 fp_acc; u_char fp_pad[6]; /* padding */ } sv_fp[8]; - struct xmmacc sv_xmm[8]; - u_char sv_pad[224]; + struct xmmacc sv_xmm[16]; + u_char sv_pad[96]; } __aligned(16); -union savefpu { - struct save87 sv_87; - struct savexmm sv_xmm; -}; - /* * The hardware default control word for i387's and later coprocessors is * 0x37F, giving: @@ -144,10 +101,10 @@ int npxdna(void); void npxdrop(void); void npxexit(struct thread *td); int npxformat(void); -int npxgetregs(struct thread *td, union savefpu *addr); +int npxgetregs(struct thread *td, struct savefpu *addr); void npxinit(u_short control); -void npxsave(union savefpu *addr); -void npxsetregs(struct thread *td, union savefpu *addr); +void npxsave(struct savefpu *addr); +void npxsetregs(struct thread *td, struct savefpu *addr); int npxtrap(void); #endif diff --git a/sys/amd64/include/param.h b/sys/amd64/include/param.h index 5955022..355e95d 100644 --- a/sys/amd64/include/param.h +++ b/sys/amd64/include/param.h @@ -40,7 +40,7 @@ */ /* - * Machine dependent constants for the AMD64. + * Machine dependent constants for AMD64. */ /* @@ -55,7 +55,7 @@ * */ #ifndef _ALIGNBYTES -#define _ALIGNBYTES (sizeof(int) - 1) +#define _ALIGNBYTES (sizeof(long) - 1) #endif #ifndef _ALIGN #define _ALIGN(p) (((u_long)(p) + _ALIGNBYTES) &~ _ALIGNBYTES) @@ -83,40 +83,71 @@ #define MACHINE_ARCH "amd64" #endif -#ifdef SMP -#define MAXCPU 16 -#else #define MAXCPU 1 -#endif /* SMP */ #define ALIGNBYTES _ALIGNBYTES #define ALIGN(p) _ALIGN(p) #define ALIGNED_POINTER(p,t) _ALIGNED_POINTER((p),(t)) +/* Size of the level 1 page table units */ +#define NPTEPG (PAGE_SIZE/(sizeof (pt_entry_t))) #define PAGE_SHIFT 12 /* LOG2(PAGE_SIZE) */ #define PAGE_SIZE (1<<PAGE_SHIFT) /* bytes/page */ #define PAGE_MASK (PAGE_SIZE-1) -#define NPTEPG (PAGE_SIZE/(sizeof (pt_entry_t))) - -#define KERNBASE 0x0000000000000000LL /* start of kernel virtual */ -#define BTOPKERNBASE ((u_long)KERNBASE >> PGSHIFT) +/* Size of the level 2 page directory units */ +#define NPDEPG (PAGE_SIZE/(sizeof (pd_entry_t))) +#define PDRSHIFT 21 /* LOG2(NBPDR) */ +#define NBPDR (1<<PDRSHIFT) /* bytes/page dir */ +#define PDRMASK (NBPDR-1) +/* Size of the level 3 page directory pointer table units */ +#define NPDPEPG (PAGE_SIZE/(sizeof (pdp_entry_t))) +#define PDPSHIFT 30 /* LOG2(NBPDP) */ +#define NBPDP (1<<PDPSHIFT) /* bytes/page dir ptr table */ +#define PDPMASK (NBPDP-1) +/* Size of the level 4 page-map level-4 table units */ +#define NPML4EPG (PAGE_SIZE/(sizeof (pml4_entry_t))) +#define PML4SHIFT 39 /* LOG2(NBPML4T) */ +#define NBPML4T (1ul<<PML4SHIFT)/* bytes/page map lev4 table */ +#define PML4MASK (NBPML4T-1) + +#define NKPML4E 1 /* addressable number of page tables/pde's */ +#define NKPDPE 1 /* addressable number of page tables/pde's */ +#define NPGPTD 4 + +#define NBPTD (NPGPTD<<PAGE_SHIFT) +#define NPDEPTD (NBPTD/(sizeof (pd_entry_t))) #define IOPAGES 2 /* pages of i/o permission bitmap */ -#ifndef KSTACK_PAGES -#define KSTACK_PAGES 2 /* pages of kstack (with pcb) */ -#endif +#define KSTACK_PAGES 4 /* pages of kstack (with pcb) */ #define UAREA_PAGES 1 /* holds struct user WITHOUT PCB (see def.) */ -#define KSTACK_GUARD 1 /* compile in the kstack guard page */ +#define KSTACK_GUARD 1 /* compile in the kstack guard page */ + +/* + * Ceiling on amount of swblock kva space, can be changed via + * the kern.maxswzone /boot/loader.conf variable. + */ +#ifndef VM_SWZONE_SIZE_MAX +#define VM_SWZONE_SIZE_MAX (32 * 1024 * 1024) +#endif + +/* + * Ceiling on size of buffer cache (really only effects write queueing, + * the VM page cache is not effected), can be changed via + * the kern.maxbcache /boot/loader.conf variable. + */ +#ifndef VM_BCACHE_SIZE_MAX +#define VM_BCACHE_SIZE_MAX (200 * 1024 * 1024) +#endif /* * Mach derived conversion macros */ #define round_page(x) ((((unsigned long)(x)) + PAGE_MASK) & ~(PAGE_MASK)) #define trunc_page(x) ((unsigned long)(x) & ~(PAGE_MASK)) -#define trunc_4mpage(x) ((unsigned)(x) & ~PDRMASK) -#define round_4mpage(x) ((((unsigned)(x)) + PDRMASK) & ~PDRMASK) +#define trunc_2mpage(x) ((unsigned long)(x) & ~PDRMASK) +#define round_2mpage(x) ((((unsigned long)(x)) + PDRMASK) & ~PDRMASK) #define atop(x) ((unsigned long)(x) >> PAGE_SHIFT) #define ptoa(x) ((unsigned long)(x) << PAGE_SHIFT) @@ -124,7 +155,7 @@ #define amd64_btop(x) ((unsigned long)(x) >> PAGE_SHIFT) #define amd64_ptob(x) ((unsigned long)(x) << PAGE_SHIFT) -#define pgtok(x) ((x) * (PAGE_SIZE / 1024)) +#define pgtok(x) ((unsigned long)(x) * (PAGE_SIZE / 1024)) #endif /* !_MACHINE_PARAM_H_ */ #endif /* !_NO_NAMESPACE_POLLUTION */ diff --git a/sys/amd64/include/pcb.h b/sys/amd64/include/pcb.h index c7a837b..551bf8f 100644 --- a/sys/amd64/include/pcb.h +++ b/sys/amd64/include/pcb.h @@ -46,34 +46,25 @@ #include <machine/npx.h> struct pcb { - int pcb_cr3; - int pcb_edi; - int pcb_esi; - int pcb_ebp; - int pcb_esp; - int pcb_ebx; - int pcb_eip; + register_t padxx[8]; + register_t pcb_cr3; + register_t pcb_r15; + register_t pcb_r14; + register_t pcb_r13; + register_t pcb_r12; + register_t pcb_rbp; + register_t pcb_rsp; + register_t pcb_rbx; + register_t pcb_rip; + register_t pcb_rflags; - int pcb_dr0; - int pcb_dr1; - int pcb_dr2; - int pcb_dr3; - int pcb_dr6; - int pcb_dr7; - - union savefpu pcb_save; - u_int pcb_flags; -#define FP_SOFTFP 0x01 /* process using software fltng pnt emulator */ -#define PCB_DBREGS 0x02 /* process using debug registers */ -#define PCB_NPXTRAP 0x04 /* npx trap pending */ -#define PCB_NPXINITDONE 0x08 /* fpu state is initialized */ -#define PCB_VM86CALL 0x10 /* in vm86 call */ + struct savefpu pcb_save; + u_long pcb_flags; +#define PCB_NPXTRAP 0x01 /* npx trap pending */ +#define PCB_NPXINITDONE 0x02 /* fpu state is initialized */ +#define PCB_FULLCTX 0x04 /* full context restore on sysret */ caddr_t pcb_onfault; /* copyin/out fault recovery */ - int pcb_gs; - struct pcb_ext *pcb_ext; /* optional pcb extension */ - int pcb_psl; /* process status long */ - u_long __pcb_spare[2]; /* adjust to avoid core dump size changes */ }; #ifdef _KERNEL diff --git a/sys/amd64/include/pcb_ext.h b/sys/amd64/include/pcb_ext.h index cf5a911..6d1bde2 100644 --- a/sys/amd64/include/pcb_ext.h +++ b/sys/amd64/include/pcb_ext.h @@ -29,24 +29,4 @@ #ifndef _I386_PCB_EXT_H_ #define _I386_PCB_EXT_H_ -/* - * Extension to the 386 process control block - */ -#include <machine/tss.h> -#include <machine/vm86.h> -#include <machine/segments.h> - -struct pcb_ext { - struct segment_descriptor ext_tssd; /* tss descriptor */ - struct i386tss ext_tss; /* per-process i386tss */ - caddr_t ext_iomap; /* i/o permission bitmap */ - struct vm86_kernel ext_vm86; /* vm86 area */ -}; - -#ifdef _KERNEL - -int i386_extend_pcb(struct thread *); - -#endif - #endif /* _I386_PCB_EXT_H_ */ diff --git a/sys/amd64/include/pci_cfgreg.h b/sys/amd64/include/pci_cfgreg.h index b173531..75840aa 100644 --- a/sys/amd64/include/pci_cfgreg.h +++ b/sys/amd64/include/pci_cfgreg.h @@ -38,12 +38,7 @@ #define CONF1_ENABLE_RES1 0x80000000ul #define CONF2_ENABLE_PORT 0x0cf8 -#ifdef PC98 -#define CONF2_FORWARD_PORT 0x0cf9 -#else #define CONF2_FORWARD_PORT 0x0cfa -#endif - #define CONF2_ENABLE_CHK 0x0e #define CONF2_ENABLE_RES 0x0e @@ -51,4 +46,3 @@ int pci_cfgregopen(void); u_int32_t pci_cfgregread(int bus, int slot, int func, int reg, int bytes); void pci_cfgregwrite(int bus, int slot, int func, int reg, u_int32_t data, int bytes); int pci_cfgintr(int bus, int device, int pin, int oldirq); -int pci_probe_route_table(int bus); diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h index a680645..9543b23 100644 --- a/sys/amd64/include/pcpu.h +++ b/sys/amd64/include/pcpu.h @@ -31,9 +31,6 @@ #ifdef _KERNEL -#include <machine/segments.h> -#include <machine/tss.h> - /* * The SMP parts are setup in pmap.c and locore.s for the BSP, and * mp_machdep.c sets up the data for the AP's to "see" when they awake. @@ -42,15 +39,8 @@ * other processors" */ #define PCPU_MD_FIELDS \ - struct pcpu *pc_prvspace; /* Self-reference */ \ - struct i386tss pc_common_tss; \ - struct segment_descriptor pc_common_tssd; \ - struct segment_descriptor *pc_tss_gdt; \ - int pc_currentldt; \ - u_int32_t pc_int_pending; /* master int pending flag */ \ - u_int32_t pc_ipending; /* pending slow interrupts */ \ - u_int32_t pc_fpending; /* pending fast interrupts */ \ - u_int32_t pc_spending /* pending soft interrupts */ + struct pcpu *pc_prvspace; /* Self-reference */ \ + register_t pc_scratch_rsp; /* User %rsp in syscall */ #if defined(lint) @@ -80,7 +70,7 @@ extern struct pcpu *pcpup; #define __PCPU_PTR(name) ({ \ __pcpu_type(name) *__p; \ \ - __asm __volatile("movl %%fs:%1,%0; addl %2,%0" \ + __asm __volatile("movq %%gs:%1,%0; addq %2,%0" \ : "=r" (__p) \ : "m" (*(struct pcpu *)(__pcpu_offset(pc_prvspace))), \ "i" (__pcpu_offset(name))); \ @@ -96,22 +86,28 @@ extern struct pcpu *pcpup; \ if (sizeof(__result) == 1) { \ u_char __b; \ - __asm __volatile("movb %%fs:%1,%0" \ + __asm __volatile("movb %%gs:%1,%0" \ : "=r" (__b) \ : "m" (*(u_char *)(__pcpu_offset(name)))); \ __result = *(__pcpu_type(name) *)&__b; \ } else if (sizeof(__result) == 2) { \ u_short __w; \ - __asm __volatile("movw %%fs:%1,%0" \ + __asm __volatile("movw %%gs:%1,%0" \ : "=r" (__w) \ : "m" (*(u_short *)(__pcpu_offset(name)))); \ __result = *(__pcpu_type(name) *)&__w; \ } else if (sizeof(__result) == 4) { \ u_int __i; \ - __asm __volatile("movl %%fs:%1,%0" \ + __asm __volatile("movl %%gs:%1,%0" \ : "=r" (__i) \ : "m" (*(u_int *)(__pcpu_offset(name)))); \ __result = *(__pcpu_type(name) *)&__i; \ + } else if (sizeof(__result) == 8) { \ + u_long __l; \ + __asm __volatile("movq %%gs:%1,%0" \ + : "=r" (__l) \ + : "m" (*(u_long *)(__pcpu_offset(name)))); \ + __result = *(__pcpu_type(name) *)&__l; \ } else { \ __result = *__PCPU_PTR(name); \ } \ @@ -128,21 +124,27 @@ extern struct pcpu *pcpup; if (sizeof(__val) == 1) { \ u_char __b; \ __b = *(u_char *)&__val; \ - __asm __volatile("movb %1,%%fs:%0" \ + __asm __volatile("movb %1,%%gs:%0" \ : "=m" (*(u_char *)(__pcpu_offset(name))) \ : "r" (__b)); \ } else if (sizeof(__val) == 2) { \ u_short __w; \ __w = *(u_short *)&__val; \ - __asm __volatile("movw %1,%%fs:%0" \ + __asm __volatile("movw %1,%%gs:%0" \ : "=m" (*(u_short *)(__pcpu_offset(name))) \ : "r" (__w)); \ } else if (sizeof(__val) == 4) { \ u_int __i; \ __i = *(u_int *)&__val; \ - __asm __volatile("movl %1,%%fs:%0" \ + __asm __volatile("movl %1,%%gs:%0" \ : "=m" (*(u_int *)(__pcpu_offset(name))) \ : "r" (__i)); \ + } else if (sizeof(__val) == 8) { \ + u_long __l; \ + __l = *(u_long *)&__val; \ + __asm __volatile("movq %1,%%gs:%0" \ + : "=m" (*(u_long *)(__pcpu_offset(name))) \ + : "r" (__l)); \ } else { \ *__PCPU_PTR(name) = __val; \ } \ diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h index 58e9182..d3e2104 100644 --- a/sys/amd64/include/pmap.h +++ b/sys/amd64/include/pmap.h @@ -83,52 +83,36 @@ #define PGEX_U 0x04 /* access from User mode (UPL) */ /* - * Size of Kernel address space. This is the number of page table pages - * (4MB each) to use for the kernel. 256 pages == 1 Gigabyte. - * This **MUST** be a multiple of 4 (eg: 252, 256, 260, etc). + * Size of Kernel address space. This is the number of level 4 (top) + * entries. We use half of them for the kernel due to the 48 bit + * virtual address sign extension. */ -#ifndef KVA_PAGES -#ifdef PAE -#define KVA_PAGES 512 -#else -#define KVA_PAGES 256 -#endif -#endif - +#define KVA_PAGES 1536 + /* - * Pte related macros + * Pte related macros. This is complicated by having to deal with + * the sign extension of the 48th bit. */ -#define VADDR(pdi, pti) ((vm_offset_t)(((pdi)<<PDRSHIFT)|((pti)<<PAGE_SHIFT))) +#define VADDR_SIGN(l4) \ + ((l4) >= NPML4EPG/2 ? ((unsigned long)-1 << 47) : 0ul) +#define VADDR(l4, l3, l2, l1) ( \ + ((unsigned long)(l4) << PML4SHIFT) | VADDR_SIGN(l4) | \ + ((unsigned long)(l3) << PDPSHIFT) | \ + ((unsigned long)(l2) << PDRSHIFT) | \ + ((unsigned long)(l1) << PAGE_SHIFT)) + #ifndef NKPT -#ifdef PAE -#define NKPT 120 /* actual number of kernel page tables */ -#else -#define NKPT 30 /* actual number of kernel page tables */ -#endif -#endif -#ifndef NKPDE -#ifdef SMP -#define NKPDE (KVA_PAGES - 1) /* number of page tables/pde's */ -#else -#define NKPDE (KVA_PAGES) /* number of page tables/pde's */ +#define NKPT 120 /* initial number of kernel page tables */ #endif +#ifndef NKPDE +#define NKPDE (KVA_PAGES) /* number of page tables/pde's */ #endif /* * The *PTDI values control the layout of virtual memory - * - * XXX This works for now, but I am not real happy with it, I'll fix it - * right after I fix locore.s and the magic 28K hole - * - * SMP_PRIVPAGES: The per-cpu address space is 0xff80000 -> 0xffbfffff */ -#ifdef SMP -#define MPPTDI (NPDEPTD-1) /* per cpu ptd entry */ -#define KPTDI (MPPTDI-NKPDE) /* start of kernel virtual pde's */ -#else -#define KPTDI (NPDEPTD-NKPDE)/* start of kernel virtual pde's */ -#endif /* SMP */ +#define KPTDI (NPDEPTD-NKPDE) /* start of kernel virtual pde's */ #define PTDPTDI (KPTDI-NPGPTD) /* ptd entry that points to ptd! */ /* @@ -141,38 +125,37 @@ #include <sys/queue.h> -#ifdef PAE - -typedef uint64_t pdpt_entry_t; -typedef uint64_t pd_entry_t; -typedef uint64_t pt_entry_t; +typedef u_int64_t pd_entry_t; +typedef u_int64_t pt_entry_t; +typedef u_int64_t pdp_entry_t; +typedef u_int64_t pml4_entry_t; +#define PML4ESHIFT (3) +#define PDPESHIFT (3) #define PTESHIFT (3) #define PDESHIFT (3) -#else - -typedef uint32_t pd_entry_t; -typedef uint32_t pt_entry_t; - -#define PTESHIFT (2) -#define PDESHIFT (2) - -#endif - /* * Address of current and alternate address space page table maps * and directories. + * XXX it might be saner to just direct map all of physical memory + * into the kernel using 2MB pages. We have enough space to do + * it (2^47 bits of KVM, while current max physical addressability + * is 2^40 physical bits). Then we can get rid of the evil hole + * in the page tables and the evil overlapping. */ #ifdef _KERNEL -extern pt_entry_t PTmap[]; -extern pd_entry_t PTD[]; -extern pd_entry_t PTDpde[]; - -#ifdef PAE -extern pdpt_entry_t *IdlePDPT; -#endif -extern pd_entry_t *IdlePTD; /* physical address of "Idle" state directory */ +extern pt_entry_t PTmap[]; +extern pd_entry_t PDmap[]; +extern pdp_entry_t PDPmap[]; +extern pml4_entry_t PML4[]; +extern pdp_entry_t PDP[]; +extern pd_entry_t PTD[]; +extern pd_entry_t PTDpde[]; + +extern u_int64_t IdlePML4; /* physical address of "Idle" state directory */ +extern u_int64_t IdlePDP; /* physical address of "Idle" state directory */ +extern u_int64_t IdlePTD; /* physical address of "Idle" state directory */ #endif #ifdef _KERNEL @@ -182,7 +165,7 @@ extern pd_entry_t *IdlePTD; /* physical address of "Idle" state directory */ * Note: these work recursively, thus vtopte of a pte will give * the corresponding pde that in turn maps it. */ -#define vtopte(va) (PTmap + i386_btop(va)) +#define vtopte(va) (PTmap + amd64_btop(va)) /* * Routine: pmap_kextract @@ -195,7 +178,8 @@ pmap_kextract(vm_offset_t va) { vm_paddr_t pa; - if ((pa = PTD[va >> PDRSHIFT]) & PG_PS) { + pa = PTD[va >> PDRSHIFT]; + if (pa & PG_PS) { pa = (pa & ~(NBPDR - 1)) | (va & (NBPDR - 1)); } else { pa = *vtopte(va); @@ -206,39 +190,6 @@ pmap_kextract(vm_offset_t va) #define vtophys(va) pmap_kextract(((vm_offset_t) (va))) -#ifdef PAE - -static __inline pt_entry_t -pte_load(pt_entry_t *ptep) -{ - pt_entry_t r; - - __asm __volatile( - "lock; cmpxchg8b %1" - : "=A" (r) - : "m" (*ptep), "a" (0), "d" (0), "b" (0), "c" (0)); - return (r); -} - -static __inline pt_entry_t -pte_load_store(pt_entry_t *ptep, pt_entry_t v) -{ - pt_entry_t r; - - r = *ptep; - __asm __volatile( - "1:\n" - "\tlock; cmpxchg8b %1\n" - "\tjnz 1b" - : "+A" (r) - : "m" (*ptep), "b" ((uint32_t)v), "c" ((uint32_t)(v >> 32))); - return (r); -} - -#define pte_load_clear(ptep) pte_load_store((ptep), (pt_entry_t)0ULL) - -#else /* PAE */ - static __inline pt_entry_t pte_load(pt_entry_t *ptep) { @@ -258,9 +209,7 @@ pte_load_store(pt_entry_t *ptep, pt_entry_t pte) return (r); } -#define pte_load_clear(pte) atomic_readandclear_int(pte) - -#endif /* PAE */ +#define pte_load_clear(pte) atomic_readandclear_long(pte) #define pte_clear(ptep) pte_load_store((ptep), (pt_entry_t)0ULL) #define pte_store(ptep, pte) pte_load_store((ptep), (pt_entry_t)pte) @@ -283,13 +232,11 @@ struct pmap { pd_entry_t *pm_pdir; /* KVA of page directory */ vm_object_t pm_pteobj; /* Container for pte's */ TAILQ_HEAD(,pv_entry) pm_pvlist; /* list of mappings in pmap */ - u_int pm_active; /* active on cpus */ + u_long pm_active; /* active on cpus */ struct pmap_statistics pm_stats; /* pmap statistics */ LIST_ENTRY(pmap) pm_list; /* List of all pmaps */ -#ifdef PAE - pdpt_entry_t *pm_pdpt; /* KVA of page director pointer - table */ -#endif + pdp_entry_t *pm_pdp; /* KVA of level 3 page table */ + pml4_entry_t *pm_pml4; /* KVA of level 4 page table */ }; #define pmap_page_is_mapped(m) (!TAILQ_EMPTY(&(m)->md.pv_list)) @@ -341,7 +288,6 @@ void pmap_kremove(vm_offset_t); void *pmap_mapdev(vm_paddr_t, vm_size_t); void pmap_unmapdev(vm_offset_t, vm_size_t); pt_entry_t *pmap_pte_quick(pmap_t, vm_offset_t) __pure2; -void pmap_set_opt(void); void pmap_invalidate_page(pmap_t, vm_offset_t); void pmap_invalidate_range(pmap_t, vm_offset_t, vm_offset_t); void pmap_invalidate_all(pmap_t); diff --git a/sys/amd64/include/proc.h b/sys/amd64/include/proc.h index bff037a..cd54c24 100644 --- a/sys/amd64/include/proc.h +++ b/sys/amd64/include/proc.h @@ -37,35 +37,14 @@ #ifndef _MACHINE_PROC_H_ #define _MACHINE_PROC_H_ -#include <machine/segments.h> - -struct proc_ldt { - caddr_t ldt_base; - int ldt_len; - int ldt_refcnt; - u_long ldt_active; - struct segment_descriptor ldt_sd; -}; - /* - * Machine-dependent part of the proc structure for i386. + * Machine-dependent part of the proc structure for AMD64. */ struct mdthread { -#ifdef lint - int dummy; -#endif + register_t md_savecrit; }; struct mdproc { - struct proc_ldt *md_ldt; /* (j) per-process ldt */ }; -#ifdef _KERNEL - -void set_user_ldt(struct mdproc *); -struct proc_ldt *user_ldt_alloc(struct mdproc *, int); -void user_ldt_free(struct thread *); - -#endif /* _KERNEL */ - #endif /* !_MACHINE_PROC_H_ */ diff --git a/sys/amd64/include/profile.h b/sys/amd64/include/profile.h index 0df3195..7330ae0 100644 --- a/sys/amd64/include/profile.h +++ b/sys/amd64/include/profile.h @@ -65,14 +65,14 @@ #define MCOUNT_DECL(s) u_long s; #ifdef SMP extern int mcount_lock; -#define MCOUNT_ENTER(s) { s = read_eflags(); disable_intr(); \ +#define MCOUNT_ENTER(s) { s = read_rflags(); disable_intr(); \ while (!atomic_cmpset_acq_int(&mcount_lock, 0, 1)) \ /* nothing */ ; } #define MCOUNT_EXIT(s) { atomic_store_rel_int(&mcount_lock, 0); \ - write_eflags(s); } + write_rflags(s); } #else -#define MCOUNT_ENTER(s) { s = read_eflags(); disable_intr(); } -#define MCOUNT_EXIT(s) (write_eflags(s)) +#define MCOUNT_ENTER(s) { s = read_rflags(); disable_intr(); } +#define MCOUNT_EXIT(s) (write_rflags(s)) #endif #endif /* GUPROF */ @@ -94,14 +94,14 @@ mcount() \ * \ * selfpc = pc pushed by call to mcount \ */ \ - asm("movl 4(%%ebp),%0" : "=r" (selfpc)); \ + asm("movq 8(%%rbp),%0" : "=r" (selfpc)); \ /* \ * frompc = pc pushed by call to mcount's caller. \ * The caller's stack frame has already been built, so %ebp is \ * the caller's frame pointer. The caller's raddr is in the \ * caller's frame following the caller's caller's frame pointer.\ */ \ - asm("movl (%%ebp),%0" : "=r" (frompc)); \ + asm("movq (%%rbp),%0" : "=r" (frompc)); \ frompc = ((uintfptr_t *)frompc)[1]; \ _mcount(frompc, selfpc); \ } @@ -113,7 +113,7 @@ mcount() \ } #endif /* __GNUC__ */ -typedef unsigned int uintfptr_t; +typedef unsigned long uintfptr_t; #endif /* _KERNEL */ diff --git a/sys/amd64/include/psl.h b/sys/amd64/include/psl.h index 302d469..04d83ff 100644 --- a/sys/amd64/include/psl.h +++ b/sys/amd64/include/psl.h @@ -55,10 +55,10 @@ #define PSL_IOPL 0x00003000 /* i/o privilege level */ #define PSL_NT 0x00004000 /* nested task bit */ #define PSL_RF 0x00010000 /* resume flag bit */ -#define PSL_VM 0x00020000 /* virtual 8086 mode bit */ +/* #define PSL_VM 0x00020000 */ /* virtual 8086 mode bit */ #define PSL_AC 0x00040000 /* alignment checking */ -#define PSL_VIF 0x00080000 /* virtual interrupt enable */ -#define PSL_VIP 0x00100000 /* virtual interrupt pending */ +/* #define PSL_VIF 0x00080000 */ /* virtual interrupt enable */ +/* #define PSL_VIP 0x00100000 */ /* virtual interrupt pending */ #define PSL_ID 0x00200000 /* identification bit */ /* diff --git a/sys/amd64/include/reg.h b/sys/amd64/include/reg.h index 56f0f91..334dd37 100644 --- a/sys/amd64/include/reg.h +++ b/sys/amd64/include/reg.h @@ -41,66 +41,31 @@ #define _MACHINE_REG_H_ /* - * Indices for registers in `struct trapframe' and `struct regs'. - * - * This interface is deprecated. In the kernel, it is only used in FPU - * emulators to convert from register numbers encoded in instructions to - * register values. Everything else just accesses the relevant struct - * members. In userland, debuggers tend to abuse this interface since - * they don't understand that `struct regs' is a struct. I hope they have - * stopped accessing the registers in the trap frame via PT_{READ,WRITE}_U - * and we can stop supporting the user area soon. - */ -#define tFS (0) -#define tES (1) -#define tDS (2) -#define tEDI (3) -#define tESI (4) -#define tEBP (5) -#define tISP (6) -#define tEBX (7) -#define tEDX (8) -#define tECX (9) -#define tEAX (10) -#define tERR (12) -#define tEIP (13) -#define tCS (14) -#define tEFLAGS (15) -#define tESP (16) -#define tSS (17) - -/* - * Indices for registers in `struct regs' only. - * - * Some registers live in the pcb and are only in an "array" with the - * other registers in application interfaces that copy all the registers - * to or from a `struct regs'. - */ -#define tGS (18) - -/* * Register set accessible via /proc/$pid/regs and PT_{SET,GET}REGS. */ struct reg { - unsigned int r_fs; - unsigned int r_es; - unsigned int r_ds; - unsigned int r_edi; - unsigned int r_esi; - unsigned int r_ebp; - unsigned int r_isp; - unsigned int r_ebx; - unsigned int r_edx; - unsigned int r_ecx; - unsigned int r_eax; - unsigned int r_trapno; - unsigned int r_err; - unsigned int r_eip; - unsigned int r_cs; - unsigned int r_eflags; - unsigned int r_esp; - unsigned int r_ss; - unsigned int r_gs; + register_t r_r15; + register_t r_r14; + register_t r_r13; + register_t r_r12; + register_t r_r11; + register_t r_r10; + register_t r_r9; + register_t r_r8; + register_t r_rdi; + register_t r_rsi; + register_t r_rbp; + register_t r_rbx; + register_t r_rdx; + register_t r_rcx; + register_t r_rax; + register_t r_trapno; + register_t r_err; + register_t r_rip; + register_t r_cs; + register_t r_rflags; + register_t r_rsp; + register_t r_ss; }; /* @@ -112,30 +77,15 @@ struct fpreg { * simplified struct. This may be too much detail. Perhaps * an array of unsigned longs is best. */ - unsigned long fpr_env[7]; - unsigned char fpr_acc[8][10]; - unsigned long fpr_ex_sw; - unsigned char fpr_pad[64]; + unsigned long fpr_env[4]; + unsigned char fpr_acc[8][16]; + unsigned char fpr_xacc[16][16]; }; -/* - * Register set accessible via /proc/$pid/dbregs. - */ struct dbreg { - unsigned int dr[8]; /* debug registers */ - /* Index 0-3: debug address registers */ - /* Index 4-5: reserved */ - /* Index 6: debug status */ - /* Index 7: debug control */ + unsigned long grrr; }; -#define DBREG_DR7_EXEC 0x00 /* break on execute */ -#define DBREG_DR7_WRONLY 0x01 /* break on write */ -#define DBREG_DR7_RDWR 0x03 /* break on read or write */ -#define DBREG_DRX(d,x) ((d)->dr[(x)]) /* reference dr0 - dr7 by - register number */ - - #ifdef _KERNEL /* * XXX these interfaces are MI, so they should be declared in a MI place. diff --git a/sys/amd64/include/segments.h b/sys/amd64/include/segments.h index 95117da..0f9a59b 100644 --- a/sys/amd64/include/segments.h +++ b/sys/amd64/include/segments.h @@ -42,8 +42,7 @@ #define _MACHINE_SEGMENTS_H_ /* - * 386 Segmentation Data Structures and definitions - * William F. Jolitz (william@ernie.berkeley.edu) 6/20/1989 + * AMD64 Segmentation Data Structures and definitions */ /* @@ -60,60 +59,66 @@ #define GSEL(s,r) (((s)<<3) | r) /* a global selector */ /* - * Memory and System segment descriptors + * User segment descriptors (%cs, %ds etc for compatability apps. 64 bit wide) + * For long-mode apps, %cs only has the conforming bit in sd_type, the sd_dpl, + * sd_p, sd_l and sd_def32 which must be zero). %ds only has sd_p. */ -struct segment_descriptor { - unsigned sd_lolimit:16 ; /* segment extent (lsb) */ - unsigned sd_lobase:24 __packed; /* segment base address (lsb) */ - unsigned sd_type:5 ; /* segment type */ - unsigned sd_dpl:2 ; /* segment descriptor priority level */ - unsigned sd_p:1 ; /* segment descriptor present */ - unsigned sd_hilimit:4 ; /* segment extent (msb) */ - unsigned sd_xx:2 ; /* unused */ - unsigned sd_def32:1 ; /* default 32 vs 16 bit size */ - unsigned sd_gran:1 ; /* limit granularity (byte/page units)*/ - unsigned sd_hibase:8 ; /* segment base address (msb) */ -} ; +struct user_segment_descriptor { + u_int64_t sd_lolimit:16; /* segment extent (lsb) */ + u_int64_t sd_lobase:24; /* segment base address (lsb) */ + u_int64_t sd_type:5; /* segment type */ + u_int64_t sd_dpl:2; /* segment descriptor priority level */ + u_int64_t sd_p:1; /* segment descriptor present */ + u_int64_t sd_hilimit:4; /* segment extent (msb) */ + u_int64_t sd_xx:1; /* unused */ + u_int64_t sd_long:1; /* long mode (cs only) */ + u_int64_t sd_def32:1; /* default 32 vs 16 bit size */ + u_int64_t sd_gran:1; /* limit granularity (byte/page units)*/ + u_int64_t sd_hibase:8; /* segment base address (msb) */ +} __packed; /* - * Gate descriptors (e.g. indirect descriptors) + * System segment descriptors (128 bit wide) */ -struct gate_descriptor { - unsigned gd_looffset:16 ; /* gate offset (lsb) */ - unsigned gd_selector:16 ; /* gate segment selector */ - unsigned gd_stkcpy:5 ; /* number of stack wds to cpy */ - unsigned gd_xx:3 ; /* unused */ - unsigned gd_type:5 ; /* segment type */ - unsigned gd_dpl:2 ; /* segment descriptor priority level */ - unsigned gd_p:1 ; /* segment descriptor present */ - unsigned gd_hioffset:16 ; /* gate offset (msb) */ -} ; +struct system_segment_descriptor { + u_int64_t sd_lolimit:16; /* segment extent (lsb) */ + u_int64_t sd_lobase:24; /* segment base address (lsb) */ + u_int64_t sd_type:5; /* segment type */ + u_int64_t sd_dpl:2; /* segment descriptor priority level */ + u_int64_t sd_p:1; /* segment descriptor present */ + u_int64_t sd_hilimit:4; /* segment extent (msb) */ + u_int64_t sd_xx0:3; /* unused */ + u_int64_t sd_gran:1; /* limit granularity (byte/page units)*/ + u_int64_t sd_hibase:40 __packed;/* segment base address (msb) */ + u_int64_t sd_xx1:8; + u_int64_t sd_mbz:5; /* MUST be zero */ + u_int64_t sd_xx2:19; +} __packed; /* - * Generic descriptor + * Gate descriptors (e.g. indirect descriptors, trap, interrupt etc. 128 bit) + * Only interrupt and trap gates have gd_ist. */ -union descriptor { - struct segment_descriptor sd; - struct gate_descriptor gd; -}; +struct gate_descriptor { + u_int64_t gd_looffset:16; /* gate offset (lsb) */ + u_int64_t gd_selector:16; /* gate segment selector */ + u_int64_t gd_ist:3; /* IST table index */ + u_int64_t gd_xx:5; /* unused */ + u_int64_t gd_type:5; /* segment type */ + u_int64_t gd_dpl:2; /* segment descriptor priority level */ + u_int64_t gd_p:1; /* segment descriptor present */ + u_int64_t gd_hioffset:48 __packed; /* gate offset (msb) */ + u_int64_t sd_xx1:32; +} __packed; /* system segments and gate types */ #define SDT_SYSNULL 0 /* system null */ -#define SDT_SYS286TSS 1 /* system 286 TSS available */ -#define SDT_SYSLDT 2 /* system local descriptor table */ -#define SDT_SYS286BSY 3 /* system 286 TSS busy */ -#define SDT_SYS286CGT 4 /* system 286 call gate */ -#define SDT_SYSTASKGT 5 /* system task gate */ -#define SDT_SYS286IGT 6 /* system 286 interrupt gate */ -#define SDT_SYS286TGT 7 /* system 286 trap gate */ -#define SDT_SYSNULL2 8 /* system null again */ -#define SDT_SYS386TSS 9 /* system 386 TSS available */ -#define SDT_SYSNULL3 10 /* system null again */ -#define SDT_SYS386BSY 11 /* system 386 TSS busy */ -#define SDT_SYS386CGT 12 /* system 386 call gate */ -#define SDT_SYSNULL4 13 /* system null again */ -#define SDT_SYS386IGT 14 /* system 386 interrupt gate */ -#define SDT_SYS386TGT 15 /* system 386 trap gate */ +#define SDT_SYSLDT 2 /* system 64 bit local descriptor table */ +#define SDT_SYSTSS 9 /* system available 64 bit TSS */ +#define SDT_SYSBSY 11 /* system busy 64 bit TSS */ +#define SDT_SYSCGT 12 /* system 64 bit call gate */ +#define SDT_SYSIGT 14 /* system 64 bit interrupt gate */ +#define SDT_SYSTGT 15 /* system 64 bit trap gate */ /* memory segment types */ #define SDT_MEMRO 16 /* memory read only */ @@ -139,45 +144,29 @@ union descriptor { * when needed to be used by the 386 hardware */ -struct soft_segment_descriptor { - unsigned ssd_base ; /* segment base address */ - unsigned ssd_limit ; /* segment extent */ - unsigned ssd_type:5 ; /* segment type */ - unsigned ssd_dpl:2 ; /* segment descriptor priority level */ - unsigned ssd_p:1 ; /* segment descriptor present */ - unsigned ssd_xx:4 ; /* unused */ - unsigned ssd_xx1:2 ; /* unused */ - unsigned ssd_def32:1 ; /* default 32 vs 16 bit size */ - unsigned ssd_gran:1 ; /* limit granularity (byte/page units)*/ -}; +struct soft_segment_descriptor { + unsigned long ssd_base; /* segment base address */ + unsigned long ssd_limit; /* segment extent */ + unsigned long ssd_type:5; /* segment type */ + unsigned long ssd_dpl:2; /* segment descriptor priority level */ + unsigned long ssd_p:1; /* segment descriptor present */ + unsigned long ssd_long:1; /* long mode (for %cs) */ + unsigned long ssd_def32:1; /* default 32 vs 16 bit size */ + unsigned long ssd_gran:1; /* limit granularity (byte/page units)*/ +} __packed; /* * region descriptors, used to load gdt/idt tables before segments yet exist. */ struct region_descriptor { - unsigned rd_limit:16; /* segment extent */ - unsigned rd_base:32 __packed; /* base address */ -}; - -/* - * Segment Protection Exception code bits - */ - -#define SEGEX_EXT 0x01 /* recursive or externally induced */ -#define SEGEX_IDT 0x02 /* interrupt descriptor table */ -#define SEGEX_TI 0x04 /* local descriptor table */ - /* other bits are affected descriptor index */ -#define SEGEX_IDX(s) (((s)>>3)&0x1fff) + unsigned long rd_limit:16; /* segment extent */ + unsigned long rd_base:64 __packed; /* base address */ +} __packed; /* * Size of IDT table */ - -#if defined(SMP) || defined(APIC_IO) -#define NIDT 256 /* we use them all */ -#else #define NIDT 129 /* 32 reserved, 16 h/w, 0 s/w, linux's 0x80 */ -#endif /* SMP || APIC_IO */ #define NRSVIDT 32 /* reserved entries for cpu exceptions */ /* @@ -186,47 +175,25 @@ struct region_descriptor { #define GNULL_SEL 0 /* Null Descriptor */ #define GCODE_SEL 1 /* Kernel Code Descriptor */ #define GDATA_SEL 2 /* Kernel Data Descriptor */ -#define GPRIV_SEL 3 /* SMP Per-Processor Private Data */ -#define GPROC0_SEL 4 /* Task state process slot zero and up */ -#define GLDT_SEL 5 /* LDT - eventually one per process */ -#define GUSERLDT_SEL 6 /* User LDT */ -#define GTGATE_SEL 7 /* Process task switch gate */ -#define GBIOSLOWMEM_SEL 8 /* BIOS low memory access (must be entry 8) */ -#define GPANIC_SEL 9 /* Task state to consider panic from */ -#define GBIOSCODE32_SEL 10 /* BIOS interface (32bit Code) */ -#define GBIOSCODE16_SEL 11 /* BIOS interface (16bit Code) */ -#define GBIOSDATA_SEL 12 /* BIOS interface (Data) */ -#define GBIOSUTIL_SEL 13 /* BIOS interface (Utility) */ -#define GBIOSARGS_SEL 14 /* BIOS interface (Arguments) */ - -#define NGDT 15 - -/* - * Entries in the Local Descriptor Table (LDT) - */ -#define LSYS5CALLS_SEL 0 /* forced by intel BCS */ -#define LSYS5SIGR_SEL 1 -#define L43BSDCALLS_SEL 2 /* notyet */ -#define LUCODE_SEL 3 -#define LSOL26CALLS_SEL 4 /* Solaris >= 2.6 system call gate */ -#define LUDATA_SEL 5 -/* separate stack, es,fs,gs sels ? */ -/* #define LPOSIXCALLS_SEL 5*/ /* notyet */ -#define LBSDICALLS_SEL 16 /* BSDI system call gate */ -#define NLDT (LBSDICALLS_SEL + 1) +#define GUCODE32_SEL 3 /* User 32 bit code Descriptor */ +#define GUDATA_SEL 4 /* User 32/64 bit Data Descriptor */ +#define GUCODE_SEL 5 /* User 64 bit Code Descriptor */ +#define GPROC0_SEL 6 /* TSS for entering kernel etc */ +/* slot 6 is second half of GPROC0_SEL */ +#define NGDT 8 #ifdef _KERNEL -extern int _default_ldt; -extern union descriptor gdt[]; +extern struct user_segment_descriptor gdt[]; extern struct soft_segment_descriptor gdt_segs[]; extern struct gate_descriptor *idt; -extern union descriptor ldt[NLDT]; void lgdt(struct region_descriptor *rdp); -void sdtossd(struct segment_descriptor *sdp, +void sdtossd(struct user_segment_descriptor *sdp, struct soft_segment_descriptor *ssdp); void ssdtosd(struct soft_segment_descriptor *ssdp, - struct segment_descriptor *sdp); + struct user_segment_descriptor *sdp); +void ssdtosyssd(struct soft_segment_descriptor *ssdp, + struct system_segment_descriptor *sdp); #endif /* _KERNEL */ #endif /* !_MACHINE_SEGMENTS_H_ */ diff --git a/sys/amd64/include/setjmp.h b/sys/amd64/include/setjmp.h index e6e0371..a6a9399 100644 --- a/sys/amd64/include/setjmp.h +++ b/sys/amd64/include/setjmp.h @@ -37,7 +37,7 @@ #include <sys/cdefs.h> -#define _JBLEN 11 /* Size of the jmp_buf on x86. */ +#define _JBLEN 22 /* Size of the jmp_buf on x86. */ /* * jmp_buf and sigjmp_buf are encapsulated in different structs to force diff --git a/sys/amd64/include/sigframe.h b/sys/amd64/include/sigframe.h index 98be731..d104507 100644 --- a/sys/amd64/include/sigframe.h +++ b/sys/amd64/include/sigframe.h @@ -34,74 +34,7 @@ /* * Signal frames, arguments passed to application signal handlers. */ -#ifdef _KERNEL -#ifdef COMPAT_43 -struct osigframe { - /* - * The first four members may be used by applications. - */ - - register_t sf_signum; - - /* - * Either 'int' for old-style FreeBSD handler or 'siginfo_t *' - * pointing to sf_siginfo for SA_SIGINFO handlers. - */ - register_t sf_arg2; - - /* Points to sf_siginfo.si_sc. */ - register_t sf_scp; - - register_t sf_addr; - - /* - * The following arguments are not constrained by the - * function call protocol. - * Applications are not supposed to access these members, - * except using the pointers we provide in the first three - * arguments. - */ - - union { - __osiginfohandler_t *sf_action; - __sighandler_t *sf_handler; - } sf_ahu; - - /* In the SA_SIGINFO case, sf_arg2 points here. */ - osiginfo_t sf_siginfo; -}; -#endif -#ifdef COMPAT_FREEBSD4 -/* FreeBSD 4.x */ -struct sigframe4 { - register_t sf_signum; - register_t sf_siginfo; /* code or pointer to sf_si */ - register_t sf_ucontext; /* points to sf_uc */ - register_t sf_addr; /* undocumented 4th arg */ - - union { - __siginfohandler_t *sf_action; - __sighandler_t *sf_handler; - } sf_ahu; - struct ucontext4 sf_uc; /* = *sf_ucontext */ - siginfo_t sf_si; /* = *sf_siginfo (SA_SIGINFO case) */ -}; -#endif -#endif - struct sigframe { - /* - * The first four members may be used by applications. - * - * NOTE: The 4th argument is undocumented, ill commented - * on and seems to be somewhat BSD "standard". Handlers - * installed with sigvec may be using it. - */ - register_t sf_signum; - register_t sf_siginfo; /* code or pointer to sf_si */ - register_t sf_ucontext; /* points to sf_uc */ - register_t sf_addr; /* undocumented 4th arg */ - union { __siginfohandler_t *sf_action; __sighandler_t *sf_handler; diff --git a/sys/amd64/include/signal.h b/sys/amd64/include/signal.h index fb2d82c..baa0c4e 100644 --- a/sys/amd64/include/signal.h +++ b/sys/amd64/include/signal.h @@ -44,7 +44,7 @@ * Machine-dependent signal definitions */ -typedef int sig_atomic_t; +typedef long sig_atomic_t; #if __XSI_VISIBLE /* @@ -60,7 +60,6 @@ typedef int sig_atomic_t; /* * Only the kernel should need these old type definitions. */ -#if defined(_KERNEL) && defined(COMPAT_43) /* * Information pushed on stack when a signal is delivered. * This is used by the kernel to restore state following @@ -68,75 +67,46 @@ typedef int sig_atomic_t; * to the handler to allow it to restore state properly if * a non-standard exit is performed. */ -struct osigcontext { - int sc_onstack; /* sigstack state to restore */ - osigset_t sc_mask; /* signal mask to restore */ - int sc_esp; /* machine state follows: */ - int sc_ebp; - int sc_isp; - int sc_eip; - int sc_efl; - int sc_es; - int sc_ds; - int sc_cs; - int sc_ss; - int sc_edi; - int sc_esi; - int sc_ebx; - int sc_edx; - int sc_ecx; - int sc_eax; - int sc_gs; - int sc_fs; - int sc_trapno; - int sc_err; -}; -#endif - /* * The sequence of the fields/registers in struct sigcontext should match * those in mcontext_t. */ struct sigcontext { struct __sigset sc_mask; /* signal mask to restore */ - int sc_onstack; /* sigstack state to restore */ - int sc_gs; /* machine state (struct trapframe) */ - int sc_fs; - int sc_es; - int sc_ds; - int sc_edi; - int sc_esi; - int sc_ebp; - int sc_isp; - int sc_ebx; - int sc_edx; - int sc_ecx; - int sc_eax; - int sc_trapno; - int sc_err; - int sc_eip; - int sc_cs; - int sc_efl; - int sc_esp; - int sc_ss; - int sc_len; /* sizeof(mcontext_t) */ + long sc_onstack; /* sigstack state to restore */ + long sc_r15; /* machine state (struct trapframe) */ + long sc_r14; + long sc_r13; + long sc_r12; + long sc_r11; + long sc_r10; + long sc_r9; + long sc_r8; + long sc_rdi; + long sc_rsi; + long sc_rbp; + long sc_rbx; + long sc_rdx; + long sc_rcx; + long sc_rax; + long sc_trapno; + long sc_err; + long sc_rip; + long sc_cs; + long sc_rflags; + long sc_rsp; + long sc_ss; + long sc_len; /* sizeof(mcontext_t) */ /* * XXX - See <machine/ucontext.h> and <machine/npx.h> for * the following fields. */ - int sc_fpformat; - int sc_ownedfp; - int sc_spare1[1]; - int sc_fpstate[128] __aligned(16); - int sc_spare2[8]; + long sc_fpformat; + long sc_ownedfp; + long sc_spare1[1]; + long sc_fpstate[128] __aligned(16); + long sc_spare2[8]; }; - -#define sc_sp sc_esp -#define sc_fp sc_ebp -#define sc_pc sc_eip -#define sc_ps sc_efl -#define sc_eflags sc_efl - #endif /* __BSD_VISIBLE */ #endif /* !_MACHINE_SIGNAL_H_ */ diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h index 6467365..3d8d117 100644 --- a/sys/amd64/include/smp.h +++ b/sys/amd64/include/smp.h @@ -13,150 +13,4 @@ #ifndef _MACHINE_SMP_H_ #define _MACHINE_SMP_H_ -#ifdef _KERNEL - -#if defined(SMP) && defined(I386_CPU) && !defined(COMPILING_LINT) -#error SMP not supported with I386_CPU -#endif -#if defined(SMP) && !defined(APIC_IO) -# error APIC_IO required for SMP, add "options APIC_IO" to your config file. -#endif /* SMP && !APIC_IO */ -#if defined(SMP) && defined(CPU_DISABLE_CMPXCHG) && !defined(COMPILING_LINT) -#error SMP not supported with CPU_DISABLE_CMPXCHG -#endif - -#if defined(SMP) || defined(APIC_IO) - -#ifndef LOCORE - -/* - * For sending values to POST displays. - * XXX FIXME: where does this really belong, isa.h/isa.c perhaps? - */ -extern int current_postcode; /** XXX currently in mp_machdep.c */ -#define POSTCODE(X) current_postcode = (X), \ - outb(0x80, current_postcode) -#define POSTCODE_LO(X) current_postcode &= 0xf0, \ - current_postcode |= ((X) & 0x0f), \ - outb(0x80, current_postcode) -#define POSTCODE_HI(X) current_postcode &= 0x0f, \ - current_postcode |= (((X) << 4) & 0xf0), \ - outb(0x80, current_postcode) - - -#include <sys/bus.h> /* XXX */ -#include <machine/apic.h> -#include <machine/frame.h> -#include <i386/isa/icu.h> -#include <i386/isa/intr_machdep.h> - -/* - * Interprocessor interrupts for SMP. - */ -#define IPI_INVLTLB XINVLTLB_OFFSET -#define IPI_INVLPG XINVLPG_OFFSET -#define IPI_INVLRNG XINVLRNG_OFFSET -#define IPI_LAZYPMAP XLAZYPMAP_OFFSET -#define IPI_RENDEZVOUS XRENDEZVOUS_OFFSET -#define IPI_AST XCPUAST_OFFSET -#define IPI_STOP XCPUSTOP_OFFSET -#define IPI_HARDCLOCK XHARDCLOCK_OFFSET -#define IPI_STATCLOCK XSTATCLOCK_OFFSET - -/* global data in mpboot.s */ -extern int bootMP_size; - -/* functions in mpboot.s */ -void bootMP(void); - -/* global data in mp_machdep.c */ -extern int bsp_apic_ready; -extern int mp_naps; -extern int mp_nbusses; -extern int mp_napics; -extern int mp_picmode; -extern int boot_cpu_id; -extern vm_offset_t cpu_apic_address; -extern vm_offset_t io_apic_address[]; -extern u_int32_t cpu_apic_versions[]; -extern u_int32_t *io_apic_versions; -extern int cpu_num_to_apic_id[]; -extern int io_num_to_apic_id[]; -extern int apic_id_to_logical[]; -#define APIC_INTMAPSIZE 32 -struct apic_intmapinfo { - int ioapic; - int int_pin; - volatile void *apic_address; - int redirindex; -}; -extern struct apic_intmapinfo int_to_apicintpin[]; -extern struct pcb stoppcbs[]; - -/* functions in mp_machdep.c */ -void i386_mp_probe(void); -u_int mp_bootaddress(u_int); -u_int isa_apic_mask(u_int); -int isa_apic_irq(int); -int pci_apic_irq(int, int, int); -int apic_irq(int, int); -int next_apic_irq(int); -int undirect_isa_irq(int); -int undirect_pci_irq(int); -int apic_bus_type(int); -int apic_src_bus_id(int, int); -int apic_src_bus_irq(int, int); -int apic_int_type(int, int); -int apic_trigger(int, int); -int apic_polarity(int, int); -int mp_grab_cpu_hlt(void); -void assign_apic_irq(int apic, int intpin, int irq); -void revoke_apic_irq(int irq); -void bsp_apic_configure(void); -void init_secondary(void); -void forward_statclock(void); -void forwarded_statclock(struct clockframe frame); -void forward_hardclock(void); -void forwarded_hardclock(struct clockframe frame); -void ipi_selected(u_int cpus, u_int ipi); -void ipi_all(u_int ipi); -void ipi_all_but_self(u_int ipi); -void ipi_self(u_int ipi); -#ifdef APIC_INTR_REORDER -void set_lapic_isrloc(int, int); -#endif /* APIC_INTR_REORDER */ -void smp_invlpg(vm_offset_t addr); -void smp_masked_invlpg(u_int mask, vm_offset_t addr); -void smp_invlpg_range(vm_offset_t startva, vm_offset_t endva); -void smp_masked_invlpg_range(u_int mask, vm_offset_t startva, - vm_offset_t endva); -void smp_invltlb(void); -void smp_masked_invltlb(u_int mask); - -/* global data in mpapic.c */ -extern volatile lapic_t lapic; -extern volatile ioapic_t **ioapic; - -/* functions in mpapic.c */ -void apic_dump(char*); -void apic_initialize(void); -void imen_dump(void); -int apic_ipi(int, int, int); -int selected_apic_ipi(u_int, int, int); -int io_apic_setup(int); -void io_apic_setup_intpin(int, int); -void io_apic_set_id(int, int); -int io_apic_get_id(int); -int ext_int_setup(int, int); - -void set_apic_timer(int); -int read_apic_timer(void); -void u_sleep(int); -u_int io_apic_read(int, int); -void io_apic_write(int, int, u_int); - -#endif /* !LOCORE */ -#endif /* SMP && !APIC_IO */ - -#endif /* _KERNEL */ #endif /* _MACHINE_SMP_H_ */ diff --git a/sys/amd64/include/specialreg.h b/sys/amd64/include/specialreg.h index 1c6ebe1..d66238c 100644 --- a/sys/amd64/include/specialreg.h +++ b/sys/amd64/include/specialreg.h @@ -75,6 +75,14 @@ #define CR4_XMM 0x00000400 /* enable SIMD/MMX2 to use except 16 */ /* + * Bits in AMD64 special registers. EFER is 64 bits wide. + */ +#define EFER_SCE 0x000000001 /* System Call Extensions (R/W) */ +#define EFER_LME 0x000000100 /* Long mode enable (R/W) */ +#define EFER_LMA 0x000000400 /* Long mode active (R) */ +#define EFER_NXE 0x000000800 /* PTE No-Execute bit enable (R/W) */ + +/* * CPUID instruction features register */ #define CPUID_FPU 0x00000001 @@ -132,6 +140,9 @@ #define MSR_PERFCTR0 0x0c1 #define MSR_PERFCTR1 0x0c2 #define MSR_MTRRcap 0x0fe +#define MSR_SYSENTER_CS 0x174 +#define MSR_SYSENTER_ESP 0x175 +#define MSR_SYSENTER_EIP 0x176 #define MSR_MCG_CAP 0x179 #define MSR_MCG_STATUS 0x17a #define MSR_MCG_CTL 0x17b @@ -147,6 +158,7 @@ #define MSR_MTRR64kBase 0x250 #define MSR_MTRR16kBase 0x258 #define MSR_MTRR4kBase 0x268 +#define MSR_PAT 0x277 #define MSR_MTRRdefType 0x2ff #define MSR_MC0_CTL 0x400 #define MSR_MC0_STATUS 0x401 @@ -176,69 +188,6 @@ #define MTRR_N16K 16 #define MTRR_N4K 64 -/* - * Cyrix configuration registers, accessible as IO ports. - */ -#define CCR0 0xc0 /* Configuration control register 0 */ -#define CCR0_NC0 0x01 /* First 64K of each 1M memory region is - non-cacheable */ -#define CCR0_NC1 0x02 /* 640K-1M region is non-cacheable */ -#define CCR0_A20M 0x04 /* Enables A20M# input pin */ -#define CCR0_KEN 0x08 /* Enables KEN# input pin */ -#define CCR0_FLUSH 0x10 /* Enables FLUSH# input pin */ -#define CCR0_BARB 0x20 /* Flushes internal cache when entering hold - state */ -#define CCR0_CO 0x40 /* Cache org: 1=direct mapped, 0=2x set - assoc */ -#define CCR0_SUSPEND 0x80 /* Enables SUSP# and SUSPA# pins */ - -#define CCR1 0xc1 /* Configuration control register 1 */ -#define CCR1_RPL 0x01 /* Enables RPLSET and RPLVAL# pins */ -#define CCR1_SMI 0x02 /* Enables SMM pins */ -#define CCR1_SMAC 0x04 /* System management memory access */ -#define CCR1_MMAC 0x08 /* Main memory access */ -#define CCR1_NO_LOCK 0x10 /* Negate LOCK# */ -#define CCR1_SM3 0x80 /* SMM address space address region 3 */ - -#define CCR2 0xc2 -#define CCR2_WB 0x02 /* Enables WB cache interface pins */ -#define CCR2_SADS 0x02 /* Slow ADS */ -#define CCR2_LOCK_NW 0x04 /* LOCK NW Bit */ -#define CCR2_SUSP_HLT 0x08 /* Suspend on HALT */ -#define CCR2_WT1 0x10 /* WT region 1 */ -#define CCR2_WPR1 0x10 /* Write-protect region 1 */ -#define CCR2_BARB 0x20 /* Flushes write-back cache when entering - hold state. */ -#define CCR2_BWRT 0x40 /* Enables burst write cycles */ -#define CCR2_USE_SUSP 0x80 /* Enables suspend pins */ - -#define CCR3 0xc3 -#define CCR3_SMILOCK 0x01 /* SMM register lock */ -#define CCR3_NMI 0x02 /* Enables NMI during SMM */ -#define CCR3_LINBRST 0x04 /* Linear address burst cycles */ -#define CCR3_SMMMODE 0x08 /* SMM Mode */ -#define CCR3_MAPEN0 0x10 /* Enables Map0 */ -#define CCR3_MAPEN1 0x20 /* Enables Map1 */ -#define CCR3_MAPEN2 0x40 /* Enables Map2 */ -#define CCR3_MAPEN3 0x80 /* Enables Map3 */ - -#define CCR4 0xe8 -#define CCR4_IOMASK 0x07 -#define CCR4_MEM 0x08 /* Enables momory bypassing */ -#define CCR4_DTE 0x10 /* Enables directory table entry cache */ -#define CCR4_FASTFPE 0x20 /* Fast FPU exception */ -#define CCR4_CPUID 0x80 /* Enables CPUID instruction */ - -#define CCR5 0xe9 -#define CCR5_WT_ALLOC 0x01 /* Write-through allocate */ -#define CCR5_SLOP 0x02 /* LOOP instruction slowed down */ -#define CCR5_LBR1 0x10 /* Local bus region 1 */ -#define CCR5_ARREN 0x20 /* Enables ARR region */ - -#define CCR6 0xea - -#define CCR7 0xeb - /* Performance Control Register (5x86 only). */ #define PCR0 0x20 #define PCR0_RSTK 0x01 /* Enables return stack */ @@ -347,21 +296,31 @@ #define AMD_WT_ALLOC_PRE 0x20000 /* programmable range enable */ #define AMD_WT_ALLOC_FRE 0x10000 /* fixed (A0000-FFFFF) range enable */ - -#ifndef LOCORE -static __inline u_char -read_cyrix_reg(u_char reg) -{ - outb(0x22, reg); - return inb(0x23); -} - -static __inline void -write_cyrix_reg(u_char reg, u_char data) -{ - outb(0x22, reg); - outb(0x23, data); -} -#endif +/* X86-64 MSR's */ +#define MSR_EFER 0xc0000080 /* extended features */ +#define MSR_STAR 0xc0000081 /* legacy mode SYSCALL target/cs/ss */ +#define MSR_LSTAR 0xc0000082 /* long mode SYSCALL target rip */ +#define MSR_CSTAR 0xc0000083 /* compat mode SYSCALL target rip */ +#define MSR_SF_MASK 0xc0000084 /* syscall flags mask */ +#define MSR_FSBASE 0xc0000100 /* base address of the %fs "segment" */ +#define MSR_GSBASE 0xc0000101 /* base address of the %gs "segment" */ +#define MSR_KGSBASE 0xc0000102 /* base address of the kernel %gs */ +#define MSR_PERFEVSEL0 0xc0010000 +#define MSR_PERFEVSEL1 0xc0010001 +#define MSR_PERFEVSEL2 0xc0010002 +#define MSR_PERFEVSEL3 0xc0010003 +#undef MSR_PERFCTR0 +#undef MSR_PERFCTR1 +#define MSR_PERFCTR0 0xc0010004 +#define MSR_PERFCTR1 0xc0010005 +#define MSR_PERFCTR2 0xc0010006 +#define MSR_PERFCTR3 0xc0010007 +#define MSR_SYSCFG 0xc0010010 +#define MSR_IORRBASE0 0xc0010016 +#define MSR_IORRMASK0 0xc0010017 +#define MSR_IORRBASE1 0xc0010018 +#define MSR_IORRMASK1 0xc0010019 +#define MSR_TOP_MEM 0xc001001a /* boundary for ram below 4G */ +#define MSR_TOP_MEM2 0xc001001d /* boundary for ram above 4G */ #endif /* !_MACHINE_SPECIALREG_H_ */ diff --git a/sys/amd64/include/sysarch.h b/sys/amd64/include/sysarch.h index c5574ff..c33f7b8 100644 --- a/sys/amd64/include/sysarch.h +++ b/sys/amd64/include/sysarch.h @@ -39,46 +39,4 @@ #ifndef _MACHINE_SYSARCH_H_ #define _MACHINE_SYSARCH_H_ -#define I386_GET_LDT 0 -#define I386_SET_LDT 1 - /* I386_IOPL */ -#define I386_GET_IOPERM 3 -#define I386_SET_IOPERM 4 - /* xxxxx */ -#define I386_VM86 6 - -struct i386_ldt_args { - unsigned int start; - union descriptor *descs; - unsigned int num; -}; - -struct i386_ioperm_args { - unsigned int start; - unsigned int length; - int enable; -}; - -struct i386_vm86_args { - int sub_op; /* sub-operation to perform */ - char *sub_args; /* args */ -}; - -#ifndef _KERNEL -#include <sys/cdefs.h> - -union descriptor; -struct dbreg; - -__BEGIN_DECLS -int i386_get_ldt(int, union descriptor *, int); -int i386_set_ldt(int, union descriptor *, int); -int i386_get_ioperm(unsigned int, unsigned int *, int *); -int i386_set_ioperm(unsigned int, unsigned int, int); -int i386_vm86(int, void *); -int i386_set_watch(int, unsigned int, int, int, struct dbreg *); -int i386_clr_watch(int, struct dbreg *); -__END_DECLS -#endif - #endif /* !_MACHINE_SYSARCH_H_ */ diff --git a/sys/amd64/include/tss.h b/sys/amd64/include/tss.h index fb2e7a5..aa60ba0 100644 --- a/sys/amd64/include/tss.h +++ b/sys/amd64/include/tss.h @@ -41,37 +41,35 @@ #define _MACHINE_TSS_H_ 1 /* - * Intel 386 Context Data Type + * amd64 Context Data Type + * + * The alignment is pretty messed up here due to reuse of the original 32 bit + * fields. It might be worth trying to set the tss on a +4 byte offset to + * make the 64 bit fields aligned in practice. */ - -struct i386tss { - int tss_link; /* actually 16 bits: top 16 bits must be zero */ - int tss_esp0; /* kernel stack pointer privilege level 0 */ - int tss_ss0; /* actually 16 bits: top 16 bits must be zero */ - int tss_esp1; /* kernel stack pointer privilege level 1 */ - int tss_ss1; /* actually 16 bits: top 16 bits must be zero */ - int tss_esp2; /* kernel stack pointer privilege level 2 */ - int tss_ss2; /* actually 16 bits: top 16 bits must be zero */ - int tss_cr3; /* page table directory */ - int tss_eip; /* program counter */ - int tss_eflags; /* program status longword */ - int tss_eax; - int tss_ecx; - int tss_edx; - int tss_ebx; - int tss_esp; /* user stack pointer */ - int tss_ebp; /* user frame pointer */ - int tss_esi; - int tss_edi; - int tss_es; /* actually 16 bits: top 16 bits must be zero */ - int tss_cs; /* actually 16 bits: top 16 bits must be zero */ - int tss_ss; /* actually 16 bits: top 16 bits must be zero */ - int tss_ds; /* actually 16 bits: top 16 bits must be zero */ - int tss_fs; /* actually 16 bits: top 16 bits must be zero */ - int tss_gs; /* actually 16 bits: top 16 bits must be zero */ - int tss_ldt; /* actually 16 bits: top 16 bits must be zero */ - int tss_ioopt; /* options & io offset bitmap: currently zero */ - /* XXX unimplemented .. i/o permission bitmap */ +struct amd64tss { + u_int32_t tss_rsvd0; + u_int64_t tss_rsp0 __packed; /* kernel stack pointer ring 0 */ + u_int64_t tss_rsp1 __packed; /* kernel stack pointer ring 1 */ + u_int64_t tss_rsp2 __packed; /* kernel stack pointer ring 2 */ + u_int32_t tss_rsvd1; + u_int32_t tss_rsvd2; + u_int32_t tss_rsvd3; + u_int64_t tss_ist1 __packed; /* Interrupt stack table 1 */ + u_int64_t tss_ist2 __packed; /* Interrupt stack table 2 */ + u_int64_t tss_ist3 __packed; /* Interrupt stack table 3 */ + u_int64_t tss_ist4 __packed; /* Interrupt stack table 4 */ + u_int64_t tss_ist5 __packed; /* Interrupt stack table 5 */ + u_int64_t tss_ist6 __packed; /* Interrupt stack table 6 */ + u_int64_t tss_ist7 __packed; /* Interrupt stack table 7 */ + u_int32_t tss_rsvd4; + u_int32_t tss_rsvd5; + u_int16_t tss_rsvd6; + u_int16_t tss_iobase; /* io bitmap offset */ }; +#ifdef _KERNEL +extern struct amd64tss common_tss; +#endif + #endif /* _MACHINE_TSS_H_ */ diff --git a/sys/amd64/include/ucontext.h b/sys/amd64/include/ucontext.h index ec3e0ee..1938707 100644 --- a/sys/amd64/include/ucontext.h +++ b/sys/amd64/include/ucontext.h @@ -37,69 +37,44 @@ typedef struct __mcontext { * sigcontext. So that we can support sigcontext * and ucontext_t at the same time. */ - int mc_onstack; /* XXX - sigcontext compat. */ - int mc_gs; /* machine state (struct trapframe) */ - int mc_fs; - int mc_es; - int mc_ds; - int mc_edi; - int mc_esi; - int mc_ebp; - int mc_isp; - int mc_ebx; - int mc_edx; - int mc_ecx; - int mc_eax; - int mc_trapno; - int mc_err; - int mc_eip; - int mc_cs; - int mc_eflags; - int mc_esp; - int mc_ss; + register_t mc_onstack; /* XXX - sigcontext compat. */ + register_t mc_r15; /* machine state (struct trapframe) */ + register_t mc_r14; + register_t mc_r13; + register_t mc_r12; + register_t mc_r11; + register_t mc_r10; + register_t mc_r9; + register_t mc_r8; + register_t mc_rdi; + register_t mc_rsi; + register_t mc_rbp; + register_t mc_rbx; + register_t mc_rdx; + register_t mc_rcx; + register_t mc_rax; + register_t mc_trapno; + register_t mc_err; + register_t mc_rip; + register_t mc_cs; + register_t mc_rflags; + register_t mc_rsp; + register_t mc_ss; - int mc_len; /* sizeof(mcontext_t) */ + long mc_len; /* sizeof(mcontext_t) */ #define _MC_FPFMT_NODEV 0x10000 /* device not present or configured */ -#define _MC_FPFMT_387 0x10001 #define _MC_FPFMT_XMM 0x10002 - int mc_fpformat; + long mc_fpformat; #define _MC_FPOWNED_NONE 0x20000 /* FP state not used */ #define _MC_FPOWNED_FPU 0x20001 /* FP state came from FPU */ #define _MC_FPOWNED_PCB 0x20002 /* FP state came from PCB */ - int mc_ownedfp; - int mc_spare1[1]; /* align next field to 16 bytes */ + long mc_ownedfp; + long mc_spare1[1]; /* align next field to 16 bytes */ /* * See <machine/npx.h> for the internals of mc_fpstate[]. */ - int mc_fpstate[128] __aligned(16); - int mc_spare2[8]; + long mc_fpstate[128] __aligned(16); + long mc_spare2[8]; } mcontext_t; -#if defined(_KERNEL) && defined(COMPAT_FREEBSD4) -struct mcontext4 { - int mc_onstack; /* XXX - sigcontext compat. */ - int mc_gs; /* machine state (struct trapframe) */ - int mc_fs; - int mc_es; - int mc_ds; - int mc_edi; - int mc_esi; - int mc_ebp; - int mc_isp; - int mc_ebx; - int mc_edx; - int mc_ecx; - int mc_eax; - int mc_trapno; - int mc_err; - int mc_eip; - int mc_cs; - int mc_eflags; - int mc_esp; /* machine state */ - int mc_ss; - int mc_fpregs[28]; /* env87 + fpacc87 + u_long */ - int __spare__[17]; -}; -#endif - #endif /* !_MACHINE_UCONTEXT_H_ */ diff --git a/sys/amd64/include/vmparam.h b/sys/amd64/include/vmparam.h index bb7c7e1..0e9697d 100644 --- a/sys/amd64/include/vmparam.h +++ b/sys/amd64/include/vmparam.h @@ -44,11 +44,9 @@ #define _MACHINE_VMPARAM_H_ 1 /* - * Machine dependent constants for 386. + * Machine dependent constants for AMD64. */ -#define VM_PROT_READ_IS_EXEC /* if you can read -- then you can exec */ - /* * Virtual memory related constants, all in bytes */ @@ -69,8 +67,6 @@ #define SGROWSIZ (128UL*1024) /* amount to grow stack */ #endif -#define USRTEXT (1*PAGE_SIZE) /* base of user text XXX bogus */ - /* * The time for a process to be blocked before being very swappable. * This is a number of seconds which the system takes as being a non-trivial @@ -90,23 +86,20 @@ * messy at times, but hey, we'll do anything to save a page :-) */ -#define VM_MAX_KERNEL_ADDRESS VADDR(KPTDI+NKPDE-1, NPTEPG-1) -#define VM_MIN_KERNEL_ADDRESS VADDR(PTDPTDI, PTDPTDI) - -#define KERNBASE VADDR(KPTDI, 0) +#define VM_MAX_KERNEL_ADDRESS VADDR(0, 0, KPTDI+NKPDE-1, NPTEPG-1) +#define VM_MIN_KERNEL_ADDRESS VADDR(0, 0, PTDPTDI, PTDPTDI) -#define KPT_MAX_ADDRESS VADDR(PTDPTDI, KPTDI+NKPT) -#define KPT_MIN_ADDRESS VADDR(PTDPTDI, KPTDI) +#define KERNBASE VADDR(0, 0, KPTDI, 0) -#define UPT_MAX_ADDRESS VADDR(PTDPTDI, PTDPTDI) -#define UPT_MIN_ADDRESS VADDR(PTDPTDI, 0) +#define UPT_MAX_ADDRESS VADDR(0, 0, PTDPTDI, PTDPTDI) +#define UPT_MIN_ADDRESS VADDR(0, 0, PTDPTDI, 0) -#define VM_MAXUSER_ADDRESS VADDR(PTDPTDI, 0) +#define VM_MAXUSER_ADDRESS UPT_MIN_ADDRESS #define USRSTACK VM_MAXUSER_ADDRESS -#define VM_MAX_ADDRESS VADDR(PTDPTDI, PTDPTDI) -#define VM_MIN_ADDRESS ((vm_offset_t)0) +#define VM_MAX_ADDRESS UPT_MAX_ADDRESS +#define VM_MIN_ADDRESS (0) /* virtual sizes (bytes) for various kernel submaps */ #ifndef VM_KMEM_SIZE |
