diff options
| author | luoqi <luoqi@FreeBSD.org> | 1999-04-28 01:04:33 +0000 |
|---|---|---|
| committer | luoqi <luoqi@FreeBSD.org> | 1999-04-28 01:04:33 +0000 |
| commit | af7e9be5cce9a2ceb819f00b3f58014d23ab57cd (patch) | |
| tree | 0e70c0e764f072d43041acca8ac52e30616dced3 /sys/amd64 | |
| parent | 0f4a245030e7f1acb03f8de0822a58b188fc0d28 (diff) | |
| download | FreeBSD-src-af7e9be5cce9a2ceb819f00b3f58014d23ab57cd.zip FreeBSD-src-af7e9be5cce9a2ceb819f00b3f58014d23ab57cd.tar.gz | |
Enable vmspace sharing on SMP. Major changes are,
- %fs register is added to trapframe and saved/restored upon kernel entry/exit.
- Per-cpu pages are no longer mapped at the same virtual address.
- Each cpu now has a separate gdt selector table. A new segment selector
is added to point to per-cpu pages, per-cpu global variables are now
accessed through this new selector (%fs). The selectors in gdt table are
rearranged for cache line optimization.
- fask_vfork is now on as default for both UP and SMP.
- Some aio code cleanup.
Reviewed by: Alan Cox <alc@cs.rice.edu>
John Dyson <dyson@iquest.net>
Julian Elischer <julian@whistel.com>
Bruce Evans <bde@zeta.org.au>
David Greenman <dg@root.com>
Diffstat (limited to 'sys/amd64')
35 files changed, 737 insertions, 850 deletions
diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S index 822375d..88340e6 100644 --- a/sys/amd64/amd64/apic_vector.S +++ b/sys/amd64/amd64/apic_vector.S @@ -1,6 +1,6 @@ /* * from: vector.s, 386BSD 0.1 unknown origin - * $Id: apic_vector.s,v 1.35 1999/04/10 19:19:02 tegge Exp $ + * $Id: apic_vector.s,v 1.36 1999/04/14 14:26:35 bde Exp $ */ @@ -58,10 +58,13 @@ IDTVEC(vec_name) ; \ pushl %edx ; \ pushl %ds ; \ MAYBE_PUSHL_ES ; \ + pushl %fs ; \ movl $KDSEL,%eax ; \ movl %ax,%ds ; \ MAYBE_MOVW_AX_ES ; \ - FAKE_MCOUNT((4+ACTUALLY_PUSHED)*4(%esp)) ; \ + movl $KPSEL,%eax ; \ + movl %ax,%fs ; \ + FAKE_MCOUNT((5+ACTUALLY_PUSHED)*4(%esp)) ; \ pushl _intr_unit + (irq_num) * 4 ; \ GET_FAST_INTR_LOCK ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ @@ -74,6 +77,7 @@ IDTVEC(vec_name) ; \ lock ; \ incl (%eax) ; \ MEXITCOUNT ; \ + popl %fs ; \ MAYBE_POPL_ES ; \ popl %ds ; \ popl %edx ; \ @@ -92,10 +96,13 @@ IDTVEC(vec_name) ; \ pushl %edx ; \ pushl %ds ; \ MAYBE_PUSHL_ES ; \ + pushl %fs ; \ movl $KDSEL, %eax ; \ movl %ax, %ds ; \ MAYBE_MOVW_AX_ES ; \ - FAKE_MCOUNT((4+ACTUALLY_PUSHED)*4(%esp)) ; \ + movl $KPSEL, %eax ; \ + movl %ax, %fs ; \ + FAKE_MCOUNT((5+ACTUALLY_PUSHED)*4(%esp)) ; \ GET_FAST_INTR_LOCK ; \ pushl _intr_unit + (irq_num) * 4 ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ @@ -113,6 +120,7 @@ IDTVEC(vec_name) ; \ 1: ; \ MEXITCOUNT ; \ REL_FAST_INTR_LOCK ; \ + popl %fs ; \ MAYBE_POPL_ES ; \ popl %ds ; \ popl %edx ; \ @@ -130,6 +138,7 @@ IDTVEC(vec_name) ; \ lock ; \ incb _intr_nesting_level ; /* ... really limit it ... */ \ sti ; /* to do this as early as possible */ \ + popl %fs ; /* discard most of thin frame ... */ \ MAYBE_POPL_ES ; /* discard most of thin frame ... */ \ popl %ecx ; /* ... original %ds ... */ \ popl %edx ; \ @@ -137,11 +146,14 @@ IDTVEC(vec_name) ; \ pushal ; /* build fat frame (grrr) ... */ \ pushl %ecx ; /* ... actually %ds ... */ \ pushl %es ; \ + pushl %fs ; movl $KDSEL, %eax ; \ movl %ax, %es ; \ - movl (2+8+0)*4(%esp), %ecx ; /* %ecx from thin frame ... */ \ - movl %ecx, (2+6)*4(%esp) ; /* ... to fat frame ... */ \ - movl (2+8+1)*4(%esp), %eax ; /* ... cpl from thin frame */ \ + movl $KPSEL, %eax ; + movl %ax, %fs ; + movl (3+8+0)*4(%esp), %ecx ; /* %ecx from thin frame ... */ \ + movl %ecx, (3+6)*4(%esp) ; /* ... to fat frame ... */ \ + movl (3+8+1)*4(%esp), %eax ; /* ... cpl from thin frame */ \ pushl %eax ; \ subl $4, %esp ; /* junk for unit number */ \ MEXITCOUNT ; \ @@ -158,9 +170,11 @@ IDTVEC(vec_name) ; \ pushl $0 ; /* dummy trap type */ \ pushal ; \ pushl %ds ; /* save data and extra segments ... */ \ - pushl %es + pushl %es ; \ + pushl %fs #define POP_FRAME \ + popl %fs ; \ popl %es ; \ popl %ds ; \ popal ; \ @@ -319,6 +333,8 @@ IDTVEC(vec_name) ; \ movl $KDSEL, %eax ; /* reload with kernel's data segment */ \ movl %ax, %ds ; \ movl %ax, %es ; \ + movl $KPSEL, %eax ; \ + movl %ax, %fs ; \ ; \ APIC_ITRACE(apic_itrace_enter, irq_num, APIC_ITRACE_ENTER) ; \ lock ; /* MP-safe */ \ @@ -344,7 +360,7 @@ IDTVEC(vec_name) ; \ ; \ /* entry point used by doreti_unpend for HWIs. */ \ __CONCAT(Xresume,irq_num): ; \ - FAKE_MCOUNT(12*4(%esp)) ; /* XXX avoid dbl cnt */ \ + FAKE_MCOUNT(13*4(%esp)) ; /* XXX avoid dbl cnt */ \ lock ; incl _cnt+V_INTR ; /* tally interrupts */ \ movl _intr_countp + (irq_num) * 4, %eax ; \ lock ; incl (%eax) ; \ @@ -429,6 +445,8 @@ IDTVEC(vec_name) ; \ movl $KDSEL, %eax ; /* reload with kernel's data segment */ \ movl %ax, %ds ; \ movl %ax, %es ; \ + movl $KPSEL, %eax ; \ + movl %ax, %fs ; \ ; \ APIC_ITRACE(apic_itrace_enter, irq_num, APIC_ITRACE_ENTER) ; \ lock ; /* MP-safe */ \ @@ -453,7 +471,7 @@ IDTVEC(vec_name) ; \ ; \ /* entry point used by doreti_unpend for HWIs. */ \ __CONCAT(Xresume,irq_num): ; \ - FAKE_MCOUNT(12*4(%esp)) ; /* XXX avoid dbl cnt */ \ + FAKE_MCOUNT(13*4(%esp)) ; /* XXX avoid dbl cnt */ \ lock ; incl _cnt+V_INTR ; /* tally interrupts */ \ movl _intr_countp + (irq_num) * 4, %eax ; \ lock ; incl (%eax) ; \ @@ -549,8 +567,11 @@ _Xinvltlb: pushl %eax #ifdef COUNT_XINVLTLB_HITS - ss + pushl %fs + movl $KPSEL, %eax + movl %ax, %fs movl _cpuid, %eax + popl %fs ss incl _xhits(,%eax,4) #endif /* COUNT_XINVLTLB_HITS */ @@ -576,7 +597,7 @@ _Xinvltlb: * * - Signals its receipt by setting bit cpuid in checkstate_probed_cpus. * - * stack: 0 -> ds, 4 -> ebx, 8 -> eax, 12 -> eip, 16 -> cs, 20 -> eflags + * stack: 0->ds, 4->fs, 8->ebx, 12->eax, 16->eip, 20->cs, 24->eflags */ .text @@ -589,19 +610,22 @@ _Xcpucheckstate: pushl %eax pushl %ebx pushl %ds /* save current data segment */ + pushl %fs movl $KDSEL, %eax movl %ax, %ds /* use KERNEL data segment */ + movl $KPSEL, %eax + movl %ax, %fs movl $0, lapic_eoi /* End Of Interrupt to APIC */ movl $0, %ebx - movl 16(%esp), %eax + movl 20(%esp), %eax andl $3, %eax cmpl $3, %eax je 1f #ifdef VM86 - testl $PSL_VM, 20(%esp) + testl $PSL_VM, 24(%esp) jne 1f #endif incl %ebx /* system or interrupt */ @@ -615,12 +639,13 @@ _Xcpucheckstate: movl %ebx, _checkstate_cpustate(,%eax,4) movl _curproc, %ebx movl %ebx, _checkstate_curproc(,%eax,4) - movl 12(%esp), %ebx + movl 16(%esp), %ebx movl %ebx, _checkstate_pc(,%eax,4) lock /* checkstate_probed_cpus |= (1<<id) */ btsl %eax, _checkstate_probed_cpus + popl %fs popl %ds /* restore previous data segment */ popl %ebx popl %eax @@ -644,6 +669,8 @@ _Xcpuast: movl $KDSEL, %eax movl %ax, %ds /* use KERNEL data segment */ movl %ax, %es + movl $KPSEL, %eax + movl %ax, %fs movl _cpuid, %eax lock /* checkstate_need_ast &= ~(1<<id) */ @@ -654,7 +681,7 @@ _Xcpuast: btsl %eax, _checkstate_pending_ast jc 1f - FAKE_MCOUNT(12*4(%esp)) + FAKE_MCOUNT(13*4(%esp)) /* * Giant locks do not come cheap. @@ -709,10 +736,12 @@ _Xforward_irq: movl $KDSEL, %eax movl %ax, %ds /* use KERNEL data segment */ movl %ax, %es + movl $KPSEL, %eax + movl %ax, %fs movl $0, lapic_eoi /* End Of Interrupt to APIC */ - FAKE_MCOUNT(12*4(%esp)) + FAKE_MCOUNT(13*4(%esp)) ISR_TRYLOCK testl %eax,%eax /* Did we get the lock ? */ @@ -812,10 +841,12 @@ _Xcpustop: pushl %ecx pushl %edx pushl %ds /* save current data segment */ - pushl %es + pushl %fs movl $KDSEL, %eax movl %ax, %ds /* use KERNEL data segment */ + movl $KPSEL, %eax + movl %ax, %fs movl $0, lapic_eoi /* End Of Interrupt to APIC */ @@ -850,7 +881,7 @@ _Xcpustop: call %eax 2: - popl %es + popl %fs popl %ds /* restore previous data segment */ popl %edx popl %ecx diff --git a/sys/amd64/amd64/cpu_switch.S b/sys/amd64/amd64/cpu_switch.S index 7e97a09..ae4ca62 100644 --- a/sys/amd64/amd64/cpu_switch.S +++ b/sys/amd64/amd64/cpu_switch.S @@ -33,7 +33,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: swtch.s,v 1.77 1999/03/20 18:44:13 alc Exp $ + * $Id: swtch.s,v 1.78 1999/04/02 17:59:39 alc Exp $ */ #include "npx.h" @@ -258,25 +258,32 @@ _idle: /* when called, we have the mplock, intr disabled */ /* use our idleproc's "context" */ - movl _my_idlePTD,%ecx - movl %ecx,%cr3 + movl _IdlePTD, %ecx + movl %cr3, %eax + cmpl %ecx, %eax + je 2f #if defined(SWTCH_OPTIM_STATS) + decl _swtch_optim_stats incl _tlb_flush_count #endif + movl %ecx, %cr3 +2: /* Keep space for nonexisting return addr, or profiling bombs */ - movl $_idlestack_top-4,%ecx - movl %ecx,%esp + movl $gd_idlestack_top-4, %ecx + addl %fs:0, %ecx + movl %ecx, %esp /* update common_tss.tss_esp0 pointer */ -#ifdef VM86 - movl _my_tr, %esi -#endif /* VM86 */ movl %ecx, _common_tss + TSS_ESP0 #ifdef VM86 - cmpl $0, _private_tss - je 1f - movl $_common_tssd, %edi + movl _cpuid, %esi + btrl %esi, _private_tss + jae 1f + + movl $GPROC0_SEL, %esi + movl $gd_common_tssd, %edi + addl %fs:0, %edi /* move correct tss descriptor into GDT slot, then reload tr */ leal _gdt(,%esi,8), %ebx /* entry in GDT */ @@ -388,14 +395,14 @@ idle_loop: #endif /* update common_tss.tss_esp0 pointer */ -#ifdef VM86 - movl _my_tr, %esi -#endif /* VM86 */ movl %esp, _common_tss + TSS_ESP0 #ifdef VM86 - cmpl $0, _private_tss - je 1f + movl $0, %esi + btrl %esi, _private_tss + jae 1f + + movl $GPROC0_SEL, %esi movl $_common_tssd, %edi /* move correct tss descriptor into GDT slot, then reload tr */ @@ -477,7 +484,6 @@ ENTRY(cpu_switch) movl %ebp,PCB_EBP(%edx) movl %esi,PCB_ESI(%edx) movl %edi,PCB_EDI(%edx) - movl %fs,PCB_FS(%edx) movl %gs,PCB_GS(%edx) #ifdef SMP @@ -610,70 +616,55 @@ swtch_com: movl %eax,P_BACK(%ecx) /* isolate process to run */ movl P_ADDR(%ecx),%edx -#ifdef SMP - movl PCB_CR3(%edx),%ebx - /* Grab the private PT pointer from the outgoing process's PTD */ - movl $_PTD, %esi - movl 4*MPPTDI(%esi), %eax /* fetch cpu's prv pt */ -#else #if defined(SWTCH_OPTIM_STATS) incl _swtch_optim_stats #endif /* switch address space */ movl %cr3,%ebx cmpl PCB_CR3(%edx),%ebx - je 4f + je 4f #if defined(SWTCH_OPTIM_STATS) decl _swtch_optim_stats incl _tlb_flush_count #endif movl PCB_CR3(%edx),%ebx -#endif /* SMP */ movl %ebx,%cr3 4: +#ifdef VM86 #ifdef SMP - /* Copy the private PT to the new process's PTD */ - /* XXX yuck, the _PTD changes when we switch, so we have to - * reload %cr3 after changing the address space. - * We need to fix this by storing a pointer to the virtual - * location of the per-process PTD in the PCB or something quick. - * Dereferencing proc->vm_map->pmap->p_pdir[] is painful in asm. - */ - movl %eax, 4*MPPTDI(%esi) /* restore cpu's prv page */ - -#if defined(SWTCH_OPTIM_STATS) - incl _tlb_flush_count + movl _cpuid, %esi +#else + xorl %esi, %esi #endif - /* XXX: we have just changed the page tables.. reload.. */ - movl %ebx, %cr3 -#endif /* SMP */ - -#ifdef VM86 - movl _my_tr, %esi cmpl $0, PCB_EXT(%edx) /* has pcb extension? */ je 1f - movl $1, _private_tss /* mark use of private tss */ + btsl %esi, _private_tss /* mark use of private tss */ movl PCB_EXT(%edx), %edi /* new tss descriptor */ jmp 2f 1: #endif /* update common_tss.tss_esp0 pointer */ - movl $_common_tss, %eax movl %edx, %ebx /* pcb */ #ifdef VM86 addl $(UPAGES * PAGE_SIZE - 16), %ebx #else addl $(UPAGES * PAGE_SIZE), %ebx #endif /* VM86 */ - movl %ebx, TSS_ESP0(%eax) + movl %ebx, _common_tss + TSS_ESP0 #ifdef VM86 - cmpl $0, _private_tss - je 3f + btrl %esi, _private_tss + jae 3f +#ifdef SMP + movl $gd_common_tssd, %edi + addl %fs:0, %edi +#else movl $_common_tssd, %edi +#endif 2: + movl $GPROC0_SEL, %esi /* move correct tss descriptor into GDT slot, then reload tr */ leal _gdt(,%esi,8), %ebx /* entry in GDT */ movl 0(%edi), %eax @@ -738,9 +729,6 @@ swtch_com: #endif /* This must be done after loading the user LDT. */ - .globl cpu_switch_load_fs -cpu_switch_load_fs: - movl PCB_FS(%edx),%fs .globl cpu_switch_load_gs cpu_switch_load_gs: movl PCB_GS(%edx),%gs @@ -791,7 +779,6 @@ ENTRY(savectx) movl %ebp,PCB_EBP(%ecx) movl %esi,PCB_ESI(%ecx) movl %edi,PCB_EDI(%ecx) - movl %fs,PCB_FS(%ecx) movl %gs,PCB_GS(%ecx) #if NNPX > 0 diff --git a/sys/amd64/amd64/db_interface.c b/sys/amd64/amd64/db_interface.c index 7f01f1d..2998597 100644 --- a/sys/amd64/amd64/db_interface.c +++ b/sys/amd64/amd64/db_interface.c @@ -23,7 +23,7 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id: db_interface.c,v 1.42 1998/12/14 05:34:33 dillon Exp $ + * $Id: db_interface.c,v 1.43 1998/12/28 23:02:56 msmith Exp $ */ /* @@ -202,6 +202,7 @@ kdb_trap(type, code, regs) regs->tf_esi = ddb_regs.tf_esi; regs->tf_edi = ddb_regs.tf_edi; regs->tf_es = ddb_regs.tf_es & 0xffff; + regs->tf_fs = ddb_regs.tf_fs & 0xffff; regs->tf_cs = ddb_regs.tf_cs & 0xffff; regs->tf_ds = ddb_regs.tf_ds & 0xffff; return (1); diff --git a/sys/amd64/amd64/db_trace.c b/sys/amd64/amd64/db_trace.c index 2ad054f..6a361a0 100644 --- a/sys/amd64/amd64/db_trace.c +++ b/sys/amd64/amd64/db_trace.c @@ -23,7 +23,7 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id: db_trace.c,v 1.32 1999/01/27 23:45:38 dillon Exp $ + * $Id: db_trace.c,v 1.33 1999/01/28 01:59:50 dillon Exp $ */ #include <sys/param.h> @@ -47,8 +47,8 @@ struct db_variable db_regs[] = { { "cs", &ddb_regs.tf_cs, FCN_NULL }, { "ds", &ddb_regs.tf_ds, FCN_NULL }, { "es", &ddb_regs.tf_es, FCN_NULL }, -#if 0 { "fs", &ddb_regs.tf_fs, FCN_NULL }, +#if 0 { "gs", &ddb_regs.tf_gs, FCN_NULL }, #endif { "ss", &ddb_regs.tf_ss, FCN_NULL }, diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S index a9286fa..b87a126 100644 --- a/sys/amd64/amd64/exception.S +++ b/sys/amd64/amd64/exception.S @@ -30,7 +30,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: exception.s,v 1.57 1999/02/28 10:53:28 bde Exp $ + * $Id: exception.s,v 1.58 1999/04/16 21:22:12 peter Exp $ */ #include "npx.h" @@ -59,10 +59,12 @@ #define AVCPL_UNLOCK #endif /* SMP */ -#define KCSEL 0x08 /* kernel code selector */ -#define KDSEL 0x10 /* kernel data selector */ +#ifdef SMP +#define MOVL_KPSEL_EAX movl $KPSEL,%eax +#else +#define MOVL_KPSEL_EAX +#endif #define SEL_RPL_MASK 0x0003 -#define TRAPF_CS_OFF (13 * 4) .text @@ -149,10 +151,13 @@ IDTVEC(fpu) pushal pushl %ds pushl %es /* now stack frame is a trap frame */ + pushl %fs movl $KDSEL,%eax movl %ax,%ds movl %ax,%es - FAKE_MCOUNT(12*4(%esp)) + MOVL_KPSEL_EAX + movl %ax,%fs + FAKE_MCOUNT(13*4(%esp)) #ifdef SMP MPLOCKED incl _cnt+V_TRAP @@ -198,11 +203,14 @@ _alltraps: pushal pushl %ds pushl %es + pushl %fs alltraps_with_regs_pushed: movl $KDSEL,%eax movl %ax,%ds movl %ax,%es - FAKE_MCOUNT(12*4(%esp)) + MOVL_KPSEL_EAX + movl %ax,%fs + FAKE_MCOUNT(13*4(%esp)) calltrap: FAKE_MCOUNT(_btrap) /* init "from" _btrap -> calltrap */ MPLOCKED incl _cnt+V_TRAP @@ -249,13 +257,16 @@ IDTVEC(syscall) pushal pushl %ds pushl %es + pushl %fs movl $KDSEL,%eax /* switch to kernel segments */ movl %ax,%ds movl %ax,%es + MOVL_KPSEL_EAX + movl %ax,%fs movl TF_ERR(%esp),%eax /* copy saved eflags to final spot */ movl %eax,TF_EFLAGS(%esp) movl $7,TF_ERR(%esp) /* sizeof "lcall 7,0" */ - FAKE_MCOUNT(12*4(%esp)) + FAKE_MCOUNT(13*4(%esp)) MPLOCKED incl _cnt+V_SYSCALL SYSCALL_LOCK ECPL_LOCK @@ -285,11 +296,14 @@ IDTVEC(int0x80_syscall) pushal pushl %ds pushl %es + pushl %fs movl $KDSEL,%eax /* switch to kernel segments */ movl %ax,%ds movl %ax,%es + MOVL_KPSEL_EAX + movl %ax,%fs movl $2,TF_ERR(%esp) /* sizeof "int 0x80" */ - FAKE_MCOUNT(12*4(%esp)) + FAKE_MCOUNT(13*4(%esp)) MPLOCKED incl _cnt+V_SYSCALL ALTSYSCALL_LOCK ECPL_LOCK @@ -316,7 +330,9 @@ ENTRY(fork_trampoline) #ifdef SMP cmpl $0,_switchtime jne 1f - pushl $_switchtime + movl $gd_switchtime,%eax + addl %fs:0,%eax + pushl %eax call _microuptime popl %edx movl _ticks,%eax diff --git a/sys/amd64/amd64/exception.s b/sys/amd64/amd64/exception.s index a9286fa..b87a126 100644 --- a/sys/amd64/amd64/exception.s +++ b/sys/amd64/amd64/exception.s @@ -30,7 +30,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: exception.s,v 1.57 1999/02/28 10:53:28 bde Exp $ + * $Id: exception.s,v 1.58 1999/04/16 21:22:12 peter Exp $ */ #include "npx.h" @@ -59,10 +59,12 @@ #define AVCPL_UNLOCK #endif /* SMP */ -#define KCSEL 0x08 /* kernel code selector */ -#define KDSEL 0x10 /* kernel data selector */ +#ifdef SMP +#define MOVL_KPSEL_EAX movl $KPSEL,%eax +#else +#define MOVL_KPSEL_EAX +#endif #define SEL_RPL_MASK 0x0003 -#define TRAPF_CS_OFF (13 * 4) .text @@ -149,10 +151,13 @@ IDTVEC(fpu) pushal pushl %ds pushl %es /* now stack frame is a trap frame */ + pushl %fs movl $KDSEL,%eax movl %ax,%ds movl %ax,%es - FAKE_MCOUNT(12*4(%esp)) + MOVL_KPSEL_EAX + movl %ax,%fs + FAKE_MCOUNT(13*4(%esp)) #ifdef SMP MPLOCKED incl _cnt+V_TRAP @@ -198,11 +203,14 @@ _alltraps: pushal pushl %ds pushl %es + pushl %fs alltraps_with_regs_pushed: movl $KDSEL,%eax movl %ax,%ds movl %ax,%es - FAKE_MCOUNT(12*4(%esp)) + MOVL_KPSEL_EAX + movl %ax,%fs + FAKE_MCOUNT(13*4(%esp)) calltrap: FAKE_MCOUNT(_btrap) /* init "from" _btrap -> calltrap */ MPLOCKED incl _cnt+V_TRAP @@ -249,13 +257,16 @@ IDTVEC(syscall) pushal pushl %ds pushl %es + pushl %fs movl $KDSEL,%eax /* switch to kernel segments */ movl %ax,%ds movl %ax,%es + MOVL_KPSEL_EAX + movl %ax,%fs movl TF_ERR(%esp),%eax /* copy saved eflags to final spot */ movl %eax,TF_EFLAGS(%esp) movl $7,TF_ERR(%esp) /* sizeof "lcall 7,0" */ - FAKE_MCOUNT(12*4(%esp)) + FAKE_MCOUNT(13*4(%esp)) MPLOCKED incl _cnt+V_SYSCALL SYSCALL_LOCK ECPL_LOCK @@ -285,11 +296,14 @@ IDTVEC(int0x80_syscall) pushal pushl %ds pushl %es + pushl %fs movl $KDSEL,%eax /* switch to kernel segments */ movl %ax,%ds movl %ax,%es + MOVL_KPSEL_EAX + movl %ax,%fs movl $2,TF_ERR(%esp) /* sizeof "int 0x80" */ - FAKE_MCOUNT(12*4(%esp)) + FAKE_MCOUNT(13*4(%esp)) MPLOCKED incl _cnt+V_SYSCALL ALTSYSCALL_LOCK ECPL_LOCK @@ -316,7 +330,9 @@ ENTRY(fork_trampoline) #ifdef SMP cmpl $0,_switchtime jne 1f - pushl $_switchtime + movl $gd_switchtime,%eax + addl %fs:0,%eax + pushl %eax call _microuptime popl %edx movl _ticks,%eax diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c index 539f3e2..3525455 100644 --- a/sys/amd64/amd64/genassym.c +++ b/sys/amd64/amd64/genassym.c @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)genassym.c 5.11 (Berkeley) 5/10/91 - * $Id: genassym.c,v 1.64 1999/02/28 10:53:28 bde Exp $ + * $Id: genassym.c,v 1.65 1999/04/02 17:59:37 alc Exp $ */ #include "opt_vm86.h" @@ -66,9 +66,7 @@ #ifdef SMP #include <machine/apic.h> #endif -#ifdef VM86 #include <machine/segments.h> -#endif #include <machine/globaldata.h> #define OS(s, m) ((u_int)offsetof(struct s, m)) @@ -126,7 +124,6 @@ main() printf("#define\tPCB_EIP %#x\n", OS(pcb, pcb_eip)); printf("#define\tTSS_ESP0 %#x\n", OS(i386tss, tss_esp0)); printf("#define\tPCB_USERLDT %#x\n", OS(pcb, pcb_ldt)); - printf("#define\tPCB_FS %#x\n", OS(pcb, pcb_fs)); printf("#define\tPCB_GS %#x\n", OS(pcb, pcb_gs)); #ifdef VM86 printf("#define\tPCB_EXT %#x\n", OS(pcb, pcb_ext)); @@ -134,6 +131,7 @@ main() #ifdef SMP printf("#define\tPCB_MPNEST %#x\n", OS(pcb, pcb_mpnest)); #endif + printf("#define\tPCB_SPARE %#x\n", OS(pcb, __pcb_spare)); printf("#define\tU_PROF %#x\n", OS(user, u_stats.p_prof)); printf("#define\tU_PROFSCALE %#x\n", OS(user, u_stats.p_prof.pr_scale)); @@ -194,42 +192,43 @@ main() printf("#define\tBI_MODULEP %#x\n", OS(bootinfo, bi_modulep)); printf("#define\tGD_SIZEOF %u\n", sizeof(struct globaldata)); - printf("#define\tGD_CURPROC %#x\n", OS(globaldata, curproc)); - printf("#define\tGD_NPXPROC %#x\n", OS(globaldata, npxproc)); - printf("#define\tGD_CURPCB %#x\n", OS(globaldata, curpcb)); - printf("#define\tGD_COMMON_TSS %#x\n", OS(globaldata, common_tss)); - printf("#define\tGD_SWITCHTIME %#x\n", OS(globaldata, switchtime)); - printf("#define\tGD_SWITCHTICKS %#x\n", OS(globaldata, switchticks)); + printf("#define\tGD_CURPROC %#x\n", OS(globaldata, gd_curproc)); + printf("#define\tGD_NPXPROC %#x\n", OS(globaldata, gd_npxproc)); + printf("#define\tGD_CURPCB %#x\n", OS(globaldata, gd_curpcb)); + printf("#define\tGD_COMMON_TSS %#x\n", OS(globaldata, gd_common_tss)); + printf("#define\tGD_SWITCHTIME %#x\n", OS(globaldata, gd_switchtime)); + printf("#define\tGD_SWITCHTICKS %#x\n", OS(globaldata, gd_switchticks)); #ifdef VM86 - printf("#define\tGD_COMMON_TSSD %#x\n", OS(globaldata, common_tssd)); - printf("#define\tGD_PRIVATE_TSS %#x\n", OS(globaldata, private_tss)); - printf("#define\tGD_MY_TR %#x\n", OS(globaldata, my_tr)); + printf("#define\tGD_COMMON_TSSD %#x\n", OS(globaldata, gd_common_tssd)); #endif #ifdef USER_LDT - printf("#define\tGD_CURRENTLDT %#x\n", OS(globaldata, currentldt)); + printf("#define\tGD_CURRENTLDT %#x\n", OS(globaldata, gd_currentldt)); #endif #ifdef SMP - printf("#define\tGD_CPUID %#x\n", OS(globaldata, cpuid)); - printf("#define\tGD_CPU_LOCKID %#x\n", OS(globaldata, cpu_lockid)); - printf("#define\tGD_OTHER_CPUS %#x\n", OS(globaldata, other_cpus)); - printf("#define\tGD_MY_IDLEPTD %#x\n", OS(globaldata, my_idlePTD)); - printf("#define\tGD_SS_EFLAGS %#x\n", OS(globaldata, ss_eflags)); - printf("#define\tGD_PRV_CMAP1 %#x\n", OS(globaldata, prv_CMAP1)); - printf("#define\tGD_PRV_CMAP2 %#x\n", OS(globaldata, prv_CMAP2)); - printf("#define\tGD_PRV_CMAP3 %#x\n", OS(globaldata, prv_CMAP3)); - printf("#define\tGD_PRV_PMAP1 %#x\n", OS(globaldata, prv_PMAP1)); - printf("#define\tGD_INSIDE_INTR %#x\n", OS(globaldata, inside_intr)); + printf("#define\tGD_CPUID %#x\n", OS(globaldata, gd_cpuid)); + printf("#define\tGD_CPU_LOCKID %#x\n", OS(globaldata, gd_cpu_lockid)); + printf("#define\tGD_OTHER_CPUS %#x\n", OS(globaldata, gd_other_cpus)); + printf("#define\tGD_SS_EFLAGS %#x\n", OS(globaldata, gd_ss_eflags)); + printf("#define\tGD_INSIDE_INTR %#x\n", OS(globaldata, gd_inside_intr)); + printf("#define\tGD_PRV_CMAP1 %#x\n", OS(globaldata, gd_prv_CMAP1)); + printf("#define\tGD_PRV_CMAP2 %#x\n", OS(globaldata, gd_prv_CMAP2)); + printf("#define\tGD_PRV_CMAP3 %#x\n", OS(globaldata, gd_prv_CMAP3)); + printf("#define\tGD_PRV_PMAP1 %#x\n", OS(globaldata, gd_prv_PMAP1)); + printf("#define\tGD_PRV_CADDR1 %#x\n", OS(globaldata, gd_prv_CADDR1)); + printf("#define\tGD_PRV_CADDR2 %#x\n", OS(globaldata, gd_prv_CADDR2)); + printf("#define\tGD_PRV_CADDR3 %#x\n", OS(globaldata, gd_prv_CADDR3)); + printf("#define\tGD_PRV_PADDR1 %#x\n", OS(globaldata, gd_prv_PADDR1)); printf("#define\tPS_GLOBALDATA %#x\n", OS(privatespace, globaldata)); - printf("#define\tPS_PRVPT %#x\n", OS(privatespace, prvpt)); - printf("#define\tPS_LAPIC %#x\n", OS(privatespace, lapic)); printf("#define\tPS_IDLESTACK %#x\n", OS(privatespace, idlestack)); - printf("#define\tPS_IDLESTACK_TOP %#x\n", OS(privatespace, CPAGE1)); - printf("#define\tPS_CPAGE1 %#x\n", OS(privatespace, CPAGE1)); - printf("#define\tPS_CPAGE2 %#x\n", OS(privatespace, CPAGE2)); - printf("#define\tPS_CPAGE3 %#x\n", OS(privatespace, CPAGE3)); - printf("#define\tPS_PPAGE1 %#x\n", OS(privatespace, PPAGE1)); - printf("#define\tPS_IOAPICS %#x\n", OS(privatespace, ioapics)); + printf("#define\tPS_IDLESTACK_TOP %#x\n", sizeof(struct privatespace)); +#endif + + printf("#define\tKCSEL %#x\n", GSEL(GCODE_SEL, SEL_KPL)); + printf("#define\tKDSEL %#x\n", GSEL(GDATA_SEL, SEL_KPL)); +#ifdef SMP + printf("#define\tKPSEL %#x\n", GSEL(GPRIV_SEL, SEL_KPL)); #endif + printf("#define\tGPROC0_SEL %#x\n", GPROC0_SEL); return (0); } diff --git a/sys/amd64/amd64/locore.S b/sys/amd64/amd64/locore.S index 981418b..63f74cc 100644 --- a/sys/amd64/amd64/locore.S +++ b/sys/amd64/amd64/locore.S @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)locore.s 7.3 (Berkeley) 5/13/91 - * $Id: locore.s,v 1.119 1999/01/30 15:38:47 kato Exp $ + * $Id: locore.s,v 1.120 1999/01/31 02:04:43 kato Exp $ * * originally from: locore.s, by William F. Jolitz * @@ -102,7 +102,7 @@ HIDENAME(tmpstk): .globl _cpu,_cpu_vendor,_cpu_id,_bootinfo .globl _cpu_high, _cpu_feature -_cpu: .long 0 /* are we 386, 386sx, or 486 */ +_cpu: .long 0 /* are we 386, 386sx, or 486 */ _cpu_id: .long 0 /* stepping ID */ _cpu_high: .long 0 /* highest arg to CPUID */ _cpu_feature: .long 0 /* features */ @@ -113,12 +113,13 @@ _KERNend: .long 0 /* phys addr end of kernel (just after bss) */ physfree: .long 0 /* phys addr of next free page */ #ifdef SMP + .globl _cpu0prvpage cpu0pp: .long 0 /* phys addr cpu0 private pg */ -cpu0pt: .long 0 /* phys addr cpu0 private pt */ - - .globl _cpu0prvpage,_cpu0prvpt _cpu0prvpage: .long 0 /* relocated version */ -_cpu0prvpt: .long 0 /* relocated version */ + + .globl _SMPpt +SMPptpa: .long 0 /* phys addr SMP page table */ +_SMPpt: .long 0 /* relocated version */ #endif /* SMP */ .globl _IdlePTD @@ -370,7 +371,6 @@ begin: movl _proc0paddr,%eax movl _IdlePTD, %esi movl %esi,PCB_CR3(%eax) - movl $_proc0,_curproc movl physfree, %esi pushl %esi /* value of first for init386(first) */ @@ -385,7 +385,7 @@ begin: pushl $PSL_USER /* eflags (IOPL 0, int enab) */ pushl __ucodesel /* cs */ pushl $0 /* eip - filled in by execve() */ - subl $(12*4),%esp /* space for rest of registers */ + subl $(13*4),%esp /* space for rest of registers */ pushl %esp /* call main with frame pointer */ call _main /* autoconfiguration, mountroot etc */ @@ -417,11 +417,11 @@ NON_GPROF_ENTRY(prepare_usermode) movl __ucodesel,%eax movl __udatasel,%ecx -#if 0 +#if 0 /* ds/es/fs are in trap frame */ movl %cx,%ds -#endif movl %cx,%es - movl %ax,%fs /* double map cs to fs */ + movl %cx,%fs +#endif movl %cx,%gs /* and ds to gs */ ret /* goto user! */ @@ -803,11 +803,11 @@ no_kernend: addl $KERNBASE, %esi movl %esi, R(_cpu0prvpage) /* relocated to KVM space */ -/* Allocate cpu0's private page table for mapping priv page, apic, etc */ +/* Allocate SMP page table page */ ALLOCPAGES(1) - movl %esi,R(cpu0pt) + movl %esi,R(SMPptpa) addl $KERNBASE, %esi - movl %esi, R(_cpu0prvpt) /* relocated to KVM space */ + movl %esi, R(_SMPpt) /* relocated to KVM space */ #endif /* SMP */ /* Map read-only from zero to the end of the kernel text section */ @@ -887,25 +887,19 @@ map_read_write: movl $1, %ecx fillkptphys($PG_RW) -/* Map cpu0's private page table into global kmem FWIW */ - movl R(cpu0pt), %eax +/* Map SMP page table page into global kmem FWIW */ + movl R(SMPptpa), %eax movl $1, %ecx fillkptphys($PG_RW) -/* Map the private page into the private page table into private space */ +/* Map the private page into the SMP page table */ movl R(cpu0pp), %eax movl $0, %ebx /* pte offset = 0 */ movl $1, %ecx /* one private page coming right up */ - fillkpt(R(cpu0pt), $PG_RW) - -/* Map the page table page into private space */ - movl R(cpu0pt), %eax - movl $1, %ebx /* pte offset = 1 */ - movl $1, %ecx /* one private pt coming right up */ - fillkpt(R(cpu0pt), $PG_RW) + fillkpt(R(SMPptpa), $PG_RW) /* ... and put the page table table in the pde. */ - movl R(cpu0pt), %eax + movl R(SMPptpa), %eax movl $MPPTDI, %ebx movl $1, %ecx fillkpt(R(_IdlePTD), $PG_RW) @@ -913,21 +907,12 @@ map_read_write: /* Fakeup VA for the local apic to allow early traps. */ ALLOCPAGES(1) movl %esi, %eax - movl $2, %ebx /* pte offset = 2 */ + movl $(NPTEPG-1), %ebx /* pte offset = NTEPG-1 */ movl $1, %ecx /* one private pt coming right up */ - fillkpt(R(cpu0pt), $PG_RW) + fillkpt(R(SMPptpa), $PG_RW) /* Initialize mp lock to allow early traps */ movl $1, R(_mp_lock) - -/* Initialize my_idlePTD to IdlePTD */ - movl R(cpu0pp), %eax - movl R(_IdlePTD), %ecx - movl %ecx,GD_MY_IDLEPTD(%eax) -/* Initialize IdlePTDS[0] */ - addl $KERNBASE, %ecx - movl %ecx, R(CNAME(IdlePTDS)) - #endif /* SMP */ /* install a pde for temporary double map of bottom of VA */ diff --git a/sys/amd64/amd64/locore.s b/sys/amd64/amd64/locore.s index 981418b..63f74cc 100644 --- a/sys/amd64/amd64/locore.s +++ b/sys/amd64/amd64/locore.s @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)locore.s 7.3 (Berkeley) 5/13/91 - * $Id: locore.s,v 1.119 1999/01/30 15:38:47 kato Exp $ + * $Id: locore.s,v 1.120 1999/01/31 02:04:43 kato Exp $ * * originally from: locore.s, by William F. Jolitz * @@ -102,7 +102,7 @@ HIDENAME(tmpstk): .globl _cpu,_cpu_vendor,_cpu_id,_bootinfo .globl _cpu_high, _cpu_feature -_cpu: .long 0 /* are we 386, 386sx, or 486 */ +_cpu: .long 0 /* are we 386, 386sx, or 486 */ _cpu_id: .long 0 /* stepping ID */ _cpu_high: .long 0 /* highest arg to CPUID */ _cpu_feature: .long 0 /* features */ @@ -113,12 +113,13 @@ _KERNend: .long 0 /* phys addr end of kernel (just after bss) */ physfree: .long 0 /* phys addr of next free page */ #ifdef SMP + .globl _cpu0prvpage cpu0pp: .long 0 /* phys addr cpu0 private pg */ -cpu0pt: .long 0 /* phys addr cpu0 private pt */ - - .globl _cpu0prvpage,_cpu0prvpt _cpu0prvpage: .long 0 /* relocated version */ -_cpu0prvpt: .long 0 /* relocated version */ + + .globl _SMPpt +SMPptpa: .long 0 /* phys addr SMP page table */ +_SMPpt: .long 0 /* relocated version */ #endif /* SMP */ .globl _IdlePTD @@ -370,7 +371,6 @@ begin: movl _proc0paddr,%eax movl _IdlePTD, %esi movl %esi,PCB_CR3(%eax) - movl $_proc0,_curproc movl physfree, %esi pushl %esi /* value of first for init386(first) */ @@ -385,7 +385,7 @@ begin: pushl $PSL_USER /* eflags (IOPL 0, int enab) */ pushl __ucodesel /* cs */ pushl $0 /* eip - filled in by execve() */ - subl $(12*4),%esp /* space for rest of registers */ + subl $(13*4),%esp /* space for rest of registers */ pushl %esp /* call main with frame pointer */ call _main /* autoconfiguration, mountroot etc */ @@ -417,11 +417,11 @@ NON_GPROF_ENTRY(prepare_usermode) movl __ucodesel,%eax movl __udatasel,%ecx -#if 0 +#if 0 /* ds/es/fs are in trap frame */ movl %cx,%ds -#endif movl %cx,%es - movl %ax,%fs /* double map cs to fs */ + movl %cx,%fs +#endif movl %cx,%gs /* and ds to gs */ ret /* goto user! */ @@ -803,11 +803,11 @@ no_kernend: addl $KERNBASE, %esi movl %esi, R(_cpu0prvpage) /* relocated to KVM space */ -/* Allocate cpu0's private page table for mapping priv page, apic, etc */ +/* Allocate SMP page table page */ ALLOCPAGES(1) - movl %esi,R(cpu0pt) + movl %esi,R(SMPptpa) addl $KERNBASE, %esi - movl %esi, R(_cpu0prvpt) /* relocated to KVM space */ + movl %esi, R(_SMPpt) /* relocated to KVM space */ #endif /* SMP */ /* Map read-only from zero to the end of the kernel text section */ @@ -887,25 +887,19 @@ map_read_write: movl $1, %ecx fillkptphys($PG_RW) -/* Map cpu0's private page table into global kmem FWIW */ - movl R(cpu0pt), %eax +/* Map SMP page table page into global kmem FWIW */ + movl R(SMPptpa), %eax movl $1, %ecx fillkptphys($PG_RW) -/* Map the private page into the private page table into private space */ +/* Map the private page into the SMP page table */ movl R(cpu0pp), %eax movl $0, %ebx /* pte offset = 0 */ movl $1, %ecx /* one private page coming right up */ - fillkpt(R(cpu0pt), $PG_RW) - -/* Map the page table page into private space */ - movl R(cpu0pt), %eax - movl $1, %ebx /* pte offset = 1 */ - movl $1, %ecx /* one private pt coming right up */ - fillkpt(R(cpu0pt), $PG_RW) + fillkpt(R(SMPptpa), $PG_RW) /* ... and put the page table table in the pde. */ - movl R(cpu0pt), %eax + movl R(SMPptpa), %eax movl $MPPTDI, %ebx movl $1, %ecx fillkpt(R(_IdlePTD), $PG_RW) @@ -913,21 +907,12 @@ map_read_write: /* Fakeup VA for the local apic to allow early traps. */ ALLOCPAGES(1) movl %esi, %eax - movl $2, %ebx /* pte offset = 2 */ + movl $(NPTEPG-1), %ebx /* pte offset = NTEPG-1 */ movl $1, %ecx /* one private pt coming right up */ - fillkpt(R(cpu0pt), $PG_RW) + fillkpt(R(SMPptpa), $PG_RW) /* Initialize mp lock to allow early traps */ movl $1, R(_mp_lock) - -/* Initialize my_idlePTD to IdlePTD */ - movl R(cpu0pp), %eax - movl R(_IdlePTD), %ecx - movl %ecx,GD_MY_IDLEPTD(%eax) -/* Initialize IdlePTDS[0] */ - addl $KERNBASE, %ecx - movl %ecx, R(CNAME(IdlePTDS)) - #endif /* SMP */ /* install a pde for temporary double map of bottom of VA */ diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c index 712996e..ac02b7a 100644 --- a/sys/amd64/amd64/machdep.c +++ b/sys/amd64/amd64/machdep.c @@ -35,7 +35,7 @@ * SUCH DAMAGE. * * from: @(#)machdep.c 7.4 (Berkeley) 6/3/91 - * $Id: machdep.c,v 1.330 1999/04/19 14:14:12 peter Exp $ + * $Id: machdep.c,v 1.331 1999/04/26 08:57:51 peter Exp $ */ #include "apm.h" @@ -114,6 +114,7 @@ #include <machine/pcb_ext.h> /* pcb.h included via sys/user.h */ #ifdef SMP #include <machine/smp.h> +#include <machine/globaldata.h> #endif #ifdef PERFMON #include <machine/perfmon.h> @@ -552,6 +553,7 @@ sendsig(catcher, sig, mask, code) sf.sf_sc.sc_ds = regs->tf_ds; sf.sf_sc.sc_ss = regs->tf_ss; sf.sf_sc.sc_es = regs->tf_es; + sf.sf_sc.sc_fs = regs->tf_fs; sf.sf_sc.sc_isp = regs->tf_isp; /* @@ -616,6 +618,7 @@ sendsig(catcher, sig, mask, code) regs->tf_cs = _ucodesel; regs->tf_ds = _udatasel; regs->tf_es = _udatasel; + regs->tf_fs = _udatasel; regs->tf_ss = _udatasel; } @@ -686,6 +689,7 @@ sigreturn(p, uap) tf->tf_vm86_gs = scp->sc_gs; tf->tf_ds = _udatasel; tf->tf_es = _udatasel; + tf->tf_fs = _udatasel; } else { #endif /* VM86 */ /* @@ -724,6 +728,7 @@ sigreturn(p, uap) } regs->tf_ds = scp->sc_ds; regs->tf_es = scp->sc_es; + regs->tf_fs = scp->sc_fs; #ifdef VM86 } #endif @@ -808,17 +813,16 @@ setregs(p, entry, stack, ps_strings) regs->tf_ss = _udatasel; regs->tf_ds = _udatasel; regs->tf_es = _udatasel; + regs->tf_fs = _udatasel; regs->tf_cs = _ucodesel; /* PS_STRINGS value for BSD/OS binaries. It is 0 for non-BSD/OS. */ regs->tf_ebx = ps_strings; - /* reset %fs and %gs as well */ - pcb->pcb_fs = _udatasel; + /* reset %gs as well */ pcb->pcb_gs = _udatasel; if (pcb == curpcb) { - __asm("movw %w0,%%fs" : : "r" (_udatasel)); - __asm("movw %w0,%%gs" : : "r" (_udatasel)); + load_gs(_udatasel); } /* @@ -887,7 +891,7 @@ SYSCTL_INT(_machdep, CPU_WALLCLOCK, wall_cmos_clock, int _default_ldt; #ifdef SMP -union descriptor gdt[NGDT + NCPU]; /* global descriptor table */ +union descriptor gdt[NGDT * NCPU]; /* global descriptor table */ #else union descriptor gdt[NGDT]; /* global descriptor table */ #endif @@ -898,11 +902,11 @@ union descriptor ldt[NLDT]; /* local descriptor table */ struct region_descriptor r_gdt, r_idt; #endif -extern struct i386tss common_tss; /* One tss per cpu */ #ifdef VM86 +#ifndef SMP extern struct segment_descriptor common_tssd; -extern int private_tss; /* flag indicating private tss */ -extern u_int my_tr; /* which task register setting */ +#endif +int private_tss; /* flag indicating private tss */ #endif /* VM86 */ #if defined(I586_CPU) && !defined(NO_F00F_HACK) @@ -917,11 +921,7 @@ extern struct user *proc0paddr; /* software prototypes -- in more palatable form */ -struct soft_segment_descriptor gdt_segs[ -#ifdef SMP - NGDT + NCPU -#endif - ] = { +struct soft_segment_descriptor gdt_segs[] = { /* GNULL_SEL 0 Null Descriptor */ { 0x0, /* segment base address */ 0x0, /* length */ @@ -949,7 +949,26 @@ struct soft_segment_descriptor gdt_segs[ 0, 0, 1, /* default 32 vs 16 bit size */ 1 /* limit granularity (byte/page units)*/ }, -/* GLDT_SEL 3 LDT Descriptor */ +/* GPRIV_SEL 3 SMP Per-Processor Private Data Descriptor */ +{ 0x0, /* segment base address */ + 0xfffff, /* length - all address space */ + SDT_MEMRWA, /* segment type */ + 0, /* segment descriptor priority level */ + 1, /* segment descriptor present */ + 0, 0, + 1, /* default 32 vs 16 bit size */ + 1 /* limit granularity (byte/page units)*/ }, +/* GPROC0_SEL 4 Proc 0 Tss Descriptor */ +{ + 0x0, /* segment base address */ + sizeof(struct i386tss)-1,/* length - all address space */ + SDT_SYS386TSS, /* segment type */ + 0, /* segment descriptor priority level */ + 1, /* segment descriptor present */ + 0, 0, + 0, /* unused - default 32 vs 16 bit size */ + 0 /* limit granularity (byte/page units)*/ }, +/* GLDT_SEL 5 LDT Descriptor */ { (int) ldt, /* segment base address */ sizeof(ldt)-1, /* length - all address space */ SDT_SYSLDT, /* segment type */ @@ -958,7 +977,16 @@ struct soft_segment_descriptor gdt_segs[ 0, 0, 0, /* unused - default 32 vs 16 bit size */ 0 /* limit granularity (byte/page units)*/ }, -/* GTGATE_SEL 4 Null Descriptor - Placeholder */ +/* GUSERLDT_SEL 6 User LDT Descriptor per process */ +{ (int) ldt, /* segment base address */ + (512 * sizeof(union descriptor)-1), /* length */ + SDT_SYSLDT, /* segment type */ + 0, /* segment descriptor priority level */ + 1, /* segment descriptor present */ + 0, 0, + 0, /* unused - default 32 vs 16 bit size */ + 0 /* limit granularity (byte/page units)*/ }, +/* GTGATE_SEL 7 Null Descriptor - Placeholder */ { 0x0, /* segment base address */ 0x0, /* length - all address space */ 0, /* segment type */ @@ -967,7 +995,7 @@ struct soft_segment_descriptor gdt_segs[ 0, 0, 0, /* default 32 vs 16 bit size */ 0 /* limit granularity (byte/page units)*/ }, -/* GPANIC_SEL 5 Panic Tss Descriptor */ +/* GPANIC_SEL 8 Panic Tss Descriptor */ { (int) &dblfault_tss, /* segment base address */ sizeof(struct i386tss)-1,/* length - all address space */ SDT_SYS386TSS, /* segment type */ @@ -976,26 +1004,7 @@ struct soft_segment_descriptor gdt_segs[ 0, 0, 0, /* unused - default 32 vs 16 bit size */ 0 /* limit granularity (byte/page units)*/ }, -/* GPROC0_SEL 6 Proc 0 Tss Descriptor */ -{ - (int) &common_tss, /* segment base address */ - sizeof(struct i386tss)-1,/* length - all address space */ - SDT_SYS386TSS, /* segment type */ - 0, /* segment descriptor priority level */ - 1, /* segment descriptor present */ - 0, 0, - 0, /* unused - default 32 vs 16 bit size */ - 0 /* limit granularity (byte/page units)*/ }, -/* GUSERLDT_SEL 7 User LDT Descriptor per process */ -{ (int) ldt, /* segment base address */ - (512 * sizeof(union descriptor)-1), /* length */ - SDT_SYSLDT, /* segment type */ - 0, /* segment descriptor priority level */ - 1, /* segment descriptor present */ - 0, 0, - 0, /* unused - default 32 vs 16 bit size */ - 0 /* limit granularity (byte/page units)*/ }, -/* GAPMCODE32_SEL 8 APM BIOS 32-bit interface (32bit Code) */ +/* GAPMCODE32_SEL 9 APM BIOS 32-bit interface (32bit Code) */ { 0, /* segment base address (overwritten by APM) */ 0xfffff, /* length */ SDT_MEMERA, /* segment type */ @@ -1004,7 +1013,7 @@ struct soft_segment_descriptor gdt_segs[ 0, 0, 1, /* default 32 vs 16 bit size */ 1 /* limit granularity (byte/page units)*/ }, -/* GAPMCODE16_SEL 9 APM BIOS 32-bit interface (16bit Code) */ +/* GAPMCODE16_SEL 10 APM BIOS 32-bit interface (16bit Code) */ { 0, /* segment base address (overwritten by APM) */ 0xfffff, /* length */ SDT_MEMERA, /* segment type */ @@ -1013,7 +1022,7 @@ struct soft_segment_descriptor gdt_segs[ 0, 0, 0, /* default 32 vs 16 bit size */ 1 /* limit granularity (byte/page units)*/ }, -/* GAPMDATA_SEL 10 APM BIOS 32-bit interface (Data) */ +/* GAPMDATA_SEL 11 APM BIOS 32-bit interface (Data) */ { 0, /* segment base address (overwritten by APM) */ 0xfffff, /* length */ SDT_MEMRWA, /* segment type */ @@ -1160,11 +1169,6 @@ init386(first) atdevbase = ISA_HOLE_START + KERNBASE; /* - * Initialize the console before we print anything out. - */ - cninit(); - - /* * make gdt memory segments, the code segment goes up to end of the * page with etext in it, the data segment goes to the end of * the address space @@ -1175,27 +1179,30 @@ init386(first) */ gdt_segs[GCODE_SEL].ssd_limit = i386_btop(0) - 1; gdt_segs[GDATA_SEL].ssd_limit = i386_btop(0) - 1; -#ifdef BDE_DEBUGGER -#define NGDT1 8 /* avoid overwriting db entries with APM ones */ +#ifdef SMP + gdt_segs[GPRIV_SEL].ssd_limit = + i386_btop(sizeof(struct privatespace)) - 1; + gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[0]; + gdt_segs[GPROC0_SEL].ssd_base = + (int) &SMP_prvspace[0].globaldata.gd_common_tss; + SMP_prvspace[0].globaldata.gd_prvspace = &SMP_prvspace[0]; #else -#define NGDT1 (sizeof gdt_segs / sizeof gdt_segs[0]) + gdt_segs[GPRIV_SEL].ssd_limit = i386_btop(0) - 1; + gdt_segs[GPROC0_SEL].ssd_base = (int) &common_tss; #endif - for (x = 0; x < NGDT1; x++) - ssdtosd(&gdt_segs[x], &gdt[x].sd); -#ifdef VM86 - common_tssd = gdt[GPROC0_SEL].sd; -#endif /* VM86 */ -#ifdef SMP - /* - * Spin these up now. init_secondary() grabs them. We could use - * #for(x,y,z) / #endfor cpp directives if they existed. - */ - for (x = 0; x < NCPU; x++) { - gdt_segs[NGDT + x] = gdt_segs[GPROC0_SEL]; - ssdtosd(&gdt_segs[NGDT + x], &gdt[NGDT + x].sd); - } + for (x = 0; x < NGDT; x++) { +#ifdef BDE_DEBUGGER + /* avoid overwriting db entries with APM ones */ + if (x >= GAPMCODE32_SEL && x <= GAPMDATA_SEL) + continue; #endif + ssdtosd(&gdt_segs[x], &gdt[x].sd); + } + + r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1; + r_gdt.rd_base = (int) gdt; + lgdt(&r_gdt); /* make ldt memory segments */ /* @@ -1221,6 +1228,12 @@ init386(first) for (x = 0; x < sizeof ldt_segs / sizeof ldt_segs[0]; x++) ssdtosd(&ldt_segs[x], &ldt[x].sd); + _default_ldt = GSEL(GLDT_SEL, SEL_KPL); + lldt(_default_ldt); +#ifdef USER_LDT + currentldt = _default_ldt; +#endif + /* exceptions */ for (x = 0; x < NIDT; x++) setidt(x, &IDTVEC(rsvd), SDT_SYS386TGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); @@ -1246,25 +1259,20 @@ init386(first) setidt(0x80, &IDTVEC(int0x80_syscall), SDT_SYS386TGT, SEL_UPL, GSEL(GCODE_SEL, SEL_KPL)); -#include "isa.h" -#if NISA >0 - isa_defaultirq(); -#endif - rand_initialize(); - - r_gdt.rd_limit = sizeof(gdt) - 1; - r_gdt.rd_base = (int) gdt; - lgdt(&r_gdt); - r_idt.rd_limit = sizeof(idt) - 1; r_idt.rd_base = (int) idt; lidt(&r_idt); - _default_ldt = GSEL(GLDT_SEL, SEL_KPL); - lldt(_default_ldt); -#ifdef USER_LDT - currentldt = _default_ldt; + /* + * Initialize the console before we print anything out. + */ + cninit(); + +#include "isa.h" +#if NISA >0 + isa_defaultirq(); #endif + rand_initialize(); #ifdef DDB kdb_init(); @@ -1289,7 +1297,7 @@ init386(first) ltr(gsel_tss); #ifdef VM86 private_tss = 0; - my_tr = GPROC0_SEL; + common_tssd = gdt[GPROC0_SEL].sd; #endif dblfault_tss.tss_esp = dblfault_tss.tss_esp0 = dblfault_tss.tss_esp1 = @@ -1607,6 +1615,7 @@ init386(first) #ifdef VM86 proc0.p_addr->u_pcb.pcb_ext = 0; #endif + SET_CURPROC(&proc0); /* Sigh, relocate physical addresses left from bootstrap */ if (bootinfo.bi_modulep) { @@ -1732,6 +1741,7 @@ fill_regs(p, regs) struct trapframe *tp; tp = p->p_md.md_regs; + regs->r_fs = tp->tf_fs; regs->r_es = tp->tf_es; regs->r_ds = tp->tf_ds; regs->r_edi = tp->tf_edi; @@ -1747,7 +1757,6 @@ fill_regs(p, regs) regs->r_esp = tp->tf_esp; regs->r_ss = tp->tf_ss; pcb = &p->p_addr->u_pcb; - regs->r_fs = pcb->pcb_fs; regs->r_gs = pcb->pcb_gs; return (0); } @@ -1764,6 +1773,7 @@ set_regs(p, regs) if (!EFLAGS_SECURE(regs->r_eflags, tp->tf_eflags) || !CS_SECURE(regs->r_cs)) return (EINVAL); + tp->tf_fs = regs->r_fs; tp->tf_es = regs->r_es; tp->tf_ds = regs->r_ds; tp->tf_edi = regs->r_edi; @@ -1779,7 +1789,6 @@ set_regs(p, regs) tp->tf_esp = regs->r_esp; tp->tf_ss = regs->r_ss; pcb = &p->p_addr->u_pcb; - pcb->pcb_fs = regs->r_fs; pcb->pcb_gs = regs->r_gs; return (0); } diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index eca2601..dcca437 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.96 1999/04/11 00:43:43 tegge Exp $ + * $Id: mp_machdep.c,v 1.97 1999/04/13 03:24:47 tegge Exp $ */ #include "opt_smp.h" @@ -297,28 +297,15 @@ int apic_id_to_logical[NAPICID]; /* Bitmap of all available CPUs */ u_int all_cpus; -/* AP uses this PTD during bootstrap. Do not staticize. */ -pd_entry_t *bootPTD; +/* AP uses this during bootstrap. Do not staticize. */ +char *bootSTK; +int boot_cpuid; /* Hotwire a 0->4MB V==P mapping */ extern pt_entry_t *KPTphys; -/* Virtual address of per-cpu common_tss */ -extern struct i386tss common_tss; -#ifdef VM86 -extern struct segment_descriptor common_tssd; -extern u_int private_tss; /* flag indicating private tss */ -extern u_int my_tr; -#endif /* VM86 */ - -/* IdlePTD per cpu */ -pd_entry_t *IdlePTDS[NCPU]; - -/* "my" private page table page, for BSP init */ -extern pt_entry_t SMP_prvpt[]; - -/* Private page pointer to curcpu's PTD, used during BSP init */ -extern pd_entry_t *my_idlePTD; +/* SMP page table page */ +extern pt_entry_t *SMPpt; struct pcb stoppcbs[NCPU]; @@ -466,41 +453,42 @@ void init_secondary(void) { int gsel_tss; -#ifndef VM86 - u_int my_tr; -#endif + int x, myid = boot_cpuid; + + gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid]; + gdt_segs[GPROC0_SEL].ssd_base = + (int) &SMP_prvspace[myid].globaldata.gd_common_tss; + SMP_prvspace[myid].globaldata.gd_prvspace = &SMP_prvspace[myid]; - r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1; - r_gdt.rd_base = (int) gdt; + for (x = 0; x < NGDT; x++) { + ssdtosd(&gdt_segs[x], &gdt[myid * NGDT + x].sd); + } + + r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1; + r_gdt.rd_base = (int) &gdt[myid * NGDT]; lgdt(&r_gdt); /* does magic intra-segment return */ + lidt(&r_idt); + lldt(_default_ldt); #ifdef USER_LDT currentldt = _default_ldt; #endif - my_tr = NGDT + cpuid; - gsel_tss = GSEL(my_tr, SEL_KPL); - gdt[my_tr].sd.sd_type = SDT_SYS386TSS; + gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); + gdt[myid * NGDT + GPROC0_SEL].sd.sd_type = SDT_SYS386TSS; common_tss.tss_esp0 = 0; /* not used until after switch */ common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); common_tss.tss_ioopt = (sizeof common_tss) << 16; #ifdef VM86 - common_tssd = gdt[my_tr].sd; - private_tss = 0; -#endif /* VM86 */ + common_tssd = gdt[myid * NGDT + GPROC0_SEL].sd; +#endif ltr(gsel_tss); load_cr0(0x8005003b); /* XXX! */ - PTD[0] = 0; pmap_set_opt((unsigned *)PTD); -#if 0 - putmtrr(); - pmap_setvidram(); -#endif - invltlb(); } @@ -558,11 +546,6 @@ mp_enable(u_int boot_addr) u_int ux; #endif /* APIC_IO */ -#if 0 - getmtrr(); - pmap_setvidram(); -#endif - POSTCODE(MP_ENABLE_POST); /* turn on 4MB of V == P addressing so we can get to MP table */ @@ -1770,14 +1753,11 @@ extern int wait_ap(unsigned int); static int start_all_aps(u_int boot_addr) { - int x, i; + int x, i, pg; u_char mpbiosreason; u_long mpbioswarmvec; - pd_entry_t *newptd; - pt_entry_t *newpt; struct globaldata *gd; char *stack; - pd_entry_t *myPTD; POSTCODE(START_ALL_APS_POST); @@ -1799,70 +1779,46 @@ start_all_aps(u_int boot_addr) /* record BSP in CPU map */ all_cpus = 1; + /* set up 0 -> 4MB P==V mapping for AP boot */ + *(int *)PTD = PG_V | PG_RW | ((uintptr_t)(void *)KPTphys & PG_FRAME); + invltlb(); + /* start each AP */ for (x = 1; x <= mp_naps; ++x) { /* This is a bit verbose, it will go away soon. */ - /* alloc new page table directory */ - newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE)); - - /* Store the virtual PTD address for this CPU */ - IdlePTDS[x] = newptd; - - /* clone currently active one (ie: IdlePTD) */ - bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */ - - /* set up 0 -> 4MB P==V mapping for AP boot */ - newptd[0] = (void *)(uintptr_t)(PG_V | PG_RW | - ((uintptr_t)(void *)KPTphys & PG_FRAME)); - - /* store PTD for this AP's boot sequence */ - myPTD = (pd_entry_t *)vtophys(newptd); - - /* alloc new page table page */ - newpt = (pt_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE)); - - /* set the new PTD's private page to point there */ - newptd[MPPTDI] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt)); - - /* install self referential entry */ - newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd)); + /* first page of AP's private space */ + pg = x * i386_btop(sizeof(struct privatespace)); /* allocate a new private data page */ gd = (struct globaldata *)kmem_alloc(kernel_map, PAGE_SIZE); /* wire it into the private page table page */ - newpt[0] = (pt_entry_t)(PG_V | PG_RW | vtophys(gd)); - - /* wire the ptp into itself for access */ - newpt[1] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt)); - - /* copy in the pointer to the local apic */ - newpt[2] = SMP_prvpt[2]; - - /* and the IO apic mapping[s] */ - for (i = 16; i < 32; i++) - newpt[i] = SMP_prvpt[i]; + SMPpt[pg] = (pt_entry_t)(PG_V | PG_RW | vtophys(gd)); /* allocate and set up an idle stack data page */ stack = (char *)kmem_alloc(kernel_map, UPAGES*PAGE_SIZE); for (i = 0; i < UPAGES; i++) - newpt[i + 3] = (pt_entry_t)(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); + SMPpt[pg + 5 + i] = (pt_entry_t) + (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); - newpt[3 + UPAGES] = 0; /* *prv_CMAP1 */ - newpt[4 + UPAGES] = 0; /* *prv_CMAP2 */ - newpt[5 + UPAGES] = 0; /* *prv_CMAP3 */ - newpt[6 + UPAGES] = 0; /* *prv_PMAP1 */ + SMPpt[pg + 1] = 0; /* *prv_CMAP1 */ + SMPpt[pg + 2] = 0; /* *prv_CMAP2 */ + SMPpt[pg + 3] = 0; /* *prv_CMAP3 */ + SMPpt[pg + 4] = 0; /* *prv_PMAP1 */ /* prime data page for it to use */ - gd->cpuid = x; - gd->cpu_lockid = x << 24; - gd->my_idlePTD = myPTD; - gd->prv_CMAP1 = &newpt[3 + UPAGES]; - gd->prv_CMAP2 = &newpt[4 + UPAGES]; - gd->prv_CMAP3 = &newpt[5 + UPAGES]; - gd->prv_PMAP1 = &newpt[6 + UPAGES]; + gd->gd_cpuid = x; + gd->gd_cpu_lockid = x << 24; + gd->gd_prv_CMAP1 = &SMPpt[pg + 1]; + gd->gd_prv_CMAP2 = &SMPpt[pg + 2]; + gd->gd_prv_CMAP3 = &SMPpt[pg + 3]; + gd->gd_prv_PMAP1 = &SMPpt[pg + 4]; + gd->gd_prv_CADDR1 = SMP_prvspace[x].CPAGE1; + gd->gd_prv_CADDR2 = SMP_prvspace[x].CPAGE2; + gd->gd_prv_CADDR3 = SMP_prvspace[x].CPAGE3; + gd->gd_prv_PADDR1 = (unsigned *)SMP_prvspace[x].PPAGE1; /* setup a vector to our boot code */ *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; @@ -1872,7 +1828,9 @@ start_all_aps(u_int boot_addr) outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ #endif - bootPTD = myPTD; + bootSTK = &SMP_prvspace[x].idlestack[UPAGES*PAGE_SIZE]; + boot_cpuid = x; + /* attempt to start the Application Processor */ CHECK_INIT(99); /* setup checkpoints */ if (!start_ap(x, boot_addr)) { @@ -1910,27 +1868,16 @@ start_all_aps(u_int boot_addr) * because the BSP is cpu#0 and the page is initially zero, and also * because we can refer to variables by name on the BSP.. */ - newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE)); - - bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */ - IdlePTDS[0] = newptd; - - /* Point PTD[] to this page instead of IdlePTD's physical page */ - newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd)); - - my_idlePTD = (pd_entry_t *)vtophys(newptd); /* Allocate and setup BSP idle stack */ stack = (char *)kmem_alloc(kernel_map, UPAGES * PAGE_SIZE); for (i = 0; i < UPAGES; i++) - SMP_prvpt[i + 3] = (pt_entry_t)(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); + SMPpt[5 + i] = (pt_entry_t) + (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); + *(int *)PTD = 0; pmap_set_opt_bsp(); - for (i = 0; i < mp_ncpus; i++) { - bcopy( (int *) PTD + KPTDI, (int *) IdlePTDS[i] + KPTDI, NKPDE * sizeof (int)); - } - /* number of APs actually started */ return mp_ncpus - 1; } @@ -2250,10 +2197,6 @@ ap_init() panic("cpuid mismatch! boom!!"); } -#if 0 - getmtrr(); -#endif - /* Init local apic for irq's */ apic_initialize(); @@ -2267,8 +2210,6 @@ ap_init() smp_started = 1; /* enable IPI's, tlb shootdown, freezes etc */ smp_active = 1; /* historic */ } - - curproc = NULL; /* make sure */ } #ifdef BETTER_CLOCK diff --git a/sys/amd64/amd64/mpboot.S b/sys/amd64/amd64/mpboot.S index 7a5d431..8be76d7 100644 --- a/sys/amd64/amd64/mpboot.S +++ b/sys/amd64/amd64/mpboot.S @@ -31,7 +31,7 @@ * mpboot.s: FreeBSD machine support for the Intel MP Spec * multiprocessor systems. * - * $Id: mpboot.s,v 1.8 1998/10/10 10:36:12 kato Exp $ + * $Id: mpboot.s,v 1.9 1999/04/10 22:58:29 tegge Exp $ */ #include "opt_vm86.h" @@ -76,12 +76,12 @@ NON_GPROF_ENTRY(MPentry) CHECKPOINT(0x36, 3) /* Now enable paging mode */ - movl _bootPTD-KERNBASE, %eax + movl _IdlePTD-KERNBASE, %eax movl %eax,%cr3 movl %cr0,%eax orl $CR0_PE|CR0_PG,%eax /* enable paging */ movl %eax,%cr0 /* let the games begin! */ - movl $_idlestack_top,%esp /* boot stack end loc. */ + movl _bootSTK,%esp /* boot stack end loc. */ pushl $mp_begin /* jump to high mem */ ret diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c index eca2601..dcca437 100644 --- a/sys/amd64/amd64/mptable.c +++ b/sys/amd64/amd64/mptable.c @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.96 1999/04/11 00:43:43 tegge Exp $ + * $Id: mp_machdep.c,v 1.97 1999/04/13 03:24:47 tegge Exp $ */ #include "opt_smp.h" @@ -297,28 +297,15 @@ int apic_id_to_logical[NAPICID]; /* Bitmap of all available CPUs */ u_int all_cpus; -/* AP uses this PTD during bootstrap. Do not staticize. */ -pd_entry_t *bootPTD; +/* AP uses this during bootstrap. Do not staticize. */ +char *bootSTK; +int boot_cpuid; /* Hotwire a 0->4MB V==P mapping */ extern pt_entry_t *KPTphys; -/* Virtual address of per-cpu common_tss */ -extern struct i386tss common_tss; -#ifdef VM86 -extern struct segment_descriptor common_tssd; -extern u_int private_tss; /* flag indicating private tss */ -extern u_int my_tr; -#endif /* VM86 */ - -/* IdlePTD per cpu */ -pd_entry_t *IdlePTDS[NCPU]; - -/* "my" private page table page, for BSP init */ -extern pt_entry_t SMP_prvpt[]; - -/* Private page pointer to curcpu's PTD, used during BSP init */ -extern pd_entry_t *my_idlePTD; +/* SMP page table page */ +extern pt_entry_t *SMPpt; struct pcb stoppcbs[NCPU]; @@ -466,41 +453,42 @@ void init_secondary(void) { int gsel_tss; -#ifndef VM86 - u_int my_tr; -#endif + int x, myid = boot_cpuid; + + gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid]; + gdt_segs[GPROC0_SEL].ssd_base = + (int) &SMP_prvspace[myid].globaldata.gd_common_tss; + SMP_prvspace[myid].globaldata.gd_prvspace = &SMP_prvspace[myid]; - r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1; - r_gdt.rd_base = (int) gdt; + for (x = 0; x < NGDT; x++) { + ssdtosd(&gdt_segs[x], &gdt[myid * NGDT + x].sd); + } + + r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1; + r_gdt.rd_base = (int) &gdt[myid * NGDT]; lgdt(&r_gdt); /* does magic intra-segment return */ + lidt(&r_idt); + lldt(_default_ldt); #ifdef USER_LDT currentldt = _default_ldt; #endif - my_tr = NGDT + cpuid; - gsel_tss = GSEL(my_tr, SEL_KPL); - gdt[my_tr].sd.sd_type = SDT_SYS386TSS; + gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); + gdt[myid * NGDT + GPROC0_SEL].sd.sd_type = SDT_SYS386TSS; common_tss.tss_esp0 = 0; /* not used until after switch */ common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); common_tss.tss_ioopt = (sizeof common_tss) << 16; #ifdef VM86 - common_tssd = gdt[my_tr].sd; - private_tss = 0; -#endif /* VM86 */ + common_tssd = gdt[myid * NGDT + GPROC0_SEL].sd; +#endif ltr(gsel_tss); load_cr0(0x8005003b); /* XXX! */ - PTD[0] = 0; pmap_set_opt((unsigned *)PTD); -#if 0 - putmtrr(); - pmap_setvidram(); -#endif - invltlb(); } @@ -558,11 +546,6 @@ mp_enable(u_int boot_addr) u_int ux; #endif /* APIC_IO */ -#if 0 - getmtrr(); - pmap_setvidram(); -#endif - POSTCODE(MP_ENABLE_POST); /* turn on 4MB of V == P addressing so we can get to MP table */ @@ -1770,14 +1753,11 @@ extern int wait_ap(unsigned int); static int start_all_aps(u_int boot_addr) { - int x, i; + int x, i, pg; u_char mpbiosreason; u_long mpbioswarmvec; - pd_entry_t *newptd; - pt_entry_t *newpt; struct globaldata *gd; char *stack; - pd_entry_t *myPTD; POSTCODE(START_ALL_APS_POST); @@ -1799,70 +1779,46 @@ start_all_aps(u_int boot_addr) /* record BSP in CPU map */ all_cpus = 1; + /* set up 0 -> 4MB P==V mapping for AP boot */ + *(int *)PTD = PG_V | PG_RW | ((uintptr_t)(void *)KPTphys & PG_FRAME); + invltlb(); + /* start each AP */ for (x = 1; x <= mp_naps; ++x) { /* This is a bit verbose, it will go away soon. */ - /* alloc new page table directory */ - newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE)); - - /* Store the virtual PTD address for this CPU */ - IdlePTDS[x] = newptd; - - /* clone currently active one (ie: IdlePTD) */ - bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */ - - /* set up 0 -> 4MB P==V mapping for AP boot */ - newptd[0] = (void *)(uintptr_t)(PG_V | PG_RW | - ((uintptr_t)(void *)KPTphys & PG_FRAME)); - - /* store PTD for this AP's boot sequence */ - myPTD = (pd_entry_t *)vtophys(newptd); - - /* alloc new page table page */ - newpt = (pt_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE)); - - /* set the new PTD's private page to point there */ - newptd[MPPTDI] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt)); - - /* install self referential entry */ - newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd)); + /* first page of AP's private space */ + pg = x * i386_btop(sizeof(struct privatespace)); /* allocate a new private data page */ gd = (struct globaldata *)kmem_alloc(kernel_map, PAGE_SIZE); /* wire it into the private page table page */ - newpt[0] = (pt_entry_t)(PG_V | PG_RW | vtophys(gd)); - - /* wire the ptp into itself for access */ - newpt[1] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt)); - - /* copy in the pointer to the local apic */ - newpt[2] = SMP_prvpt[2]; - - /* and the IO apic mapping[s] */ - for (i = 16; i < 32; i++) - newpt[i] = SMP_prvpt[i]; + SMPpt[pg] = (pt_entry_t)(PG_V | PG_RW | vtophys(gd)); /* allocate and set up an idle stack data page */ stack = (char *)kmem_alloc(kernel_map, UPAGES*PAGE_SIZE); for (i = 0; i < UPAGES; i++) - newpt[i + 3] = (pt_entry_t)(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); + SMPpt[pg + 5 + i] = (pt_entry_t) + (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); - newpt[3 + UPAGES] = 0; /* *prv_CMAP1 */ - newpt[4 + UPAGES] = 0; /* *prv_CMAP2 */ - newpt[5 + UPAGES] = 0; /* *prv_CMAP3 */ - newpt[6 + UPAGES] = 0; /* *prv_PMAP1 */ + SMPpt[pg + 1] = 0; /* *prv_CMAP1 */ + SMPpt[pg + 2] = 0; /* *prv_CMAP2 */ + SMPpt[pg + 3] = 0; /* *prv_CMAP3 */ + SMPpt[pg + 4] = 0; /* *prv_PMAP1 */ /* prime data page for it to use */ - gd->cpuid = x; - gd->cpu_lockid = x << 24; - gd->my_idlePTD = myPTD; - gd->prv_CMAP1 = &newpt[3 + UPAGES]; - gd->prv_CMAP2 = &newpt[4 + UPAGES]; - gd->prv_CMAP3 = &newpt[5 + UPAGES]; - gd->prv_PMAP1 = &newpt[6 + UPAGES]; + gd->gd_cpuid = x; + gd->gd_cpu_lockid = x << 24; + gd->gd_prv_CMAP1 = &SMPpt[pg + 1]; + gd->gd_prv_CMAP2 = &SMPpt[pg + 2]; + gd->gd_prv_CMAP3 = &SMPpt[pg + 3]; + gd->gd_prv_PMAP1 = &SMPpt[pg + 4]; + gd->gd_prv_CADDR1 = SMP_prvspace[x].CPAGE1; + gd->gd_prv_CADDR2 = SMP_prvspace[x].CPAGE2; + gd->gd_prv_CADDR3 = SMP_prvspace[x].CPAGE3; + gd->gd_prv_PADDR1 = (unsigned *)SMP_prvspace[x].PPAGE1; /* setup a vector to our boot code */ *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; @@ -1872,7 +1828,9 @@ start_all_aps(u_int boot_addr) outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ #endif - bootPTD = myPTD; + bootSTK = &SMP_prvspace[x].idlestack[UPAGES*PAGE_SIZE]; + boot_cpuid = x; + /* attempt to start the Application Processor */ CHECK_INIT(99); /* setup checkpoints */ if (!start_ap(x, boot_addr)) { @@ -1910,27 +1868,16 @@ start_all_aps(u_int boot_addr) * because the BSP is cpu#0 and the page is initially zero, and also * because we can refer to variables by name on the BSP.. */ - newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE)); - - bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */ - IdlePTDS[0] = newptd; - - /* Point PTD[] to this page instead of IdlePTD's physical page */ - newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd)); - - my_idlePTD = (pd_entry_t *)vtophys(newptd); /* Allocate and setup BSP idle stack */ stack = (char *)kmem_alloc(kernel_map, UPAGES * PAGE_SIZE); for (i = 0; i < UPAGES; i++) - SMP_prvpt[i + 3] = (pt_entry_t)(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); + SMPpt[5 + i] = (pt_entry_t) + (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); + *(int *)PTD = 0; pmap_set_opt_bsp(); - for (i = 0; i < mp_ncpus; i++) { - bcopy( (int *) PTD + KPTDI, (int *) IdlePTDS[i] + KPTDI, NKPDE * sizeof (int)); - } - /* number of APs actually started */ return mp_ncpus - 1; } @@ -2250,10 +2197,6 @@ ap_init() panic("cpuid mismatch! boom!!"); } -#if 0 - getmtrr(); -#endif - /* Init local apic for irq's */ apic_initialize(); @@ -2267,8 +2210,6 @@ ap_init() smp_started = 1; /* enable IPI's, tlb shootdown, freezes etc */ smp_active = 1; /* historic */ } - - curproc = NULL; /* make sure */ } #ifdef BETTER_CLOCK diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index bca25cc..7aa92de 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -39,7 +39,7 @@ * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 - * $Id: pmap.c,v 1.232 1999/04/23 20:29:58 dt Exp $ + * $Id: pmap.c,v 1.233 1999/04/25 18:40:05 alc Exp $ */ /* @@ -71,6 +71,8 @@ #include "opt_disable_pse.h" #include "opt_pmap.h" #include "opt_msgbuf.h" +#include "opt_vm86.h" +#include "opt_user_ldt.h" #include <sys/param.h> #include <sys/systm.h> @@ -100,6 +102,9 @@ #if defined(SMP) || defined(APIC_IO) #include <machine/smp.h> #include <machine/apic.h> +#include <machine/segments.h> +#include <machine/tss.h> +#include <machine/globaldata.h> #endif /* SMP || APIC_IO */ #define PMAP_KEEP_PDIRS @@ -146,7 +151,6 @@ static int protection_codes[8]; static struct pmap kernel_pmap_store; pmap_t kernel_pmap; -extern pd_entry_t my_idlePTD; vm_offset_t avail_start; /* PA of first available physical page */ vm_offset_t avail_end; /* PA of last available physical page */ @@ -184,19 +188,8 @@ static caddr_t CADDR2; static pt_entry_t *msgbufmap; struct msgbuf *msgbufp=0; -/* AIO support */ -extern struct vmspace *aiovmspace; - -#ifdef SMP -extern char prv_CPAGE1[], prv_CPAGE2[], prv_CPAGE3[]; -extern pt_entry_t *prv_CMAP1, *prv_CMAP2, *prv_CMAP3; -extern pd_entry_t *IdlePTDS[]; -extern pt_entry_t SMP_prvpt[]; -#endif - #ifdef SMP -extern unsigned int prv_PPAGE1[]; -extern pt_entry_t *prv_PMAP1; +extern pt_entry_t *SMPpt; #else static pt_entry_t *PMAP1 = 0; static unsigned *PADDR1 = 0; @@ -294,6 +287,7 @@ pmap_bootstrap(firstaddr, loadaddr) pt_entry_t *pte; #ifdef SMP int i, j; + struct globaldata *gd; #endif avail_start = firstaddr; @@ -404,12 +398,17 @@ pmap_bootstrap(firstaddr, loadaddr) ptditmp &= ~(NBPDR - 1); ptditmp |= PG_V | PG_RW | PG_PS | PG_U | pgeflag; pdir4mb = ptditmp; + +#if !defined(SMP) /* * We can do the mapping here for the single processor * case. We simply ignore the old page table page from * now on. */ -#if !defined(SMP) + /* + * For SMP, we still need 4K pages to bootstrap APs, + * PSE will be enabled as soon as all APs are up. + */ PTD[KPTDI] = (pd_entry_t) ptditmp; kernel_pmap->pm_pdir[KPTDI] = (pd_entry_t) ptditmp; invltlb(); @@ -421,44 +420,46 @@ pmap_bootstrap(firstaddr, loadaddr) if (cpu_apic_address == 0) panic("pmap_bootstrap: no local apic!"); - /* 0 = private page */ - /* 1 = page table page */ - /* 2 = local apic */ - /* 16-31 = io apics */ - SMP_prvpt[2] = (pt_entry_t)(PG_V | PG_RW | pgeflag | + /* local apic is mapped on last page */ + SMPpt[NPTEPG - 1] = (pt_entry_t)(PG_V | PG_RW | PG_N | pgeflag | (cpu_apic_address & PG_FRAME)); for (i = 0; i < mp_napics; i++) { - for (j = 0; j < 16; j++) { + for (j = 0; j < mp_napics; j++) { /* same page frame as a previous IO apic? */ - if (((vm_offset_t)SMP_prvpt[j + 16] & PG_FRAME) == + if (((vm_offset_t)SMPpt[NPTEPG-2-j] & PG_FRAME) == (io_apic_address[0] & PG_FRAME)) { - ioapic[i] = (ioapic_t *)&SMP_ioapic[j * PAGE_SIZE]; + ioapic[i] = (ioapic_t *)((u_int)SMP_prvspace + + (NPTEPG-2-j)*PAGE_SIZE); break; } /* use this slot if available */ - if (((vm_offset_t)SMP_prvpt[j + 16] & PG_FRAME) == 0) { - SMP_prvpt[j + 16] = (pt_entry_t)(PG_V | PG_RW | + if (((vm_offset_t)SMPpt[NPTEPG-2-j] & PG_FRAME) == 0) { + SMPpt[NPTEPG-2-j] = (pt_entry_t)(PG_V | PG_RW | pgeflag | (io_apic_address[i] & PG_FRAME)); - ioapic[i] = (ioapic_t *)&SMP_ioapic[j * PAGE_SIZE]; + ioapic[i] = (ioapic_t *)((u_int)SMP_prvspace + + (NPTEPG-2-j)*PAGE_SIZE); break; } } - if (j == 16) - panic("no space to map IO apic %d!", i); } /* BSP does this itself, AP's get it pre-set */ - prv_CMAP1 = &SMP_prvpt[3 + UPAGES]; - prv_CMAP2 = &SMP_prvpt[4 + UPAGES]; - prv_CMAP3 = &SMP_prvpt[5 + UPAGES]; - prv_PMAP1 = &SMP_prvpt[6 + UPAGES]; + gd = &SMP_prvspace[0].globaldata; + gd->gd_prv_CMAP1 = &SMPpt[1]; + gd->gd_prv_CMAP2 = &SMPpt[2]; + gd->gd_prv_CMAP3 = &SMPpt[3]; + gd->gd_prv_PMAP1 = &SMPpt[4]; + gd->gd_prv_CADDR1 = SMP_prvspace[0].CPAGE1; + gd->gd_prv_CADDR2 = SMP_prvspace[0].CPAGE2; + gd->gd_prv_CADDR3 = SMP_prvspace[0].CPAGE3; + gd->gd_prv_PADDR1 = (unsigned *)SMP_prvspace[0].PPAGE1; #endif invltlb(); - } +#ifdef SMP /* * Set 4mb pdir for mp startup, and global flags */ @@ -493,6 +494,7 @@ pmap_set_opt_bsp(void) pmap_set_opt((unsigned *)PTD); invltlb(); } +#endif /* * Initialize the pmap module. @@ -716,9 +718,9 @@ pmap_pte_quick(pmap, va) #ifdef SMP if ( ((* (unsigned *) prv_PMAP1) & PG_FRAME) != newpf) { * (unsigned *) prv_PMAP1 = newpf | PG_RW | PG_V; - cpu_invlpg(&prv_PPAGE1); + cpu_invlpg(prv_PADDR1); } - return prv_PPAGE1 + ((unsigned) index & (NPTEPG - 1)); + return prv_PADDR1 + ((unsigned) index & (NPTEPG - 1)); #else if ( ((* (unsigned *) PMAP1) & PG_FRAME) != newpf) { * (unsigned *) PMAP1 = newpf | PG_RW | PG_V; @@ -1122,7 +1124,6 @@ pmap_unuse_pt(pmap, va, mpte) return pmap_unwire_pte_hold(pmap, mpte); } -#if !defined(SMP) void pmap_pinit0(pmap) struct pmap *pmap; @@ -1136,14 +1137,6 @@ pmap_pinit0(pmap) TAILQ_INIT(&pmap->pm_pvlist); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); } -#else -void -pmap_pinit0(pmap) - struct pmap *pmap; -{ - pmap_pinit(pmap); -} -#endif /* * Initialize a preallocated and zeroed pmap structure, @@ -1189,6 +1182,9 @@ pmap_pinit(pmap) /* wire in kernel global address entries */ /* XXX copies current process, does not fill in MPPTDI */ bcopy(PTD + KPTDI, pmap->pm_pdir + KPTDI, nkpt * PTESIZE); +#ifdef SMP + pmap->pm_pdir[MPPTDI] = PTD[MPPTDI]; +#endif /* install self-referential address mapping entry */ *(unsigned *) (pmap->pm_pdir + PTDPTDI) = @@ -1430,9 +1426,6 @@ pmap_growkernel(vm_offset_t addr) int s; vm_offset_t ptppaddr; vm_page_t nkpg; -#ifdef SMP - int i; -#endif pd_entry_t newpdir; s = splhigh(); @@ -1468,23 +1461,12 @@ pmap_growkernel(vm_offset_t addr) newpdir = (pd_entry_t) (ptppaddr | PG_V | PG_RW | PG_A | PG_M); pdir_pde(PTD, kernel_vm_end) = newpdir; -#ifdef SMP - for (i = 0; i < mp_ncpus; i++) { - if (IdlePTDS[i]) - pdir_pde(IdlePTDS[i], kernel_vm_end) = newpdir; - } -#endif - for (p = allproc.lh_first; p != 0; p = p->p_list.le_next) { if (p->p_vmspace) { pmap = vmspace_pmap(p->p_vmspace); *pmap_pde(pmap, kernel_vm_end) = newpdir; } } - if (aiovmspace != NULL) { - pmap = vmspace_pmap(aiovmspace); - *pmap_pde(pmap, kernel_vm_end) = newpdir; - } *pmap_pde(kernel_pmap, kernel_vm_end) = newpdir; kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); } @@ -2739,14 +2721,14 @@ pmap_zero_page(phys) #endif *(int *) prv_CMAP3 = PG_V | PG_RW | (phys & PG_FRAME) | PG_A | PG_M; - cpu_invlpg(&prv_CPAGE3); + cpu_invlpg(prv_CADDR3); #if defined(I686_CPU) if (cpu_class == CPUCLASS_686) - i686_pagezero(&prv_CPAGE3); + i686_pagezero(prv_CADDR3); else #endif - bzero(&prv_CPAGE3, PAGE_SIZE); + bzero(prv_CADDR3, PAGE_SIZE); *(int *) prv_CMAP3 = 0; #else @@ -2787,14 +2769,14 @@ pmap_zero_page_area(phys, off, size) #endif *(int *) prv_CMAP3 = PG_V | PG_RW | (phys & PG_FRAME) | PG_A | PG_M; - cpu_invlpg(&prv_CPAGE3); + cpu_invlpg(prv_CADDR3); #if defined(I686_CPU) if (cpu_class == CPUCLASS_686 && off == 0 && size == PAGE_SIZE) - i686_pagezero(&prv_CPAGE3); + i686_pagezero(prv_CADDR3); else #endif - bzero((char *)&prv_CPAGE3 + off, size); + bzero((char *)prv_CADDR3 + off, size); *(int *) prv_CMAP3 = 0; #else @@ -2838,10 +2820,10 @@ pmap_copy_page(src, dst) *(int *) prv_CMAP1 = PG_V | (src & PG_FRAME) | PG_A; *(int *) prv_CMAP2 = PG_V | PG_RW | (dst & PG_FRAME) | PG_A | PG_M; - cpu_invlpg(&prv_CPAGE1); - cpu_invlpg(&prv_CPAGE2); + cpu_invlpg(prv_CADDR1); + cpu_invlpg(prv_CADDR2); - bcopy(&prv_CPAGE1, &prv_CPAGE2, PAGE_SIZE); + bcopy(prv_CADDR1, prv_CADDR2, PAGE_SIZE); *(int *) prv_CMAP1 = 0; *(int *) prv_CMAP2 = 0; diff --git a/sys/amd64/amd64/support.S b/sys/amd64/amd64/support.S index cb9fd3e..7093dbb 100644 --- a/sys/amd64/amd64/support.S +++ b/sys/amd64/amd64/support.S @@ -30,9 +30,10 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: support.s,v 1.61 1999/03/21 12:30:50 phk Exp $ + * $Id: support.s,v 1.62 1999/03/30 09:00:40 phk Exp $ */ +#include "opt_smp.h" #include "npx.h" #include <machine/asmacros.h> @@ -42,8 +43,6 @@ #include "assym.s" -#define KDSEL 0x10 /* kernel data selector */ -#define KCSEL 0x8 /* kernel code selector */ #define IDXSHIFT 10 .data @@ -1495,9 +1494,12 @@ ENTRY(lgdt) movl $KDSEL,%eax movl %ax,%ds movl %ax,%es - movl %ax,%fs movl %ax,%gs movl %ax,%ss +#ifdef SMP + movl $KPSEL,%eax +#endif + movl %ax,%fs /* reload code selector by turning return into intersegmental return */ movl (%esp),%eax diff --git a/sys/amd64/amd64/support.s b/sys/amd64/amd64/support.s index cb9fd3e..7093dbb 100644 --- a/sys/amd64/amd64/support.s +++ b/sys/amd64/amd64/support.s @@ -30,9 +30,10 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: support.s,v 1.61 1999/03/21 12:30:50 phk Exp $ + * $Id: support.s,v 1.62 1999/03/30 09:00:40 phk Exp $ */ +#include "opt_smp.h" #include "npx.h" #include <machine/asmacros.h> @@ -42,8 +43,6 @@ #include "assym.s" -#define KDSEL 0x10 /* kernel data selector */ -#define KCSEL 0x8 /* kernel code selector */ #define IDXSHIFT 10 .data @@ -1495,9 +1494,12 @@ ENTRY(lgdt) movl $KDSEL,%eax movl %ax,%ds movl %ax,%es - movl %ax,%fs movl %ax,%gs movl %ax,%ss +#ifdef SMP + movl $KPSEL,%eax +#endif + movl %ax,%fs /* reload code selector by turning return into intersegmental return */ movl (%esp),%eax diff --git a/sys/amd64/amd64/swtch.s b/sys/amd64/amd64/swtch.s index 7e97a09..ae4ca62 100644 --- a/sys/amd64/amd64/swtch.s +++ b/sys/amd64/amd64/swtch.s @@ -33,7 +33,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: swtch.s,v 1.77 1999/03/20 18:44:13 alc Exp $ + * $Id: swtch.s,v 1.78 1999/04/02 17:59:39 alc Exp $ */ #include "npx.h" @@ -258,25 +258,32 @@ _idle: /* when called, we have the mplock, intr disabled */ /* use our idleproc's "context" */ - movl _my_idlePTD,%ecx - movl %ecx,%cr3 + movl _IdlePTD, %ecx + movl %cr3, %eax + cmpl %ecx, %eax + je 2f #if defined(SWTCH_OPTIM_STATS) + decl _swtch_optim_stats incl _tlb_flush_count #endif + movl %ecx, %cr3 +2: /* Keep space for nonexisting return addr, or profiling bombs */ - movl $_idlestack_top-4,%ecx - movl %ecx,%esp + movl $gd_idlestack_top-4, %ecx + addl %fs:0, %ecx + movl %ecx, %esp /* update common_tss.tss_esp0 pointer */ -#ifdef VM86 - movl _my_tr, %esi -#endif /* VM86 */ movl %ecx, _common_tss + TSS_ESP0 #ifdef VM86 - cmpl $0, _private_tss - je 1f - movl $_common_tssd, %edi + movl _cpuid, %esi + btrl %esi, _private_tss + jae 1f + + movl $GPROC0_SEL, %esi + movl $gd_common_tssd, %edi + addl %fs:0, %edi /* move correct tss descriptor into GDT slot, then reload tr */ leal _gdt(,%esi,8), %ebx /* entry in GDT */ @@ -388,14 +395,14 @@ idle_loop: #endif /* update common_tss.tss_esp0 pointer */ -#ifdef VM86 - movl _my_tr, %esi -#endif /* VM86 */ movl %esp, _common_tss + TSS_ESP0 #ifdef VM86 - cmpl $0, _private_tss - je 1f + movl $0, %esi + btrl %esi, _private_tss + jae 1f + + movl $GPROC0_SEL, %esi movl $_common_tssd, %edi /* move correct tss descriptor into GDT slot, then reload tr */ @@ -477,7 +484,6 @@ ENTRY(cpu_switch) movl %ebp,PCB_EBP(%edx) movl %esi,PCB_ESI(%edx) movl %edi,PCB_EDI(%edx) - movl %fs,PCB_FS(%edx) movl %gs,PCB_GS(%edx) #ifdef SMP @@ -610,70 +616,55 @@ swtch_com: movl %eax,P_BACK(%ecx) /* isolate process to run */ movl P_ADDR(%ecx),%edx -#ifdef SMP - movl PCB_CR3(%edx),%ebx - /* Grab the private PT pointer from the outgoing process's PTD */ - movl $_PTD, %esi - movl 4*MPPTDI(%esi), %eax /* fetch cpu's prv pt */ -#else #if defined(SWTCH_OPTIM_STATS) incl _swtch_optim_stats #endif /* switch address space */ movl %cr3,%ebx cmpl PCB_CR3(%edx),%ebx - je 4f + je 4f #if defined(SWTCH_OPTIM_STATS) decl _swtch_optim_stats incl _tlb_flush_count #endif movl PCB_CR3(%edx),%ebx -#endif /* SMP */ movl %ebx,%cr3 4: +#ifdef VM86 #ifdef SMP - /* Copy the private PT to the new process's PTD */ - /* XXX yuck, the _PTD changes when we switch, so we have to - * reload %cr3 after changing the address space. - * We need to fix this by storing a pointer to the virtual - * location of the per-process PTD in the PCB or something quick. - * Dereferencing proc->vm_map->pmap->p_pdir[] is painful in asm. - */ - movl %eax, 4*MPPTDI(%esi) /* restore cpu's prv page */ - -#if defined(SWTCH_OPTIM_STATS) - incl _tlb_flush_count + movl _cpuid, %esi +#else + xorl %esi, %esi #endif - /* XXX: we have just changed the page tables.. reload.. */ - movl %ebx, %cr3 -#endif /* SMP */ - -#ifdef VM86 - movl _my_tr, %esi cmpl $0, PCB_EXT(%edx) /* has pcb extension? */ je 1f - movl $1, _private_tss /* mark use of private tss */ + btsl %esi, _private_tss /* mark use of private tss */ movl PCB_EXT(%edx), %edi /* new tss descriptor */ jmp 2f 1: #endif /* update common_tss.tss_esp0 pointer */ - movl $_common_tss, %eax movl %edx, %ebx /* pcb */ #ifdef VM86 addl $(UPAGES * PAGE_SIZE - 16), %ebx #else addl $(UPAGES * PAGE_SIZE), %ebx #endif /* VM86 */ - movl %ebx, TSS_ESP0(%eax) + movl %ebx, _common_tss + TSS_ESP0 #ifdef VM86 - cmpl $0, _private_tss - je 3f + btrl %esi, _private_tss + jae 3f +#ifdef SMP + movl $gd_common_tssd, %edi + addl %fs:0, %edi +#else movl $_common_tssd, %edi +#endif 2: + movl $GPROC0_SEL, %esi /* move correct tss descriptor into GDT slot, then reload tr */ leal _gdt(,%esi,8), %ebx /* entry in GDT */ movl 0(%edi), %eax @@ -738,9 +729,6 @@ swtch_com: #endif /* This must be done after loading the user LDT. */ - .globl cpu_switch_load_fs -cpu_switch_load_fs: - movl PCB_FS(%edx),%fs .globl cpu_switch_load_gs cpu_switch_load_gs: movl PCB_GS(%edx),%gs @@ -791,7 +779,6 @@ ENTRY(savectx) movl %ebp,PCB_EBP(%ecx) movl %esi,PCB_ESI(%ecx) movl %edi,PCB_EDI(%ecx) - movl %fs,PCB_FS(%ecx) movl %gs,PCB_GS(%ecx) #if NNPX > 0 diff --git a/sys/amd64/amd64/sys_machdep.c b/sys/amd64/amd64/sys_machdep.c index 9c3df36..4c40ed4 100644 --- a/sys/amd64/amd64/sys_machdep.c +++ b/sys/amd64/amd64/sys_machdep.c @@ -31,12 +31,13 @@ * SUCH DAMAGE. * * from: @(#)sys_machdep.c 5.5 (Berkeley) 1/19/91 - * $Id: sys_machdep.c,v 1.39 1999/01/28 01:59:50 dillon Exp $ + * $Id: sys_machdep.c,v 1.40 1999/04/27 11:14:33 phk Exp $ * */ #include "opt_user_ldt.h" #include "opt_vm86.h" +#include "opt_smp.h" #include <sys/param.h> #include <sys/systm.h> @@ -54,6 +55,7 @@ #include <machine/cpu.h> #include <machine/pcb_ext.h> /* pcb.h included by sys/user.h */ #include <machine/sysarch.h> +#include <machine/smp.h> #include <vm/vm_kern.h> /* for kernel_map */ @@ -261,7 +263,11 @@ set_user_ldt(struct pcb *pcb) { gdt_segs[GUSERLDT_SEL].ssd_base = (unsigned)pcb->pcb_ldt; gdt_segs[GUSERLDT_SEL].ssd_limit = (pcb->pcb_ldt_len * sizeof(union descriptor)) - 1; +#ifdef SMP + ssdtosd(&gdt_segs[GUSERLDT_SEL], &gdt[cpuid * NGDT + GUSERLDT_SEL].sd); +#else ssdtosd(&gdt_segs[GUSERLDT_SEL], &gdt[GUSERLDT_SEL].sd); +#endif lldt(GSEL(GUSERLDT_SEL, SEL_KPL)); currentldt = GSEL(GUSERLDT_SEL, SEL_KPL); } diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c index d08306e..c0807b0 100644 --- a/sys/amd64/amd64/trap.c +++ b/sys/amd64/amd64/trap.c @@ -35,7 +35,7 @@ * SUCH DAMAGE. * * from: @(#)trap.c 7.4 (Berkeley) 5/13/91 - * $Id: trap.c,v 1.134 1999/03/09 20:20:09 phk Exp $ + * $Id: trap.c,v 1.135 1999/04/19 14:14:13 peter Exp $ */ /* @@ -101,8 +101,6 @@ #include "isa.h" #include "npx.h" -extern struct i386tss common_tss; - int (*pmath_emulate) __P((struct trapframe *)); extern void trap __P((struct trapframe frame)); @@ -480,11 +478,6 @@ kernel_trap: * (XXX) so that we can continue, and generate * a signal. */ - if (frame.tf_eip == (int)cpu_switch_load_fs) { - curpcb->pcb_fs = 0; - psignal(p, SIGBUS); - return; - } if (frame.tf_eip == (int)cpu_switch_load_gs) { curpcb->pcb_gs = 0; psignal(p, SIGBUS); @@ -496,6 +489,8 @@ kernel_trap: doreti_popl_ds_fault); MAYBE_DORETI_FAULT(doreti_popl_es, doreti_popl_es_fault); + MAYBE_DORETI_FAULT(doreti_popl_fs, + doreti_popl_fs_fault); if (curpcb && curpcb->pcb_onfault) { frame.tf_eip = (int)curpcb->pcb_onfault; return; diff --git a/sys/amd64/include/cpufunc.h b/sys/amd64/include/cpufunc.h index ce6bf05..c1f9f17 100644 --- a/sys/amd64/include/cpufunc.h +++ b/sys/amd64/include/cpufunc.h @@ -30,7 +30,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: cpufunc.h,v 1.84 1999/01/08 19:51:02 bde Exp $ + * $Id: cpufunc.h,v 1.85 1999/01/09 13:00:27 bde Exp $ */ /* @@ -426,6 +426,34 @@ wrmsr(u_int msr, u_int64_t newval) __asm __volatile(".byte 0x0f, 0x30" : : "A" (newval), "c" (msr)); } +static __inline u_int +rfs(void) +{ + u_int sel; + __asm __volatile("movl %%fs,%0" : "=r" (sel)); + return (sel); +} + +static __inline u_int +rgs(void) +{ + u_int sel; + __asm __volatile("movl %%gs,%0" : "=r" (sel)); + return (sel); +} + +static __inline void +load_fs(u_int sel) +{ + __asm __volatile("movl %0,%%fs" : : "r" (sel)); +} + +static __inline void +load_gs(u_int sel) +{ + __asm __volatile("movl %0,%%gs" : : "r" (sel)); +} + #else /* !__GNUC__ */ int breakpoint __P((void)); @@ -456,6 +484,10 @@ void setbits __P((volatile u_int *addr, u_int bits)); void wbinvd __P((void)); void write_eflags __P((u_int ef)); void wrmsr __P((u_int msr, u_int64_t newval)); +u_int rfs __P((void)); +u_int rgs __P((void)); +void load_fs __P((u_int sel)); +void load_gs __P((u_int sel)); #endif /* __GNUC__ */ diff --git a/sys/amd64/include/fpu.h b/sys/amd64/include/fpu.h index a7608d5..10384c6 100644 --- a/sys/amd64/include/fpu.h +++ b/sys/amd64/include/fpu.h @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)npx.h 5.3 (Berkeley) 1/18/91 - * $Id: npx.h,v 1.13 1997/06/22 16:03:50 peter Exp $ + * $Id: npx.h,v 1.14 1997/07/20 11:06:44 bde Exp $ */ /* @@ -45,6 +45,8 @@ #ifndef _MACHINE_NPX_H_ #define _MACHINE_NPX_H_ +#include <machine/globals.h> + /* Environment information of floating point unit */ struct env87 { long en_cw; /* control word (16bits) */ @@ -135,7 +137,9 @@ struct save87 { #endif #ifdef KERNEL +#ifndef npxproc extern struct proc *npxproc; +#endif int npxdna __P((void)); void npxexit __P((struct proc *p)); diff --git a/sys/amd64/include/frame.h b/sys/amd64/include/frame.h index 05092c2..7d70c14 100644 --- a/sys/amd64/include/frame.h +++ b/sys/amd64/include/frame.h @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)frame.h 5.2 (Berkeley) 1/18/91 - * $Id: frame.h,v 1.14 1997/02/22 09:34:38 peter Exp $ + * $Id: frame.h,v 1.15 1997/08/09 00:03:12 dyson Exp $ */ #ifndef _MACHINE_FRAME_H_ @@ -51,6 +51,7 @@ */ struct trapframe { + int tf_fs; int tf_es; int tf_ds; int tf_edi; @@ -75,6 +76,7 @@ struct trapframe { /* Superset of trap frame, for traps from virtual-8086 mode */ struct trapframe_vm86 { + int tf_fs; int tf_es; int tf_ds; int tf_edi; @@ -106,6 +108,7 @@ struct trapframe_vm86 { struct intrframe { int if_vec; int if_ppl; + int if_fs; int if_es; int if_ds; int if_edi; @@ -132,6 +135,7 @@ struct intrframe { struct clockframe { int cf_vec; int cf_ppl; + int cf_fs; int cf_es; int cf_ds; int cf_edi; diff --git a/sys/amd64/include/md_var.h b/sys/amd64/include/md_var.h index fd7bcfd..35b6545 100644 --- a/sys/amd64/include/md_var.h +++ b/sys/amd64/include/md_var.h @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: md_var.h,v 1.27 1998/10/30 05:41:15 msmith Exp $ + * $Id: md_var.h,v 1.28 1999/01/08 16:29:58 bde Exp $ */ #ifndef _MACHINE_MD_VAR_H_ @@ -69,7 +69,6 @@ void bcopyb __P((const void *from, void *to, size_t len)); void busdma_swi __P((void)); void cpu_halt __P((void)); void cpu_reset __P((void)); -void cpu_switch_load_fs __P((void)) __asm(__STRING(cpu_switch_load_fs)); void cpu_switch_load_gs __P((void)) __asm(__STRING(cpu_switch_load_gs)); void doreti_iret __P((void)) __asm(__STRING(doreti_iret)); void doreti_iret_fault __P((void)) __asm(__STRING(doreti_iret_fault)); @@ -77,6 +76,8 @@ void doreti_popl_ds __P((void)) __asm(__STRING(doreti_popl_ds)); void doreti_popl_ds_fault __P((void)) __asm(__STRING(doreti_popl_ds_fault)); void doreti_popl_es __P((void)) __asm(__STRING(doreti_popl_es)); void doreti_popl_es_fault __P((void)) __asm(__STRING(doreti_popl_es_fault)); +void doreti_popl_fs __P((void)) __asm(__STRING(doreti_popl_fs)); +void doreti_popl_fs_fault __P((void)) __asm(__STRING(doreti_popl_fs_fault)); int fill_fpregs __P((struct proc *, struct fpreg *)); int fill_regs __P((struct proc *p, struct reg *regs)); void fillw __P((int /*u_short*/ pat, void *base, size_t cnt)); diff --git a/sys/amd64/include/mptable.h b/sys/amd64/include/mptable.h index eca2601..dcca437 100644 --- a/sys/amd64/include/mptable.h +++ b/sys/amd64/include/mptable.h @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.96 1999/04/11 00:43:43 tegge Exp $ + * $Id: mp_machdep.c,v 1.97 1999/04/13 03:24:47 tegge Exp $ */ #include "opt_smp.h" @@ -297,28 +297,15 @@ int apic_id_to_logical[NAPICID]; /* Bitmap of all available CPUs */ u_int all_cpus; -/* AP uses this PTD during bootstrap. Do not staticize. */ -pd_entry_t *bootPTD; +/* AP uses this during bootstrap. Do not staticize. */ +char *bootSTK; +int boot_cpuid; /* Hotwire a 0->4MB V==P mapping */ extern pt_entry_t *KPTphys; -/* Virtual address of per-cpu common_tss */ -extern struct i386tss common_tss; -#ifdef VM86 -extern struct segment_descriptor common_tssd; -extern u_int private_tss; /* flag indicating private tss */ -extern u_int my_tr; -#endif /* VM86 */ - -/* IdlePTD per cpu */ -pd_entry_t *IdlePTDS[NCPU]; - -/* "my" private page table page, for BSP init */ -extern pt_entry_t SMP_prvpt[]; - -/* Private page pointer to curcpu's PTD, used during BSP init */ -extern pd_entry_t *my_idlePTD; +/* SMP page table page */ +extern pt_entry_t *SMPpt; struct pcb stoppcbs[NCPU]; @@ -466,41 +453,42 @@ void init_secondary(void) { int gsel_tss; -#ifndef VM86 - u_int my_tr; -#endif + int x, myid = boot_cpuid; + + gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid]; + gdt_segs[GPROC0_SEL].ssd_base = + (int) &SMP_prvspace[myid].globaldata.gd_common_tss; + SMP_prvspace[myid].globaldata.gd_prvspace = &SMP_prvspace[myid]; - r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1; - r_gdt.rd_base = (int) gdt; + for (x = 0; x < NGDT; x++) { + ssdtosd(&gdt_segs[x], &gdt[myid * NGDT + x].sd); + } + + r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1; + r_gdt.rd_base = (int) &gdt[myid * NGDT]; lgdt(&r_gdt); /* does magic intra-segment return */ + lidt(&r_idt); + lldt(_default_ldt); #ifdef USER_LDT currentldt = _default_ldt; #endif - my_tr = NGDT + cpuid; - gsel_tss = GSEL(my_tr, SEL_KPL); - gdt[my_tr].sd.sd_type = SDT_SYS386TSS; + gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); + gdt[myid * NGDT + GPROC0_SEL].sd.sd_type = SDT_SYS386TSS; common_tss.tss_esp0 = 0; /* not used until after switch */ common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); common_tss.tss_ioopt = (sizeof common_tss) << 16; #ifdef VM86 - common_tssd = gdt[my_tr].sd; - private_tss = 0; -#endif /* VM86 */ + common_tssd = gdt[myid * NGDT + GPROC0_SEL].sd; +#endif ltr(gsel_tss); load_cr0(0x8005003b); /* XXX! */ - PTD[0] = 0; pmap_set_opt((unsigned *)PTD); -#if 0 - putmtrr(); - pmap_setvidram(); -#endif - invltlb(); } @@ -558,11 +546,6 @@ mp_enable(u_int boot_addr) u_int ux; #endif /* APIC_IO */ -#if 0 - getmtrr(); - pmap_setvidram(); -#endif - POSTCODE(MP_ENABLE_POST); /* turn on 4MB of V == P addressing so we can get to MP table */ @@ -1770,14 +1753,11 @@ extern int wait_ap(unsigned int); static int start_all_aps(u_int boot_addr) { - int x, i; + int x, i, pg; u_char mpbiosreason; u_long mpbioswarmvec; - pd_entry_t *newptd; - pt_entry_t *newpt; struct globaldata *gd; char *stack; - pd_entry_t *myPTD; POSTCODE(START_ALL_APS_POST); @@ -1799,70 +1779,46 @@ start_all_aps(u_int boot_addr) /* record BSP in CPU map */ all_cpus = 1; + /* set up 0 -> 4MB P==V mapping for AP boot */ + *(int *)PTD = PG_V | PG_RW | ((uintptr_t)(void *)KPTphys & PG_FRAME); + invltlb(); + /* start each AP */ for (x = 1; x <= mp_naps; ++x) { /* This is a bit verbose, it will go away soon. */ - /* alloc new page table directory */ - newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE)); - - /* Store the virtual PTD address for this CPU */ - IdlePTDS[x] = newptd; - - /* clone currently active one (ie: IdlePTD) */ - bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */ - - /* set up 0 -> 4MB P==V mapping for AP boot */ - newptd[0] = (void *)(uintptr_t)(PG_V | PG_RW | - ((uintptr_t)(void *)KPTphys & PG_FRAME)); - - /* store PTD for this AP's boot sequence */ - myPTD = (pd_entry_t *)vtophys(newptd); - - /* alloc new page table page */ - newpt = (pt_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE)); - - /* set the new PTD's private page to point there */ - newptd[MPPTDI] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt)); - - /* install self referential entry */ - newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd)); + /* first page of AP's private space */ + pg = x * i386_btop(sizeof(struct privatespace)); /* allocate a new private data page */ gd = (struct globaldata *)kmem_alloc(kernel_map, PAGE_SIZE); /* wire it into the private page table page */ - newpt[0] = (pt_entry_t)(PG_V | PG_RW | vtophys(gd)); - - /* wire the ptp into itself for access */ - newpt[1] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt)); - - /* copy in the pointer to the local apic */ - newpt[2] = SMP_prvpt[2]; - - /* and the IO apic mapping[s] */ - for (i = 16; i < 32; i++) - newpt[i] = SMP_prvpt[i]; + SMPpt[pg] = (pt_entry_t)(PG_V | PG_RW | vtophys(gd)); /* allocate and set up an idle stack data page */ stack = (char *)kmem_alloc(kernel_map, UPAGES*PAGE_SIZE); for (i = 0; i < UPAGES; i++) - newpt[i + 3] = (pt_entry_t)(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); + SMPpt[pg + 5 + i] = (pt_entry_t) + (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); - newpt[3 + UPAGES] = 0; /* *prv_CMAP1 */ - newpt[4 + UPAGES] = 0; /* *prv_CMAP2 */ - newpt[5 + UPAGES] = 0; /* *prv_CMAP3 */ - newpt[6 + UPAGES] = 0; /* *prv_PMAP1 */ + SMPpt[pg + 1] = 0; /* *prv_CMAP1 */ + SMPpt[pg + 2] = 0; /* *prv_CMAP2 */ + SMPpt[pg + 3] = 0; /* *prv_CMAP3 */ + SMPpt[pg + 4] = 0; /* *prv_PMAP1 */ /* prime data page for it to use */ - gd->cpuid = x; - gd->cpu_lockid = x << 24; - gd->my_idlePTD = myPTD; - gd->prv_CMAP1 = &newpt[3 + UPAGES]; - gd->prv_CMAP2 = &newpt[4 + UPAGES]; - gd->prv_CMAP3 = &newpt[5 + UPAGES]; - gd->prv_PMAP1 = &newpt[6 + UPAGES]; + gd->gd_cpuid = x; + gd->gd_cpu_lockid = x << 24; + gd->gd_prv_CMAP1 = &SMPpt[pg + 1]; + gd->gd_prv_CMAP2 = &SMPpt[pg + 2]; + gd->gd_prv_CMAP3 = &SMPpt[pg + 3]; + gd->gd_prv_PMAP1 = &SMPpt[pg + 4]; + gd->gd_prv_CADDR1 = SMP_prvspace[x].CPAGE1; + gd->gd_prv_CADDR2 = SMP_prvspace[x].CPAGE2; + gd->gd_prv_CADDR3 = SMP_prvspace[x].CPAGE3; + gd->gd_prv_PADDR1 = (unsigned *)SMP_prvspace[x].PPAGE1; /* setup a vector to our boot code */ *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; @@ -1872,7 +1828,9 @@ start_all_aps(u_int boot_addr) outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ #endif - bootPTD = myPTD; + bootSTK = &SMP_prvspace[x].idlestack[UPAGES*PAGE_SIZE]; + boot_cpuid = x; + /* attempt to start the Application Processor */ CHECK_INIT(99); /* setup checkpoints */ if (!start_ap(x, boot_addr)) { @@ -1910,27 +1868,16 @@ start_all_aps(u_int boot_addr) * because the BSP is cpu#0 and the page is initially zero, and also * because we can refer to variables by name on the BSP.. */ - newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE)); - - bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */ - IdlePTDS[0] = newptd; - - /* Point PTD[] to this page instead of IdlePTD's physical page */ - newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd)); - - my_idlePTD = (pd_entry_t *)vtophys(newptd); /* Allocate and setup BSP idle stack */ stack = (char *)kmem_alloc(kernel_map, UPAGES * PAGE_SIZE); for (i = 0; i < UPAGES; i++) - SMP_prvpt[i + 3] = (pt_entry_t)(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); + SMPpt[5 + i] = (pt_entry_t) + (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); + *(int *)PTD = 0; pmap_set_opt_bsp(); - for (i = 0; i < mp_ncpus; i++) { - bcopy( (int *) PTD + KPTDI, (int *) IdlePTDS[i] + KPTDI, NKPDE * sizeof (int)); - } - /* number of APs actually started */ return mp_ncpus - 1; } @@ -2250,10 +2197,6 @@ ap_init() panic("cpuid mismatch! boom!!"); } -#if 0 - getmtrr(); -#endif - /* Init local apic for irq's */ apic_initialize(); @@ -2267,8 +2210,6 @@ ap_init() smp_started = 1; /* enable IPI's, tlb shootdown, freezes etc */ smp_active = 1; /* historic */ } - - curproc = NULL; /* make sure */ } #ifdef BETTER_CLOCK diff --git a/sys/amd64/include/npx.h b/sys/amd64/include/npx.h index a7608d5..10384c6 100644 --- a/sys/amd64/include/npx.h +++ b/sys/amd64/include/npx.h @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)npx.h 5.3 (Berkeley) 1/18/91 - * $Id: npx.h,v 1.13 1997/06/22 16:03:50 peter Exp $ + * $Id: npx.h,v 1.14 1997/07/20 11:06:44 bde Exp $ */ /* @@ -45,6 +45,8 @@ #ifndef _MACHINE_NPX_H_ #define _MACHINE_NPX_H_ +#include <machine/globals.h> + /* Environment information of floating point unit */ struct env87 { long en_cw; /* control word (16bits) */ @@ -135,7 +137,9 @@ struct save87 { #endif #ifdef KERNEL +#ifndef npxproc extern struct proc *npxproc; +#endif int npxdna __P((void)); void npxexit __P((struct proc *p)); diff --git a/sys/amd64/include/pcb.h b/sys/amd64/include/pcb.h index d76217f..9ab0856 100644 --- a/sys/amd64/include/pcb.h +++ b/sys/amd64/include/pcb.h @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)pcb.h 5.10 (Berkeley) 5/12/91 - * $Id: pcb.h,v 1.25 1997/10/10 12:40:09 peter Exp $ + * $Id: pcb.h,v 1.26 1998/02/03 21:27:50 bde Exp $ */ #ifndef _I386_PCB_H_ @@ -43,6 +43,7 @@ /* * Intel 386 process control block */ +#include <machine/globals.h> #include <machine/npx.h> struct pcb { @@ -64,14 +65,13 @@ struct pcb { #else u_long pcb_mpnest_dontuse; #endif - int pcb_fs; int pcb_gs; #ifdef VM86 struct pcb_ext *pcb_ext; /* optional pcb extension */ #else struct pcb_ext *pcb_ext_dontuse; #endif - u_long __pcb_spare[1]; /* adjust to avoid core dump size changes */ + u_long __pcb_spare[2]; /* adjust to avoid core dump size changes */ }; /* @@ -83,7 +83,9 @@ struct md_coredump { #ifdef KERNEL +#ifndef curpcb extern struct pcb *curpcb; /* our current running pcb */ +#endif void savectx __P((struct pcb *)); #endif diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h index ec58799..f1d4fdd 100644 --- a/sys/amd64/include/pcpu.h +++ b/sys/amd64/include/pcpu.h @@ -23,7 +23,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: globaldata.h,v 1.6 1998/08/18 07:47:12 msmith Exp $ + * $Id: globaldata.h,v 1.7 1999/02/22 15:13:34 bde Exp $ */ /* @@ -39,31 +39,33 @@ * other processors" */ struct globaldata { - struct proc *curproc; - struct proc *npxproc; - struct pcb *curpcb; - struct i386tss common_tss; - struct timeval switchtime; - int switchticks; + struct privatespace *gd_prvspace; /* self-reference */ + struct proc *gd_curproc; + struct proc *gd_npxproc; + struct pcb *gd_curpcb; + struct timeval gd_switchtime; + struct i386tss gd_common_tss; + int gd_switchticks; #ifdef VM86 - struct segment_descriptor common_tssd; - u_int private_tss; - u_int my_tr; + struct segment_descriptor gd_common_tssd; #endif #ifdef USER_LDT - int currentldt; + int gd_currentldt; #endif #ifdef SMP - u_int cpuid; - u_int cpu_lockid; - u_int other_cpus; - pd_entry_t *my_idlePTD; - u_int ss_eflags; - pt_entry_t *prv_CMAP1; - pt_entry_t *prv_CMAP2; - pt_entry_t *prv_CMAP3; - pt_entry_t *prv_PMAP1; - int inside_intr; + u_int gd_cpuid; + u_int gd_cpu_lockid; + u_int gd_other_cpus; + int gd_inside_intr; + u_int gd_ss_eflags; + pt_entry_t *gd_prv_CMAP1; + pt_entry_t *gd_prv_CMAP2; + pt_entry_t *gd_prv_CMAP3; + pt_entry_t *gd_prv_PMAP1; + caddr_t gd_prv_CADDR1; + caddr_t gd_prv_CADDR2; + caddr_t gd_prv_CADDR3; + unsigned *gd_prv_PADDR1; #endif }; @@ -78,28 +80,16 @@ struct privatespace { struct globaldata globaldata; char __filler0[PAGE_SIZE - sizeof(struct globaldata)]; - /* page 1 - page table page */ - pt_entry_t prvpt[NPTEPG]; - - /* page 2 - local apic mapping */ - lapic_t lapic; - char __filler1[PAGE_SIZE - sizeof(lapic_t)]; - - /* page 3..2+UPAGES - idle stack (UPAGES pages) */ - char idlestack[UPAGES * PAGE_SIZE]; - - /* page 3+UPAGES..6+UPAGES - CPAGE1,CPAGE2,CPAGE3,PPAGE1 */ + /* page 1..4 - CPAGE1,CPAGE2,CPAGE3,PPAGE1 */ char CPAGE1[PAGE_SIZE]; char CPAGE2[PAGE_SIZE]; char CPAGE3[PAGE_SIZE]; char PPAGE1[PAGE_SIZE]; - /* page 7+UPAGES..15 - spare, unmapped */ - char __filler2[(9-UPAGES) * PAGE_SIZE]; + /* page 5..4+UPAGES - idle stack (UPAGES pages) */ + char idlestack[UPAGES * PAGE_SIZE]; +}; - /* page 16-31 - space for IO apics */ - char ioapics[16 * PAGE_SIZE]; +extern struct privatespace SMP_prvspace[]; - /* page 32-47 - maybe other cpu's globaldata pages? */ -}; #endif diff --git a/sys/amd64/include/proc.h b/sys/amd64/include/proc.h index e7a4744..ae6cd63 100644 --- a/sys/amd64/include/proc.h +++ b/sys/amd64/include/proc.h @@ -31,12 +31,14 @@ * SUCH DAMAGE. * * from: @(#)proc.h 7.1 (Berkeley) 5/15/91 - * $Id: proc.h,v 1.7 1997/02/22 09:34:59 peter Exp $ + * $Id: proc.h,v 1.8 1997/05/07 19:55:13 peter Exp $ */ #ifndef _MACHINE_PROC_H_ #define _MACHINE_PROC_H_ +#include <machine/globals.h> + /* * Machine-dependent part of the proc structure for i386. */ diff --git a/sys/amd64/include/reg.h b/sys/amd64/include/reg.h index 28466a8..2470116 100644 --- a/sys/amd64/include/reg.h +++ b/sys/amd64/include/reg.h @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)reg.h 5.5 (Berkeley) 1/18/91 - * $Id: reg.h,v 1.16 1998/09/14 22:43:40 jdp Exp $ + * $Id: reg.h,v 1.17 1999/04/03 22:19:59 jdp Exp $ */ #ifndef _MACHINE_REG_H_ @@ -51,22 +51,23 @@ * stopped accessing the registers in the trap frame via PT_{READ,WRITE}_U * and we can stop supporting the user area soon. */ -#define tES (0) -#define tDS (1) -#define tEDI (2) -#define tESI (3) -#define tEBP (4) -#define tISP (5) -#define tEBX (6) -#define tEDX (7) -#define tECX (8) -#define tEAX (9) -#define tERR (11) -#define tEIP (12) -#define tCS (13) -#define tEFLAGS (14) -#define tESP (15) -#define tSS (16) +#define tFS (0) +#define tES (1) +#define tDS (2) +#define tEDI (3) +#define tESI (4) +#define tEBP (5) +#define tISP (6) +#define tEBX (7) +#define tEDX (8) +#define tECX (9) +#define tEAX (10) +#define tERR (12) +#define tEIP (13) +#define tCS (14) +#define tEFLAGS (15) +#define tESP (16) +#define tSS (17) /* * Indices for registers in `struct regs' only. @@ -75,13 +76,13 @@ * other registers in application interfaces that copy all the registers * to or from a `struct regs'. */ -#define tFS (17) #define tGS (18) /* * Register set accessible via /proc/$pid/regs and PT_{SET,GET}REGS. */ struct reg { + unsigned int r_fs; unsigned int r_es; unsigned int r_ds; unsigned int r_edi; @@ -99,7 +100,6 @@ struct reg { unsigned int r_eflags; unsigned int r_esp; unsigned int r_ss; - unsigned int r_fs; unsigned int r_gs; }; diff --git a/sys/amd64/include/segments.h b/sys/amd64/include/segments.h index 674b0f3..eaecfa3 100644 --- a/sys/amd64/include/segments.h +++ b/sys/amd64/include/segments.h @@ -35,12 +35,14 @@ * SUCH DAMAGE. * * from: @(#)segments.h 7.1 (Berkeley) 5/9/91 - * $Id: segments.h,v 1.17 1997/08/21 06:32:49 charnier Exp $ + * $Id: segments.h,v 1.18 1999/01/28 11:45:49 newton Exp $ */ #ifndef _MACHINE_SEGMENTS_H_ #define _MACHINE_SEGMENTS_H_ +#include <machine/globals.h> + /* * 386 Segmentation Data Structures and definitions * William F. Jolitz (william@ernie.berkeley.edu) 6/20/1989 @@ -207,19 +209,20 @@ struct region_descriptor { #define GNULL_SEL 0 /* Null Descriptor */ #define GCODE_SEL 1 /* Kernel Code Descriptor */ #define GDATA_SEL 2 /* Kernel Data Descriptor */ -#define GLDT_SEL 3 /* LDT - eventually one per process */ -#define GTGATE_SEL 4 /* Process task switch gate */ -#define GPANIC_SEL 5 /* Task state to consider panic from */ -#define GPROC0_SEL 6 /* Task state process slot zero and up */ -#define GUSERLDT_SEL 7 /* User LDT */ -#define GAPMCODE32_SEL 8 /* APM BIOS 32-bit interface (32bit Code) */ -#define GAPMCODE16_SEL 9 /* APM BIOS 32-bit interface (16bit Code) */ -#define GAPMDATA_SEL 10 /* APM BIOS 32-bit interface (Data) */ +#define GPRIV_SEL 3 /* SMP Per-Processor Private Data */ +#define GPROC0_SEL 4 /* Task state process slot zero and up */ +#define GLDT_SEL 5 /* LDT - eventually one per process */ +#define GUSERLDT_SEL 6 /* User LDT */ +#define GTGATE_SEL 7 /* Process task switch gate */ +#define GPANIC_SEL 8 /* Task state to consider panic from */ +#define GAPMCODE32_SEL 9 /* APM BIOS 32-bit interface (32bit Code) */ +#define GAPMCODE16_SEL 10 /* APM BIOS 32-bit interface (16bit Code) */ +#define GAPMDATA_SEL 11 /* APM BIOS 32-bit interface (Data) */ #ifdef BDE_DEBUGGER #define NGDT 18 /* some of 11-17 are reserved for debugger */ #else -#define NGDT (GAPMDATA_SEL + 1) +#define NGDT 12 #endif /* @@ -237,7 +240,9 @@ struct region_descriptor { #define NLDT (LBSDICALLS_SEL + 1) #ifdef KERNEL +#ifndef currentldt extern int currentldt; +#endif extern int _default_ldt; extern union descriptor gdt[]; extern struct soft_segment_descriptor gdt_segs[]; diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h index 0f4f915..3750c72 100644 --- a/sys/amd64/include/smp.h +++ b/sys/amd64/include/smp.h @@ -6,7 +6,7 @@ * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp * ---------------------------------------------------------------------------- * - * $Id: smp.h,v 1.43 1998/05/17 22:12:05 tegge Exp $ + * $Id: smp.h,v 1.44 1998/09/06 22:41:40 tegge Exp $ * */ @@ -112,7 +112,6 @@ struct apic_intmapinfo { }; extern struct apic_intmapinfo int_to_apicintpin[]; extern u_int all_cpus; -extern u_char SMP_ioapic[]; extern struct pcb stoppcbs[]; /* functions in mp_machdep.c */ @@ -175,12 +174,6 @@ extern int invltlb_ok; extern int smp_active; extern volatile int smp_idle_loops; -/* 'private' global data in locore.s */ -extern volatile u_int cpuid; -extern volatile u_int cpu_lockid; -extern int inside_intr; -extern volatile u_int other_cpus; - #endif /* !LOCORE */ #endif /* SMP || APIC_IO */ #endif /* KERNEL */ diff --git a/sys/amd64/include/tss.h b/sys/amd64/include/tss.h index 565defa..636133a 100644 --- a/sys/amd64/include/tss.h +++ b/sys/amd64/include/tss.h @@ -34,12 +34,14 @@ * SUCH DAMAGE. * * from: @(#)tss.h 5.4 (Berkeley) 1/18/91 - * $Id$ + * $Id: tss.h,v 1.8 1997/02/22 09:35:20 peter Exp $ */ #ifndef _MACHINE_TSS_H_ #define _MACHINE_TSS_H_ 1 +#include <machine/globals.h> + /* * Intel 386 Context Data Type */ @@ -79,4 +81,11 @@ struct i386tss { int tss_ioopt; /* options & io offset bitmap: currently zero */ /* XXX unimplemented .. i/o permission bitmap */ }; + +#ifdef KERNEL +#ifndef common_tss +extern struct i386tss common_tss; +#endif +#endif + #endif /* _MACHINE_TSS_H_ */ diff --git a/sys/amd64/isa/atpic_vector.S b/sys/amd64/isa/atpic_vector.S index 2dcda02..4460303 100644 --- a/sys/amd64/isa/atpic_vector.S +++ b/sys/amd64/isa/atpic_vector.S @@ -1,6 +1,6 @@ /* * from: vector.s, 386BSD 0.1 unknown origin - * $Id: icu_vector.s,v 1.9 1998/08/11 17:01:32 bde Exp $ + * $Id: icu_vector.s,v 1.10 1999/04/14 14:26:36 bde Exp $ */ /* @@ -94,11 +94,13 @@ IDTVEC(vec_name) ; \ pushal ; /* build fat frame (grrr) ... */ \ pushl %ecx ; /* ... actually %ds ... */ \ pushl %es ; \ + pushl %fs ; \ movl $KDSEL,%eax ; \ movl %ax,%es ; \ - movl (2+8+0)*4(%esp),%ecx ; /* ... %ecx from thin frame ... */ \ - movl %ecx,(2+6)*4(%esp) ; /* ... to fat frame ... */ \ - movl (2+8+1)*4(%esp),%eax ; /* ... cpl from thin frame */ \ + movl %ax,%fs ; \ + movl (3+8+0)*4(%esp),%ecx ; /* ... %ecx from thin frame ... */ \ + movl %ecx,(3+6)*4(%esp) ; /* ... to fat frame ... */ \ + movl (3+8+1)*4(%esp),%eax ; /* ... cpl from thin frame */ \ pushl %eax ; \ subl $4,%esp ; /* junk for unit number */ \ MEXITCOUNT ; \ @@ -113,9 +115,11 @@ IDTVEC(vec_name) ; \ pushal ; \ pushl %ds ; /* save our data and extra segments ... */ \ pushl %es ; \ + pushl %fs ; \ movl $KDSEL,%eax ; /* ... and reload with kernel's own ... */ \ movl %ax,%ds ; /* ... early for obsolete reasons */ \ movl %ax,%es ; \ + movl %ax,%fs ; \ movb _imen + IRQ_BYTE(irq_num),%al ; \ orb $IRQ_BIT(irq_num),%al ; \ movb %al,_imen + IRQ_BYTE(irq_num) ; \ @@ -126,7 +130,7 @@ IDTVEC(vec_name) ; \ jne 2f ; \ incb _intr_nesting_level ; \ __CONCAT(Xresume,irq_num): ; \ - FAKE_MCOUNT(12*4(%esp)) ; /* XXX late to avoid double count */ \ + FAKE_MCOUNT(13*4(%esp)) ; /* XXX late to avoid double count */ \ incl _cnt+V_INTR ; /* tally interrupts */ \ movl _intr_countp + (irq_num) * 4,%eax ; \ incl (%eax) ; \ @@ -152,6 +156,7 @@ __CONCAT(Xresume,irq_num): ; \ 2: ; \ /* XXX skip mcounting here to avoid double count */ \ orb $IRQ_BIT(irq_num),_ipending + IRQ_BYTE(irq_num) ; \ + popl %fs ; \ popl %es ; \ popl %ds ; \ popal ; \ diff --git a/sys/amd64/isa/icu_vector.S b/sys/amd64/isa/icu_vector.S index 2dcda02..4460303 100644 --- a/sys/amd64/isa/icu_vector.S +++ b/sys/amd64/isa/icu_vector.S @@ -1,6 +1,6 @@ /* * from: vector.s, 386BSD 0.1 unknown origin - * $Id: icu_vector.s,v 1.9 1998/08/11 17:01:32 bde Exp $ + * $Id: icu_vector.s,v 1.10 1999/04/14 14:26:36 bde Exp $ */ /* @@ -94,11 +94,13 @@ IDTVEC(vec_name) ; \ pushal ; /* build fat frame (grrr) ... */ \ pushl %ecx ; /* ... actually %ds ... */ \ pushl %es ; \ + pushl %fs ; \ movl $KDSEL,%eax ; \ movl %ax,%es ; \ - movl (2+8+0)*4(%esp),%ecx ; /* ... %ecx from thin frame ... */ \ - movl %ecx,(2+6)*4(%esp) ; /* ... to fat frame ... */ \ - movl (2+8+1)*4(%esp),%eax ; /* ... cpl from thin frame */ \ + movl %ax,%fs ; \ + movl (3+8+0)*4(%esp),%ecx ; /* ... %ecx from thin frame ... */ \ + movl %ecx,(3+6)*4(%esp) ; /* ... to fat frame ... */ \ + movl (3+8+1)*4(%esp),%eax ; /* ... cpl from thin frame */ \ pushl %eax ; \ subl $4,%esp ; /* junk for unit number */ \ MEXITCOUNT ; \ @@ -113,9 +115,11 @@ IDTVEC(vec_name) ; \ pushal ; \ pushl %ds ; /* save our data and extra segments ... */ \ pushl %es ; \ + pushl %fs ; \ movl $KDSEL,%eax ; /* ... and reload with kernel's own ... */ \ movl %ax,%ds ; /* ... early for obsolete reasons */ \ movl %ax,%es ; \ + movl %ax,%fs ; \ movb _imen + IRQ_BYTE(irq_num),%al ; \ orb $IRQ_BIT(irq_num),%al ; \ movb %al,_imen + IRQ_BYTE(irq_num) ; \ @@ -126,7 +130,7 @@ IDTVEC(vec_name) ; \ jne 2f ; \ incb _intr_nesting_level ; \ __CONCAT(Xresume,irq_num): ; \ - FAKE_MCOUNT(12*4(%esp)) ; /* XXX late to avoid double count */ \ + FAKE_MCOUNT(13*4(%esp)) ; /* XXX late to avoid double count */ \ incl _cnt+V_INTR ; /* tally interrupts */ \ movl _intr_countp + (irq_num) * 4,%eax ; \ incl (%eax) ; \ @@ -152,6 +156,7 @@ __CONCAT(Xresume,irq_num): ; \ 2: ; \ /* XXX skip mcounting here to avoid double count */ \ orb $IRQ_BIT(irq_num),_ipending + IRQ_BYTE(irq_num) ; \ + popl %fs ; \ popl %es ; \ popl %ds ; \ popal ; \ diff --git a/sys/amd64/isa/icu_vector.s b/sys/amd64/isa/icu_vector.s index 2dcda02..4460303 100644 --- a/sys/amd64/isa/icu_vector.s +++ b/sys/amd64/isa/icu_vector.s @@ -1,6 +1,6 @@ /* * from: vector.s, 386BSD 0.1 unknown origin - * $Id: icu_vector.s,v 1.9 1998/08/11 17:01:32 bde Exp $ + * $Id: icu_vector.s,v 1.10 1999/04/14 14:26:36 bde Exp $ */ /* @@ -94,11 +94,13 @@ IDTVEC(vec_name) ; \ pushal ; /* build fat frame (grrr) ... */ \ pushl %ecx ; /* ... actually %ds ... */ \ pushl %es ; \ + pushl %fs ; \ movl $KDSEL,%eax ; \ movl %ax,%es ; \ - movl (2+8+0)*4(%esp),%ecx ; /* ... %ecx from thin frame ... */ \ - movl %ecx,(2+6)*4(%esp) ; /* ... to fat frame ... */ \ - movl (2+8+1)*4(%esp),%eax ; /* ... cpl from thin frame */ \ + movl %ax,%fs ; \ + movl (3+8+0)*4(%esp),%ecx ; /* ... %ecx from thin frame ... */ \ + movl %ecx,(3+6)*4(%esp) ; /* ... to fat frame ... */ \ + movl (3+8+1)*4(%esp),%eax ; /* ... cpl from thin frame */ \ pushl %eax ; \ subl $4,%esp ; /* junk for unit number */ \ MEXITCOUNT ; \ @@ -113,9 +115,11 @@ IDTVEC(vec_name) ; \ pushal ; \ pushl %ds ; /* save our data and extra segments ... */ \ pushl %es ; \ + pushl %fs ; \ movl $KDSEL,%eax ; /* ... and reload with kernel's own ... */ \ movl %ax,%ds ; /* ... early for obsolete reasons */ \ movl %ax,%es ; \ + movl %ax,%fs ; \ movb _imen + IRQ_BYTE(irq_num),%al ; \ orb $IRQ_BIT(irq_num),%al ; \ movb %al,_imen + IRQ_BYTE(irq_num) ; \ @@ -126,7 +130,7 @@ IDTVEC(vec_name) ; \ jne 2f ; \ incb _intr_nesting_level ; \ __CONCAT(Xresume,irq_num): ; \ - FAKE_MCOUNT(12*4(%esp)) ; /* XXX late to avoid double count */ \ + FAKE_MCOUNT(13*4(%esp)) ; /* XXX late to avoid double count */ \ incl _cnt+V_INTR ; /* tally interrupts */ \ movl _intr_countp + (irq_num) * 4,%eax ; \ incl (%eax) ; \ @@ -152,6 +156,7 @@ __CONCAT(Xresume,irq_num): ; \ 2: ; \ /* XXX skip mcounting here to avoid double count */ \ orb $IRQ_BIT(irq_num),_ipending + IRQ_BYTE(irq_num) ; \ + popl %fs ; \ popl %es ; \ popl %ds ; \ popal ; \ |
