diff options
author | luoqi <luoqi@FreeBSD.org> | 1999-04-28 01:04:33 +0000 |
---|---|---|
committer | luoqi <luoqi@FreeBSD.org> | 1999-04-28 01:04:33 +0000 |
commit | af7e9be5cce9a2ceb819f00b3f58014d23ab57cd (patch) | |
tree | 0e70c0e764f072d43041acca8ac52e30616dced3 /sys/kern | |
parent | 0f4a245030e7f1acb03f8de0822a58b188fc0d28 (diff) | |
download | FreeBSD-src-af7e9be5cce9a2ceb819f00b3f58014d23ab57cd.zip FreeBSD-src-af7e9be5cce9a2ceb819f00b3f58014d23ab57cd.tar.gz |
Enable vmspace sharing on SMP. Major changes are,
- %fs register is added to trapframe and saved/restored upon kernel entry/exit.
- Per-cpu pages are no longer mapped at the same virtual address.
- Each cpu now has a separate gdt selector table. A new segment selector
is added to point to per-cpu pages, per-cpu global variables are now
accessed through this new selector (%fs). The selectors in gdt table are
rearranged for cache line optimization.
- fask_vfork is now on as default for both UP and SMP.
- Some aio code cleanup.
Reviewed by: Alan Cox <alc@cs.rice.edu>
John Dyson <dyson@iquest.net>
Julian Elischer <julian@whistel.com>
Bruce Evans <bde@zeta.org.au>
David Greenman <dg@root.com>
Diffstat (limited to 'sys/kern')
-rw-r--r-- | sys/kern/init_main.c | 5 | ||||
-rw-r--r-- | sys/kern/kern_exit.c | 4 | ||||
-rw-r--r-- | sys/kern/kern_fork.c | 22 | ||||
-rw-r--r-- | sys/kern/kern_linker.c | 4 | ||||
-rw-r--r-- | sys/kern/subr_smp.c | 167 | ||||
-rw-r--r-- | sys/kern/subr_trap.c | 11 | ||||
-rw-r--r-- | sys/kern/vfs_aio.c | 48 |
7 files changed, 67 insertions, 194 deletions
diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c index 8d9377e..bf6ec7f 100644 --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -39,7 +39,7 @@ * SUCH DAMAGE. * * @(#)init_main.c 8.9 (Berkeley) 1/21/94 - * $Id: init_main.c,v 1.112 1999/04/20 21:15:13 des Exp $ + * $Id: init_main.c,v 1.113 1999/04/24 18:50:48 dt Exp $ */ #include "opt_devfs.h" @@ -232,14 +232,12 @@ restart: break; case SI_TYPE_KTHREAD: -#if !defined(SMP) /* kernel thread*/ if (fork1(&proc0, RFMEM|RFFDG|RFPROC)) panic("fork kernel thread"); cpu_set_fork_handler(pfind(proc0.p_retval[0]), (*sipp)->func, (*sipp)->udata); break; -#endif case SI_TYPE_KPROCESS: if (fork1(&proc0, RFFDG|RFPROC)) @@ -360,7 +358,6 @@ proc0_init(dummy) * any possible traps/probes to simplify trap processing. */ p = &proc0; - curproc = p; /* XXX redundant*/ /* * Initialize process and pgrp structures. diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c index 0ffc9bb..0e97656 100644 --- a/sys/kern/kern_exit.c +++ b/sys/kern/kern_exit.c @@ -36,7 +36,7 @@ * SUCH DAMAGE. * * @(#)kern_exit.c 8.7 (Berkeley) 2/12/94 - * $Id: kern_exit.c,v 1.77 1999/03/11 21:53:12 bde Exp $ + * $Id: kern_exit.c,v 1.78 1999/04/17 08:36:04 peter Exp $ */ #include "opt_compat.h" @@ -350,7 +350,7 @@ exit1(p, rv) * * Other substructures are freed from wait(). */ - curproc = NULL; + SET_CURPROC(NULL); if (--p->p_limit->p_refcnt == 0) { FREE(p->p_limit, M_SUBPROC); p->p_limit = NULL; diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c index 801b90c..d3f40ea 100644 --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -36,7 +36,7 @@ * SUCH DAMAGE. * * @(#)kern_fork.c 8.6 (Berkeley) 4/8/94 - * $Id: kern_fork.c,v 1.58 1999/04/17 08:36:04 peter Exp $ + * $Id: kern_fork.c,v 1.59 1999/04/24 11:25:01 dt Exp $ */ #include "opt_ktrace.h" @@ -65,11 +65,7 @@ #include <machine/frame.h> #include <sys/user.h> -#ifdef SMP -static int fast_vfork = 0; /* Doesn't work on SMP yet. */ -#else static int fast_vfork = 1; -#endif SYSCTL_INT(_kern, OID_AUTO, fast_vfork, CTLFLAG_RW, &fast_vfork, 0, ""); /* @@ -140,22 +136,6 @@ fork1(p1, flags) if ((flags & (RFFDG|RFCFDG)) == (RFFDG|RFCFDG)) return (EINVAL); -#ifdef SMP - /* - * FATAL now, we cannot have the same PTD on both cpus, the PTD - * needs to move out of PTmap and be per-process, even for shared - * page table processes. Unfortunately, this means either removing - * PTD[] as a fixed virtual address, or move it to the per-cpu map - * area for SMP mode. Both cases require seperate management of - * the per-process-even-if-PTmap-is-shared PTD. - */ - if (flags & RFMEM) { - printf("shared address space fork attempted: pid: %d\n", - p1->p_pid); - return (EOPNOTSUPP); - } -#endif - /* * Here we don't create a new process, but we divorce * certain parts of a process from itself. diff --git a/sys/kern/kern_linker.c b/sys/kern/kern_linker.c index b184a98..ba60a8b 100644 --- a/sys/kern/kern_linker.c +++ b/sys/kern/kern_linker.c @@ -23,7 +23,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: kern_linker.c,v 1.29 1999/04/06 03:02:11 peter Exp $ + * $Id: kern_linker.c,v 1.30 1999/04/27 11:15:57 phk Exp $ */ #include "opt_ddb.h" @@ -151,14 +151,12 @@ linker_file_sysinit(linker_file_t lf) break; case SI_TYPE_KTHREAD: -#if !defined(SMP) /* kernel thread*/ if (fork1(&proc0, RFFDG|RFPROC|RFMEM)) panic("fork kernel thread"); cpu_set_fork_handler(pfind(proc0.p_retval[0]), (*sipp)->func, (*sipp)->udata); break; -#endif case SI_TYPE_KPROCESS: /* kernel thread*/ diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c index eca2601..dcca437 100644 --- a/sys/kern/subr_smp.c +++ b/sys/kern/subr_smp.c @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.96 1999/04/11 00:43:43 tegge Exp $ + * $Id: mp_machdep.c,v 1.97 1999/04/13 03:24:47 tegge Exp $ */ #include "opt_smp.h" @@ -297,28 +297,15 @@ int apic_id_to_logical[NAPICID]; /* Bitmap of all available CPUs */ u_int all_cpus; -/* AP uses this PTD during bootstrap. Do not staticize. */ -pd_entry_t *bootPTD; +/* AP uses this during bootstrap. Do not staticize. */ +char *bootSTK; +int boot_cpuid; /* Hotwire a 0->4MB V==P mapping */ extern pt_entry_t *KPTphys; -/* Virtual address of per-cpu common_tss */ -extern struct i386tss common_tss; -#ifdef VM86 -extern struct segment_descriptor common_tssd; -extern u_int private_tss; /* flag indicating private tss */ -extern u_int my_tr; -#endif /* VM86 */ - -/* IdlePTD per cpu */ -pd_entry_t *IdlePTDS[NCPU]; - -/* "my" private page table page, for BSP init */ -extern pt_entry_t SMP_prvpt[]; - -/* Private page pointer to curcpu's PTD, used during BSP init */ -extern pd_entry_t *my_idlePTD; +/* SMP page table page */ +extern pt_entry_t *SMPpt; struct pcb stoppcbs[NCPU]; @@ -466,41 +453,42 @@ void init_secondary(void) { int gsel_tss; -#ifndef VM86 - u_int my_tr; -#endif + int x, myid = boot_cpuid; + + gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid]; + gdt_segs[GPROC0_SEL].ssd_base = + (int) &SMP_prvspace[myid].globaldata.gd_common_tss; + SMP_prvspace[myid].globaldata.gd_prvspace = &SMP_prvspace[myid]; - r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1; - r_gdt.rd_base = (int) gdt; + for (x = 0; x < NGDT; x++) { + ssdtosd(&gdt_segs[x], &gdt[myid * NGDT + x].sd); + } + + r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1; + r_gdt.rd_base = (int) &gdt[myid * NGDT]; lgdt(&r_gdt); /* does magic intra-segment return */ + lidt(&r_idt); + lldt(_default_ldt); #ifdef USER_LDT currentldt = _default_ldt; #endif - my_tr = NGDT + cpuid; - gsel_tss = GSEL(my_tr, SEL_KPL); - gdt[my_tr].sd.sd_type = SDT_SYS386TSS; + gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); + gdt[myid * NGDT + GPROC0_SEL].sd.sd_type = SDT_SYS386TSS; common_tss.tss_esp0 = 0; /* not used until after switch */ common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); common_tss.tss_ioopt = (sizeof common_tss) << 16; #ifdef VM86 - common_tssd = gdt[my_tr].sd; - private_tss = 0; -#endif /* VM86 */ + common_tssd = gdt[myid * NGDT + GPROC0_SEL].sd; +#endif ltr(gsel_tss); load_cr0(0x8005003b); /* XXX! */ - PTD[0] = 0; pmap_set_opt((unsigned *)PTD); -#if 0 - putmtrr(); - pmap_setvidram(); -#endif - invltlb(); } @@ -558,11 +546,6 @@ mp_enable(u_int boot_addr) u_int ux; #endif /* APIC_IO */ -#if 0 - getmtrr(); - pmap_setvidram(); -#endif - POSTCODE(MP_ENABLE_POST); /* turn on 4MB of V == P addressing so we can get to MP table */ @@ -1770,14 +1753,11 @@ extern int wait_ap(unsigned int); static int start_all_aps(u_int boot_addr) { - int x, i; + int x, i, pg; u_char mpbiosreason; u_long mpbioswarmvec; - pd_entry_t *newptd; - pt_entry_t *newpt; struct globaldata *gd; char *stack; - pd_entry_t *myPTD; POSTCODE(START_ALL_APS_POST); @@ -1799,70 +1779,46 @@ start_all_aps(u_int boot_addr) /* record BSP in CPU map */ all_cpus = 1; + /* set up 0 -> 4MB P==V mapping for AP boot */ + *(int *)PTD = PG_V | PG_RW | ((uintptr_t)(void *)KPTphys & PG_FRAME); + invltlb(); + /* start each AP */ for (x = 1; x <= mp_naps; ++x) { /* This is a bit verbose, it will go away soon. */ - /* alloc new page table directory */ - newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE)); - - /* Store the virtual PTD address for this CPU */ - IdlePTDS[x] = newptd; - - /* clone currently active one (ie: IdlePTD) */ - bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */ - - /* set up 0 -> 4MB P==V mapping for AP boot */ - newptd[0] = (void *)(uintptr_t)(PG_V | PG_RW | - ((uintptr_t)(void *)KPTphys & PG_FRAME)); - - /* store PTD for this AP's boot sequence */ - myPTD = (pd_entry_t *)vtophys(newptd); - - /* alloc new page table page */ - newpt = (pt_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE)); - - /* set the new PTD's private page to point there */ - newptd[MPPTDI] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt)); - - /* install self referential entry */ - newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd)); + /* first page of AP's private space */ + pg = x * i386_btop(sizeof(struct privatespace)); /* allocate a new private data page */ gd = (struct globaldata *)kmem_alloc(kernel_map, PAGE_SIZE); /* wire it into the private page table page */ - newpt[0] = (pt_entry_t)(PG_V | PG_RW | vtophys(gd)); - - /* wire the ptp into itself for access */ - newpt[1] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt)); - - /* copy in the pointer to the local apic */ - newpt[2] = SMP_prvpt[2]; - - /* and the IO apic mapping[s] */ - for (i = 16; i < 32; i++) - newpt[i] = SMP_prvpt[i]; + SMPpt[pg] = (pt_entry_t)(PG_V | PG_RW | vtophys(gd)); /* allocate and set up an idle stack data page */ stack = (char *)kmem_alloc(kernel_map, UPAGES*PAGE_SIZE); for (i = 0; i < UPAGES; i++) - newpt[i + 3] = (pt_entry_t)(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); + SMPpt[pg + 5 + i] = (pt_entry_t) + (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); - newpt[3 + UPAGES] = 0; /* *prv_CMAP1 */ - newpt[4 + UPAGES] = 0; /* *prv_CMAP2 */ - newpt[5 + UPAGES] = 0; /* *prv_CMAP3 */ - newpt[6 + UPAGES] = 0; /* *prv_PMAP1 */ + SMPpt[pg + 1] = 0; /* *prv_CMAP1 */ + SMPpt[pg + 2] = 0; /* *prv_CMAP2 */ + SMPpt[pg + 3] = 0; /* *prv_CMAP3 */ + SMPpt[pg + 4] = 0; /* *prv_PMAP1 */ /* prime data page for it to use */ - gd->cpuid = x; - gd->cpu_lockid = x << 24; - gd->my_idlePTD = myPTD; - gd->prv_CMAP1 = &newpt[3 + UPAGES]; - gd->prv_CMAP2 = &newpt[4 + UPAGES]; - gd->prv_CMAP3 = &newpt[5 + UPAGES]; - gd->prv_PMAP1 = &newpt[6 + UPAGES]; + gd->gd_cpuid = x; + gd->gd_cpu_lockid = x << 24; + gd->gd_prv_CMAP1 = &SMPpt[pg + 1]; + gd->gd_prv_CMAP2 = &SMPpt[pg + 2]; + gd->gd_prv_CMAP3 = &SMPpt[pg + 3]; + gd->gd_prv_PMAP1 = &SMPpt[pg + 4]; + gd->gd_prv_CADDR1 = SMP_prvspace[x].CPAGE1; + gd->gd_prv_CADDR2 = SMP_prvspace[x].CPAGE2; + gd->gd_prv_CADDR3 = SMP_prvspace[x].CPAGE3; + gd->gd_prv_PADDR1 = (unsigned *)SMP_prvspace[x].PPAGE1; /* setup a vector to our boot code */ *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; @@ -1872,7 +1828,9 @@ start_all_aps(u_int boot_addr) outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ #endif - bootPTD = myPTD; + bootSTK = &SMP_prvspace[x].idlestack[UPAGES*PAGE_SIZE]; + boot_cpuid = x; + /* attempt to start the Application Processor */ CHECK_INIT(99); /* setup checkpoints */ if (!start_ap(x, boot_addr)) { @@ -1910,27 +1868,16 @@ start_all_aps(u_int boot_addr) * because the BSP is cpu#0 and the page is initially zero, and also * because we can refer to variables by name on the BSP.. */ - newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE)); - - bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */ - IdlePTDS[0] = newptd; - - /* Point PTD[] to this page instead of IdlePTD's physical page */ - newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd)); - - my_idlePTD = (pd_entry_t *)vtophys(newptd); /* Allocate and setup BSP idle stack */ stack = (char *)kmem_alloc(kernel_map, UPAGES * PAGE_SIZE); for (i = 0; i < UPAGES; i++) - SMP_prvpt[i + 3] = (pt_entry_t)(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); + SMPpt[5 + i] = (pt_entry_t) + (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); + *(int *)PTD = 0; pmap_set_opt_bsp(); - for (i = 0; i < mp_ncpus; i++) { - bcopy( (int *) PTD + KPTDI, (int *) IdlePTDS[i] + KPTDI, NKPDE * sizeof (int)); - } - /* number of APs actually started */ return mp_ncpus - 1; } @@ -2250,10 +2197,6 @@ ap_init() panic("cpuid mismatch! boom!!"); } -#if 0 - getmtrr(); -#endif - /* Init local apic for irq's */ apic_initialize(); @@ -2267,8 +2210,6 @@ ap_init() smp_started = 1; /* enable IPI's, tlb shootdown, freezes etc */ smp_active = 1; /* historic */ } - - curproc = NULL; /* make sure */ } #ifdef BETTER_CLOCK diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c index d08306e..c0807b0 100644 --- a/sys/kern/subr_trap.c +++ b/sys/kern/subr_trap.c @@ -35,7 +35,7 @@ * SUCH DAMAGE. * * from: @(#)trap.c 7.4 (Berkeley) 5/13/91 - * $Id: trap.c,v 1.134 1999/03/09 20:20:09 phk Exp $ + * $Id: trap.c,v 1.135 1999/04/19 14:14:13 peter Exp $ */ /* @@ -101,8 +101,6 @@ #include "isa.h" #include "npx.h" -extern struct i386tss common_tss; - int (*pmath_emulate) __P((struct trapframe *)); extern void trap __P((struct trapframe frame)); @@ -480,11 +478,6 @@ kernel_trap: * (XXX) so that we can continue, and generate * a signal. */ - if (frame.tf_eip == (int)cpu_switch_load_fs) { - curpcb->pcb_fs = 0; - psignal(p, SIGBUS); - return; - } if (frame.tf_eip == (int)cpu_switch_load_gs) { curpcb->pcb_gs = 0; psignal(p, SIGBUS); @@ -496,6 +489,8 @@ kernel_trap: doreti_popl_ds_fault); MAYBE_DORETI_FAULT(doreti_popl_es, doreti_popl_es_fault); + MAYBE_DORETI_FAULT(doreti_popl_fs, + doreti_popl_fs_fault); if (curpcb && curpcb->pcb_onfault) { frame.tf_eip = (int)curpcb->pcb_onfault; return; diff --git a/sys/kern/vfs_aio.c b/sys/kern/vfs_aio.c index 0c4f448..577c499 100644 --- a/sys/kern/vfs_aio.c +++ b/sys/kern/vfs_aio.c @@ -13,7 +13,7 @@ * bad that happens because of using this software isn't the responsibility * of the author. This software is distributed AS-IS. * - * $Id: vfs_aio.c,v 1.44 1999/02/25 15:54:06 bde Exp $ + * $Id: vfs_aio.c,v 1.45 1999/04/04 21:41:16 dt Exp $ */ /* @@ -244,11 +244,6 @@ static vm_zone_t kaio_zone=0, aiop_zone=0, aiocb_zone=0, aiol_zone=0, aiolio_zone=0; /* - * Single AIOD vmspace shared amongst all of them - */ -struct vmspace *aiovmspace = NULL; - -/* * Startup initialization */ void @@ -607,7 +602,7 @@ aio_daemon(const void *uproc) { int s; struct aioproclist *aiop; - struct vmspace *myvm, *aiovm; + struct vmspace *myvm; struct proc *mycp; /* @@ -616,35 +611,6 @@ aio_daemon(const void *uproc) mycp = curproc; myvm = mycp->p_vmspace; - /* - * We manage to create only one VM space for all AIOD processes. - * The VM space for the first AIOD created becomes the shared VM - * space for all of them. We add an additional reference count, - * even for the first AIOD, so the address space does not go away, - * and we continue to use that original VM space even if the first - * AIOD exits. - */ - if ((aiovm = aiovmspace) == NULL) { - aiovmspace = myvm; - myvm->vm_refcnt++; - /* - * Remove userland cruft from address space. - */ - if (myvm->vm_shm) - shmexit(mycp); - pmap_remove_pages(vmspace_pmap(myvm), 0, USRSTACK); - vm_map_remove(&myvm->vm_map, 0, USRSTACK); - myvm->vm_tsize = 0; - myvm->vm_dsize = 0; - myvm->vm_ssize = 0; - } else { - aiovm->vm_refcnt++; - mycp->p_vmspace = aiovm; - pmap_activate(mycp); - vmspace_free(myvm); - myvm = aiovm; - } - if (mycp->p_textvp) { vrele(mycp->p_textvp); mycp->p_textvp = NULL; @@ -921,17 +887,13 @@ static int aio_newproc() { int error; - struct rfork_args rfa; struct proc *p, *np; - rfa.flags = RFPROC | RFCFDG; - - p = curproc; - if ((error = rfork(p, &rfa)) != 0) + p = &proc0; + if (error = fork1(p, RFPROC|RFMEM|RFNOWAIT)) return error; - np = pfind(p->p_retval[0]); - cpu_set_fork_handler(np, aio_daemon, p); + cpu_set_fork_handler(np, aio_daemon, curproc); /* * Wait until daemon is started, but continue on just in case (to |