Enable vmspace sharing on SMP. Major changes are,

- %fs register is added to trapframe and saved/restored upon kernel entry/exit.
- Per-cpu pages are no longer mapped at the same virtual address.
- Each cpu now has a separate gdt selector table. A new segment selector
  is added to point to per-cpu pages, per-cpu global variables are now
  accessed through this new selector (%fs). The selectors in gdt table are
  rearranged for cache line optimization.
- fask_vfork is now on as default for both UP and SMP.
- Some aio code cleanup.

Reviewed by:	Alan Cox	<alc@cs.rice.edu>
		John Dyson	<dyson@iquest.net>
		Julian Elischer	<julian@whistel.com>
		Bruce Evans	<bde@zeta.org.au>
		David Greenman	<dg@root.com>

7 files changed, 67 insertions, 194 deletions

diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index 8d9377e..bf6ec7f 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -39,7 +39,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)init_main.c	8.9 (Berkeley) 1/21/94
- * $Id: init_main.c,v 1.112 1999/04/20 21:15:13 des Exp $
+ * $Id: init_main.c,v 1.113 1999/04/24 18:50:48 dt Exp $
  */
 
 #include "opt_devfs.h"
@@ -232,14 +232,12 @@ restart:
 			break;
 
 		case SI_TYPE_KTHREAD:
-#if !defined(SMP)
 			/* kernel thread*/
 			if (fork1(&proc0, RFMEM|RFFDG|RFPROC))
 				panic("fork kernel thread");
 			cpu_set_fork_handler(pfind(proc0.p_retval[0]),
 			    (*sipp)->func, (*sipp)->udata);
 			break;
-#endif
 
 		case SI_TYPE_KPROCESS:
 			if (fork1(&proc0, RFFDG|RFPROC))
@@ -360,7 +358,6 @@ proc0_init(dummy)
 	 * any possible traps/probes to simplify trap processing.
 	 */
 	p = &proc0;
-	curproc = p;			/* XXX redundant*/
 
 	/*
 	 * Initialize process and pgrp structures.
diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c
index 0ffc9bb..0e97656 100644
--- a/sys/kern/kern_exit.c
+++ b/sys/kern/kern_exit.c
@@ -36,7 +36,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)kern_exit.c	8.7 (Berkeley) 2/12/94
- * $Id: kern_exit.c,v 1.77 1999/03/11 21:53:12 bde Exp $
+ * $Id: kern_exit.c,v 1.78 1999/04/17 08:36:04 peter Exp $
  */
 
 #include "opt_compat.h"
@@ -350,7 +350,7 @@ exit1(p, rv)
 	 *
 	 * Other substructures are freed from wait().
 	 */
-	curproc = NULL;
+	SET_CURPROC(NULL);
 	if (--p->p_limit->p_refcnt == 0) {
 		FREE(p->p_limit, M_SUBPROC);
 		p->p_limit = NULL;
diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index 801b90c..d3f40ea 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -36,7 +36,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)kern_fork.c	8.6 (Berkeley) 4/8/94
- * $Id: kern_fork.c,v 1.58 1999/04/17 08:36:04 peter Exp $
+ * $Id: kern_fork.c,v 1.59 1999/04/24 11:25:01 dt Exp $
  */
 
 #include "opt_ktrace.h"
@@ -65,11 +65,7 @@
 #include <machine/frame.h>
 #include <sys/user.h>
 
-#ifdef SMP
-static int	fast_vfork = 0;	/* Doesn't work on SMP yet. */
-#else
 static int	fast_vfork = 1;
-#endif
 SYSCTL_INT(_kern, OID_AUTO, fast_vfork, CTLFLAG_RW, &fast_vfork, 0, "");
 
 /*
@@ -140,22 +136,6 @@ fork1(p1, flags)
 	if ((flags & (RFFDG|RFCFDG)) == (RFFDG|RFCFDG))
 		return (EINVAL);
 
-#ifdef SMP
-	/*
-	 * FATAL now, we cannot have the same PTD on both cpus, the PTD
-	 * needs to move out of PTmap and be per-process, even for shared
-	 * page table processes.  Unfortunately, this means either removing
-	 * PTD[] as a fixed virtual address, or move it to the per-cpu map
-	 * area for SMP mode.  Both cases require seperate management of
-	 * the per-process-even-if-PTmap-is-shared PTD.
-	 */
-	if (flags & RFMEM) {
-		printf("shared address space fork attempted: pid: %d\n",
-		    p1->p_pid);
-		return (EOPNOTSUPP);
-	}
-#endif
-
 	/*
 	 * Here we don't create a new process, but we divorce
 	 * certain parts of a process from itself.
diff --git a/sys/kern/kern_linker.c b/sys/kern/kern_linker.c
index b184a98..ba60a8b 100644
--- a/sys/kern/kern_linker.c
+++ b/sys/kern/kern_linker.c
@@ -23,7 +23,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: kern_linker.c,v 1.29 1999/04/06 03:02:11 peter Exp $
+ *	$Id: kern_linker.c,v 1.30 1999/04/27 11:15:57 phk Exp $
  */
 
 #include "opt_ddb.h"
@@ -151,14 +151,12 @@ linker_file_sysinit(linker_file_t lf)
 	    break;
 
 	case SI_TYPE_KTHREAD:
-#if !defined(SMP)
 	    /* kernel thread*/
 	    if (fork1(&proc0, RFFDG|RFPROC|RFMEM))
 		panic("fork kernel thread");
 	    cpu_set_fork_handler(pfind(proc0.p_retval[0]),
 		(*sipp)->func, (*sipp)->udata);
 	    break;
-#endif
 
 	case SI_TYPE_KPROCESS:
 	    /* kernel thread*/
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index eca2601..dcca437 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.96 1999/04/11 00:43:43 tegge Exp $
+ *	$Id: mp_machdep.c,v 1.97 1999/04/13 03:24:47 tegge Exp $
  */
 
 #include "opt_smp.h"
@@ -297,28 +297,15 @@ int     apic_id_to_logical[NAPICID];
 /* Bitmap of all available CPUs */
 u_int	all_cpus;
 
-/* AP uses this PTD during bootstrap.  Do not staticize.  */
-pd_entry_t *bootPTD;
+/* AP uses this during bootstrap.  Do not staticize.  */
+char *bootSTK;
+int boot_cpuid;
 
 /* Hotwire a 0->4MB V==P mapping */
 extern pt_entry_t *KPTphys;
 
-/* Virtual address of per-cpu common_tss */
-extern struct i386tss common_tss;
-#ifdef VM86
-extern struct segment_descriptor common_tssd;
-extern u_int private_tss;		/* flag indicating private tss */
-extern u_int my_tr;
-#endif /* VM86 */
-
-/* IdlePTD per cpu */
-pd_entry_t *IdlePTDS[NCPU];
-
-/* "my" private page table page, for BSP init */
-extern pt_entry_t SMP_prvpt[];
-
-/* Private page pointer to curcpu's PTD, used during BSP init */
-extern pd_entry_t *my_idlePTD;
+/* SMP page table page */
+extern pt_entry_t *SMPpt;
 
 struct pcb stoppcbs[NCPU];
 
@@ -466,41 +453,42 @@ void
 init_secondary(void)
 {
 	int	gsel_tss;
-#ifndef VM86
-	u_int	my_tr;
-#endif
+	int	x, myid = boot_cpuid;
+
+	gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid];
+	gdt_segs[GPROC0_SEL].ssd_base =
+		(int) &SMP_prvspace[myid].globaldata.gd_common_tss;
+	SMP_prvspace[myid].globaldata.gd_prvspace = &SMP_prvspace[myid];
 
-	r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1;
-	r_gdt.rd_base = (int) gdt;
+	for (x = 0; x < NGDT; x++) {
+		ssdtosd(&gdt_segs[x], &gdt[myid * NGDT + x].sd);
+	}
+
+	r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1;
+	r_gdt.rd_base = (int) &gdt[myid * NGDT];
 	lgdt(&r_gdt);			/* does magic intra-segment return */
+
 	lidt(&r_idt);
+
 	lldt(_default_ldt);
 #ifdef USER_LDT
 	currentldt = _default_ldt;
 #endif
 
-	my_tr = NGDT + cpuid;
-	gsel_tss = GSEL(my_tr, SEL_KPL);
-	gdt[my_tr].sd.sd_type = SDT_SYS386TSS;
+	gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
+	gdt[myid * NGDT + GPROC0_SEL].sd.sd_type = SDT_SYS386TSS;
 	common_tss.tss_esp0 = 0;	/* not used until after switch */
 	common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
 	common_tss.tss_ioopt = (sizeof common_tss) << 16;
 #ifdef VM86
-	common_tssd = gdt[my_tr].sd;
-	private_tss = 0;
-#endif /* VM86 */
+	common_tssd = gdt[myid * NGDT + GPROC0_SEL].sd;
+#endif
 	ltr(gsel_tss);
 
 	load_cr0(0x8005003b);		/* XXX! */
 
-	PTD[0] = 0;
 	pmap_set_opt((unsigned *)PTD);
 
-#if 0
-	putmtrr();
-	pmap_setvidram();
-#endif
-
 	invltlb();
 }
 
@@ -558,11 +546,6 @@ mp_enable(u_int boot_addr)
 	u_int   ux;
 #endif	/* APIC_IO */
 
-#if 0
-	getmtrr();
-	pmap_setvidram();
-#endif
-
 	POSTCODE(MP_ENABLE_POST);
 
 	/* turn on 4MB of V == P addressing so we can get to MP table */
@@ -1770,14 +1753,11 @@ extern int wait_ap(unsigned int);
 static int
 start_all_aps(u_int boot_addr)
 {
-	int     x, i;
+	int     x, i, pg;
 	u_char  mpbiosreason;
 	u_long  mpbioswarmvec;
-	pd_entry_t *newptd;
-	pt_entry_t *newpt;
 	struct globaldata *gd;
 	char *stack;
-	pd_entry_t	*myPTD;
 
 	POSTCODE(START_ALL_APS_POST);
 
@@ -1799,70 +1779,46 @@ start_all_aps(u_int boot_addr)
 	/* record BSP in CPU map */
 	all_cpus = 1;
 
+	/* set up 0 -> 4MB P==V mapping for AP boot */
+	*(int *)PTD = PG_V | PG_RW | ((uintptr_t)(void *)KPTphys & PG_FRAME);
+	invltlb();
+
 	/* start each AP */
 	for (x = 1; x <= mp_naps; ++x) {
 
 		/* This is a bit verbose, it will go away soon.  */
 
-		/* alloc new page table directory */
-		newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE));
-
-		/* Store the virtual PTD address for this CPU */
-		IdlePTDS[x] = newptd;
-
-		/* clone currently active one (ie: IdlePTD) */
-		bcopy(PTD, newptd, PAGE_SIZE);	/* inc prv page pde */
-
-		/* set up 0 -> 4MB P==V mapping for AP boot */
-		newptd[0] = (void *)(uintptr_t)(PG_V | PG_RW |
-		    ((uintptr_t)(void *)KPTphys & PG_FRAME));
-
-		/* store PTD for this AP's boot sequence */
-		myPTD = (pd_entry_t *)vtophys(newptd);
-
-		/* alloc new page table page */
-		newpt = (pt_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE));
-
-		/* set the new PTD's private page to point there */
-		newptd[MPPTDI] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt));
-
-		/* install self referential entry */
-		newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd));
+		/* first page of AP's private space */
+		pg = x * i386_btop(sizeof(struct privatespace));
 
 		/* allocate a new private data page */
 		gd = (struct globaldata *)kmem_alloc(kernel_map, PAGE_SIZE);
 
 		/* wire it into the private page table page */
-		newpt[0] = (pt_entry_t)(PG_V | PG_RW | vtophys(gd));
-
-		/* wire the ptp into itself for access */
-		newpt[1] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt));
-
-		/* copy in the pointer to the local apic */
-		newpt[2] = SMP_prvpt[2];
-
-		/* and the IO apic mapping[s] */
-		for (i = 16; i < 32; i++)
-			newpt[i] = SMP_prvpt[i];
+		SMPpt[pg] = (pt_entry_t)(PG_V | PG_RW | vtophys(gd));
 
 		/* allocate and set up an idle stack data page */
 		stack = (char *)kmem_alloc(kernel_map, UPAGES*PAGE_SIZE);
 		for (i = 0; i < UPAGES; i++)
-			newpt[i + 3] = (pt_entry_t)(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
+			SMPpt[pg + 5 + i] = (pt_entry_t)
+			    (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
 
-		newpt[3 + UPAGES] = 0;		/* *prv_CMAP1 */
-		newpt[4 + UPAGES] = 0;		/* *prv_CMAP2 */
-		newpt[5 + UPAGES] = 0;		/* *prv_CMAP3 */
-		newpt[6 + UPAGES] = 0;		/* *prv_PMAP1 */
+		SMPpt[pg + 1] = 0;		/* *prv_CMAP1 */
+		SMPpt[pg + 2] = 0;		/* *prv_CMAP2 */
+		SMPpt[pg + 3] = 0;		/* *prv_CMAP3 */
+		SMPpt[pg + 4] = 0;		/* *prv_PMAP1 */
 
 		/* prime data page for it to use */
-		gd->cpuid = x;
-		gd->cpu_lockid = x << 24;
-		gd->my_idlePTD = myPTD;
-		gd->prv_CMAP1 = &newpt[3 + UPAGES];
-		gd->prv_CMAP2 = &newpt[4 + UPAGES];
-		gd->prv_CMAP3 = &newpt[5 + UPAGES];
-		gd->prv_PMAP1 = &newpt[6 + UPAGES];
+		gd->gd_cpuid = x;
+		gd->gd_cpu_lockid = x << 24;
+		gd->gd_prv_CMAP1 = &SMPpt[pg + 1];
+		gd->gd_prv_CMAP2 = &SMPpt[pg + 2];
+		gd->gd_prv_CMAP3 = &SMPpt[pg + 3];
+		gd->gd_prv_PMAP1 = &SMPpt[pg + 4];
+		gd->gd_prv_CADDR1 = SMP_prvspace[x].CPAGE1;
+		gd->gd_prv_CADDR2 = SMP_prvspace[x].CPAGE2;
+		gd->gd_prv_CADDR3 = SMP_prvspace[x].CPAGE3;
+		gd->gd_prv_PADDR1 = (unsigned *)SMP_prvspace[x].PPAGE1;
 
 		/* setup a vector to our boot code */
 		*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
@@ -1872,7 +1828,9 @@ start_all_aps(u_int boot_addr)
 		outb(CMOS_DATA, BIOS_WARM);	/* 'warm-start' */
 #endif
 
-		bootPTD = myPTD;
+		bootSTK = &SMP_prvspace[x].idlestack[UPAGES*PAGE_SIZE];
+		boot_cpuid = x;
+
 		/* attempt to start the Application Processor */
 		CHECK_INIT(99);	/* setup checkpoints */
 		if (!start_ap(x, boot_addr)) {
@@ -1910,27 +1868,16 @@ start_all_aps(u_int boot_addr)
 	 * because the BSP is cpu#0 and the page is initially zero, and also
 	 * because we can refer to variables by name on the BSP..
 	 */
-	newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE));
-
-	bcopy(PTD, newptd, PAGE_SIZE);	/* inc prv page pde */
-	IdlePTDS[0] = newptd;
-
-	/* Point PTD[] to this page instead of IdlePTD's physical page */
-	newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd));
-
-	my_idlePTD = (pd_entry_t *)vtophys(newptd);
 
 	/* Allocate and setup BSP idle stack */
 	stack = (char *)kmem_alloc(kernel_map, UPAGES * PAGE_SIZE);
 	for (i = 0; i < UPAGES; i++)
-		SMP_prvpt[i + 3] = (pt_entry_t)(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
+		SMPpt[5 + i] = (pt_entry_t)
+		    (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
 
+	*(int *)PTD = 0;
 	pmap_set_opt_bsp();
 
-	for (i = 0; i < mp_ncpus; i++) {
-		bcopy( (int *) PTD + KPTDI, (int *) IdlePTDS[i] + KPTDI, NKPDE * sizeof (int));
-	}
-
 	/* number of APs actually started */
 	return mp_ncpus - 1;
 }
@@ -2250,10 +2197,6 @@ ap_init()
 		panic("cpuid mismatch! boom!!");
 	}
 
-#if 0
-	getmtrr();
-#endif
-
 	/* Init local apic for irq's */
 	apic_initialize();
 
@@ -2267,8 +2210,6 @@ ap_init()
 		smp_started = 1; /* enable IPI's, tlb shootdown, freezes etc */
 		smp_active = 1;	 /* historic */
 	}
-
-	curproc = NULL;		/* make sure */
 }
 
 #ifdef BETTER_CLOCK
diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c
index d08306e..c0807b0 100644
--- a/sys/kern/subr_trap.c
+++ b/sys/kern/subr_trap.c
@@ -35,7 +35,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)trap.c	7.4 (Berkeley) 5/13/91
- *	$Id: trap.c,v 1.134 1999/03/09 20:20:09 phk Exp $
+ *	$Id: trap.c,v 1.135 1999/04/19 14:14:13 peter Exp $
  */
 
 /*
@@ -101,8 +101,6 @@
 #include "isa.h"
 #include "npx.h"
 
-extern struct i386tss common_tss;
-
 int (*pmath_emulate) __P((struct trapframe *));
 
 extern void trap __P((struct trapframe frame));
@@ -480,11 +478,6 @@ kernel_trap:
 				 * (XXX) so that we can continue, and generate
 				 * a signal.
 				 */
-				if (frame.tf_eip == (int)cpu_switch_load_fs) {
-					curpcb->pcb_fs = 0;
-					psignal(p, SIGBUS);
-					return;
-				}
 				if (frame.tf_eip == (int)cpu_switch_load_gs) {
 					curpcb->pcb_gs = 0;
 					psignal(p, SIGBUS);
@@ -496,6 +489,8 @@ kernel_trap:
 						   doreti_popl_ds_fault);
 				MAYBE_DORETI_FAULT(doreti_popl_es,
 						   doreti_popl_es_fault);
+				MAYBE_DORETI_FAULT(doreti_popl_fs,
+						   doreti_popl_fs_fault);
 				if (curpcb && curpcb->pcb_onfault) {
 					frame.tf_eip = (int)curpcb->pcb_onfault;
 					return;
diff --git a/sys/kern/vfs_aio.c b/sys/kern/vfs_aio.c
index 0c4f448..577c499 100644
--- a/sys/kern/vfs_aio.c
+++ b/sys/kern/vfs_aio.c
@@ -13,7 +13,7 @@
  * bad that happens because of using this software isn't the responsibility
  * of the author.  This software is distributed AS-IS.
  *
- * $Id: vfs_aio.c,v 1.44 1999/02/25 15:54:06 bde Exp $
+ * $Id: vfs_aio.c,v 1.45 1999/04/04 21:41:16 dt Exp $
  */
 
 /*
@@ -244,11 +244,6 @@ static vm_zone_t kaio_zone=0, aiop_zone=0,
 	aiocb_zone=0, aiol_zone=0, aiolio_zone=0;
 
 /*
- * Single AIOD vmspace shared amongst all of them
- */
-struct vmspace *aiovmspace = NULL;
-
-/*
  * Startup initialization
  */
 void
@@ -607,7 +602,7 @@ aio_daemon(const void *uproc)
 {
 	int s;
 	struct aioproclist *aiop;
-	struct vmspace *myvm, *aiovm;
+	struct vmspace *myvm;
 	struct proc *mycp;
 
 	/*
@@ -616,35 +611,6 @@ aio_daemon(const void *uproc)
 	mycp = curproc;
 	myvm = mycp->p_vmspace;
 
-	/*
-	 * We manage to create only one VM space for all AIOD processes.
-	 * The VM space for the first AIOD created becomes the shared VM
-	 * space for all of them.  We add an additional reference count,
-	 * even for the first AIOD, so the address space does not go away,
-	 * and we continue to use that original VM space even if the first
-	 * AIOD exits.
-	 */
-	if ((aiovm = aiovmspace) == NULL) {
-		aiovmspace = myvm;
-		myvm->vm_refcnt++;
-		/*
-		 * Remove userland cruft from address space.
-		 */
-		if (myvm->vm_shm)
-			shmexit(mycp);
-		pmap_remove_pages(vmspace_pmap(myvm), 0, USRSTACK);
-		vm_map_remove(&myvm->vm_map, 0, USRSTACK);
-		myvm->vm_tsize = 0;
-		myvm->vm_dsize = 0;
-		myvm->vm_ssize = 0;
-	} else {
-		aiovm->vm_refcnt++;
-		mycp->p_vmspace = aiovm;
-		pmap_activate(mycp);
-		vmspace_free(myvm);
-		myvm = aiovm;
-	}
-
 	if (mycp->p_textvp) {
 		vrele(mycp->p_textvp);
 		mycp->p_textvp = NULL;
@@ -921,17 +887,13 @@ static int
 aio_newproc()
 {
 	int error;
-	struct rfork_args rfa;
 	struct proc *p, *np;
 
-	rfa.flags = RFPROC | RFCFDG;
-
-	p = curproc;
-	if ((error = rfork(p, &rfa)) != 0)
+	p = &proc0;
+	if (error = fork1(p, RFPROC|RFMEM|RFNOWAIT))
 		return error;
-
 	np = pfind(p->p_retval[0]);
-	cpu_set_fork_handler(np, aio_daemon, p);
+	cpu_set_fork_handler(np, aio_daemon, curproc);
 
 	/*
 	 * Wait until daemon is started, but continue on just in case (to