Implement a facility for dynamic per-cpu variables.

 - Modules and kernel code alike may use DPCPU_DEFINE(),
   DPCPU_GET(), DPCPU_SET(), etc. akin to the statically defined
   PCPU_*.  Requires only one extra instruction more than PCPU_* and is
   virtually the same as __thread for builtin and much faster for shared
   objects.  DPCPU variables can be initialized when defined.
 - Modules are supported by relocating the module's per-cpu linker set
   over space reserved in the kernel.  Modules may fail to load if there
   is insufficient space available.
 - Track space available for modules with a one-off extent allocator.
   Free may block for memory to allocate space for an extent.

Reviewed by:    jhb, rwatson, kan, sam, grehan, marius, marcel, stas

4 files changed, 11 insertions, 0 deletions

diff --git a/sys/sun4v/include/pcpu.h b/sys/sun4v/include/pcpu.h
index ec89212..350e29a 100644
--- a/sys/sun4v/include/pcpu.h
+++ b/sys/sun4v/include/pcpu.h
@@ -89,6 +89,8 @@ struct pmap;
 
 #ifdef _KERNEL
 
+extern void *dpcpu0;
+
 struct pcpu;
 
 register struct pcpu *pcpup __asm__(__XSTRING(PCPU_REG));
diff --git a/sys/sun4v/sun4v/machdep.c b/sys/sun4v/sun4v/machdep.c
index 7975040..5430460 100644
--- a/sys/sun4v/sun4v/machdep.c
+++ b/sys/sun4v/sun4v/machdep.c
@@ -129,6 +129,7 @@ int cold = 1;
 long Maxmem;
 long realmem;
 
+void *dpcpu0;
 char pcpu0[PCPU_PAGES * PAGE_SIZE];
 struct trapframe frame0;
 int trap_conversion[256];
@@ -500,6 +501,7 @@ sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_long o3, ofw_vec_t *vec)
 	 * Initialize the message buffer (after setting trap table).
 	 */
 	BVPRINTF("initialize msgbuf\n");
+	dpcpu_init(dpcpu0, 0);
 	msgbufinit(msgbufp, MSGBUF_SIZE);
 
 	BVPRINTF("initialize mutexes\n");
diff --git a/sys/sun4v/sun4v/mp_machdep.c b/sys/sun4v/sun4v/mp_machdep.c
index 5f4f632..5b5df89 100644
--- a/sys/sun4v/sun4v/mp_machdep.c
+++ b/sys/sun4v/sun4v/mp_machdep.c
@@ -324,6 +324,8 @@ cpu_mp_start(void)
 		va = kmem_alloc(kernel_map, PCPU_PAGES * PAGE_SIZE);
 		pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1;
 		pcpu_init(pc, cpuid, sizeof(*pc));
+		dpcpu_init((void *)kmem_alloc(kernel_map, DPCPU_SIZE),
+		    cpuid);
 		pc->pc_addr = va;
 
 		all_cpus |= 1 << cpuid;
diff --git a/sys/sun4v/sun4v/pmap.c b/sys/sun4v/sun4v/pmap.c
index 4a5b7d6..e5c2382 100644
--- a/sys/sun4v/sun4v/pmap.c
+++ b/sys/sun4v/sun4v/pmap.c
@@ -767,6 +767,11 @@ skipshuffle:
 	mmu_fault_status_area = pmap_bootstrap_alloc(MMFSA_SIZE*MAXCPU);
 
 	/*
+	 * Allocate and map the dynamic per-CPU area for the BSP.
+	 */
+	dpcpu0 = (void *)TLB_PHYS_TO_DIRECT(pmap_bootstrap_alloc(DPCPU_SIZE));
+
+	/*
 	 * Allocate and map the message buffer.
 	 */
 	msgbuf_phys = pmap_bootstrap_alloc(MSGBUF_SIZE);