Implement a facility for dynamic per-cpu variables.

 - Modules and kernel code alike may use DPCPU_DEFINE(),
   DPCPU_GET(), DPCPU_SET(), etc. akin to the statically defined
   PCPU_*.  Requires only one extra instruction more than PCPU_* and is
   virtually the same as __thread for builtin and much faster for shared
   objects.  DPCPU variables can be initialized when defined.
 - Modules are supported by relocating the module's per-cpu linker set
   over space reserved in the kernel.  Modules may fail to load if there
   is insufficient space available.
 - Track space available for modules with a one-off extent allocator.
   Free may block for memory to allocate space for an extent.

Reviewed by:    jhb, rwatson, kan, sam, grehan, marius, marcel, stas

5 files changed, 17 insertions, 4 deletions

diff --git a/sys/sparc64/include/pcpu.h b/sys/sparc64/include/pcpu.h
index efb5174..7d2f5a0 100644
--- a/sys/sparc64/include/pcpu.h
+++ b/sys/sparc64/include/pcpu.h
@@ -62,6 +62,8 @@ struct pmap;
 
 #ifdef _KERNEL
 
+extern void *dpcpu0;
+
 struct pcb;
 struct pcpu;
 
diff --git a/sys/sparc64/sparc64/elf_machdep.c b/sys/sparc64/sparc64/elf_machdep.c
index b0a6998..d113b49 100644
--- a/sys/sparc64/sparc64/elf_machdep.c
+++ b/sys/sparc64/sparc64/elf_machdep.c
@@ -285,7 +285,7 @@ elf_reloc_local(linker_file_t lf, Elf_Addr relocbase, const void *data,
 	value = rela->r_addend + (Elf_Addr)lf->address;
 	where = (Elf_Addr *)((Elf_Addr)lf->address + rela->r_offset);
 
-	*where = value;
+	*where = elf_relocaddr(lf, value);
 
 	return (0);
 }
@@ -338,8 +338,9 @@ elf_reloc(linker_file_t lf, Elf_Addr relocbase, const void *data, int type,
 	if (RELOC_PC_RELATIVE(rtype))
 		value -= (Elf_Addr)where;
 
-	if (RELOC_BASE_RELATIVE(rtype))
-		value += relocbase;
+	if (RELOC_BASE_RELATIVE(rtype)) {
+		value = elf_relocaddr(lf, value + relocbase);
+	}
 
 	mask = RELOC_VALUE_BITMASK(rtype);
 	value >>= RELOC_VALUE_RIGHTSHIFT(rtype);
diff --git a/sys/sparc64/sparc64/machdep.c b/sys/sparc64/sparc64/machdep.c
index 164a8b3..b90a54d 100644
--- a/sys/sparc64/sparc64/machdep.c
+++ b/sys/sparc64/sparc64/machdep.c
@@ -124,6 +124,7 @@ int cold = 1;
 long Maxmem;
 long realmem;
 
+void *dpcpu0;
 char pcpu0[PCPU_PAGES * PAGE_SIZE];
 struct trapframe frame0;
 
@@ -480,8 +481,10 @@ sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_long o3, ofw_vec_t *vec)
 	delay_func = delay_tick;
 
 	/*
-	 * Initialize the message buffer (after setting trap table).
+	 * Initialize the dynamic per-CPU area for the BSP and the message
+	 * buffer (after setting the trap table).
 	 */
+	dpcpu_init(dpcpu0, 0);
 	msgbufinit(msgbufp, MSGBUF_SIZE);
 
 	mutex_init();
diff --git a/sys/sparc64/sparc64/mp_machdep.c b/sys/sparc64/sparc64/mp_machdep.c
index ee4ffe7..997f0a4 100644
--- a/sys/sparc64/sparc64/mp_machdep.c
+++ b/sys/sparc64/sparc64/mp_machdep.c
@@ -290,6 +290,8 @@ cpu_mp_start(void)
 		va = kmem_alloc(kernel_map, PCPU_PAGES * PAGE_SIZE);
 		pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1;
 		pcpu_init(pc, cpuid, sizeof(*pc));
+		dpcpu_init((void *)kmem_alloc(kernel_map, DPCPU_SIZE),
+		    cpuid);
 		pc->pc_addr = va;
 		pc->pc_clock = clock;
 		pc->pc_mid = mid;
diff --git a/sys/sparc64/sparc64/pmap.c b/sys/sparc64/sparc64/pmap.c
index 08c9ac2..1e1de83 100644
--- a/sys/sparc64/sparc64/pmap.c
+++ b/sys/sparc64/sparc64/pmap.c
@@ -363,6 +363,11 @@ pmap_bootstrap(vm_offset_t ekva)
 	bzero(tsb_kernel, tsb_kernel_size);
 
 	/*
+	 * Allocate and map the dynamic per-CPU area for the BSP.
+	 */
+	dpcpu0 = (void *)TLB_PHYS_TO_DIRECT(pmap_bootstrap_alloc(DPCPU_SIZE));
+
+	/*
 	 * Allocate and map the message buffer.
 	 */
 	msgbuf_phys = pmap_bootstrap_alloc(MSGBUF_SIZE);