Use the new arch_loadaddr I/F to align ELF objects to PBVM page

boundaries. For good measure, align all other objects to cache
lines boundaries.

Use the new arch_loadseg I/F to keep track of kernel text and
data so that we can wire as much of it as is possible. It is
the responsibility of the kernel to link critical (read IVT
related) code and data at the front of the respective segment
so that it's covered by TRs before the kernel has a chance to
add more translations.

Use a better way of determining whether we're loading a legacy
kernel or not. We can't check for the presence of the PBVM page
table, because we may have unloaded that kernel and loaded an
older (legacy) kernel after that. Simply use the latest load
address for it.

6 files changed, 135 insertions, 51 deletions

diff --git a/sys/boot/ia64/common/copy.c b/sys/boot/ia64/common/copy.c
index 93ef77b..95523c2 100644
--- a/sys/boot/ia64/common/copy.c
+++ b/sys/boot/ia64/common/copy.c
@@ -28,10 +28,12 @@
 __FBSDID("$FreeBSD$");
 
 #include <stand.h>
-#include <ia64/include/vmparam.h>
+#include <machine/param.h>
 
 #include "libia64.h"
 
+u_int ia64_legacy_kernel;
+
 uint64_t *ia64_pgtbl;
 uint32_t ia64_pgtblsz;
 
@@ -80,8 +82,8 @@ pgtbl_extend(u_int idx)
 	return (0);
 }
 
-static void *
-va2pa(vm_offset_t va, size_t *len)
+void *
+ia64_va2pa(vm_offset_t va, size_t *len)
 {
 	uint64_t pa;
 	u_int idx, ofs;
@@ -89,6 +91,7 @@ va2pa(vm_offset_t va, size_t *len)
 
 	/* Backward compatibility. */
 	if (va >= IA64_RR_BASE(7)) {
+		ia64_legacy_kernel = 1;
 		pa = IA64_RR_MASK(va);
 		return ((void *)pa);
 	}
@@ -98,6 +101,8 @@ va2pa(vm_offset_t va, size_t *len)
 		goto fail;
 	}
 
+	ia64_legacy_kernel = 0;
+
 	idx = (va - IA64_PBVM_BASE) >> IA64_PBVM_PAGE_SHIFT;
 	if (idx >= (ia64_pgtblsz >> 3)) {
 		error = pgtbl_extend(idx);
@@ -138,7 +143,7 @@ ia64_copyin(const void *src, vm_offset_t va, size_t len)
 	res = 0;
 	while (len > 0) {
 		sz = len;
-		pa = va2pa(va, &sz);
+		pa = ia64_va2pa(va, &sz);
 		if (sz == 0)
 			break;
 		bcopy(src, pa, sz);
@@ -159,7 +164,7 @@ ia64_copyout(vm_offset_t va, void *dst, size_t len)
 	res = 0;
 	while (len > 0) {
 		sz = len;
-		pa = va2pa(va, &sz);
+		pa = ia64_va2pa(va, &sz);
 		if (sz == 0)
 			break;
 		bcopy(pa, dst, sz);
@@ -170,6 +175,19 @@ ia64_copyout(vm_offset_t va, void *dst, size_t len)
 	return (res);
 }
 
+uint64_t
+ia64_loadaddr(u_int type, void *data, uint64_t addr)
+{
+	uint64_t align;
+
+	/*
+	 * Align ELF objects at PBVM page boundaries.  Align all other
+	 * objects at cache line boundaries for good measure.
+	 */
+	align = (type == LOAD_ELF) ? IA64_PBVM_PAGE_SIZE : CACHE_LINE_SIZE;
+	return ((addr + align - 1) & ~(align - 1));
+}
+
 ssize_t
 ia64_readin(int fd, vm_offset_t va, size_t len)
 {
@@ -180,7 +198,7 @@ ia64_readin(int fd, vm_offset_t va, size_t len)
 	res = 0;
 	while (len > 0) {
 		sz = len;
-		pa = va2pa(va, &sz);
+		pa = ia64_va2pa(va, &sz);
 		if (sz == 0)
 			break;
 		s = read(fd, pa, sz);
diff --git a/sys/boot/ia64/common/exec.c b/sys/boot/ia64/common/exec.c
index cb91141..6c7f027 100644
--- a/sys/boot/ia64/common/exec.c
+++ b/sys/boot/ia64/common/exec.c
@@ -36,13 +36,20 @@ __FBSDID("$FreeBSD$");
 #include <machine/ia64_cpu.h>
 #include <machine/pte.h>
 
-#include <ia64/include/vmparam.h>
-
 #include <efi.h>
 #include <efilib.h>
 
 #include "libia64.h"
 
+static u_int itr_idx = 0;
+static u_int dtr_idx = 0;
+
+static vm_offset_t ia64_text_start;
+static size_t ia64_text_size;
+
+static vm_offset_t ia64_data_start;
+static size_t ia64_data_size;
+
 static int elf64_exec(struct preloaded_file *amp);
 static int elf64_obj_exec(struct preloaded_file *amp);
 
@@ -61,6 +68,26 @@ struct file_format *file_formats[] = {
 	NULL
 };
 
+static u_int
+sz2shft(vm_offset_t ofs, vm_size_t sz)
+{
+	vm_size_t s;
+	u_int shft;
+
+	shft = 12;	/* Start with 4K */
+	s = 1 << shft;
+	while (s < sz) {
+		shft++;
+		s <<= 1;
+	}
+	do {
+		shft--;
+		s >>= 1;
+	} while (ofs & (s - 1));
+
+	return (shft);
+}
+
 /*
  * Entered with psr.ic and psr.i both zero.
  */
@@ -84,49 +111,43 @@ enter_kernel(uint64_t start, struct bootinfo *bi)
 	/* NOTREACHED */
 }
 
-static void
-mmu_wire(vm_offset_t va, vm_paddr_t pa, vm_size_t sz, u_int acc)
+static u_int
+mmu_wire(vm_offset_t va, vm_paddr_t pa, u_int pgshft, u_int acc)
 {
-	static u_int iidx = 0, didx = 0;
 	pt_entry_t pte;
-	u_int shft;
 
 	/* Round up to the smallest possible page size. */
-	if (sz < 4096)
-		sz = 4096;
-	/* Determine the exponent (base 2). */
-	shft = 0;
-	while (sz > 1) {
-		shft++;
-		sz >>= 1;
-	}
+	if (pgshft < 12)
+		pgshft = 12;
 	/* Truncate to the largest possible page size (256MB). */
-	if (shft > 28)
-		shft = 28;
+	if (pgshft > 28)
+		pgshft = 28;
 	/* Round down to a valid (mappable) page size. */
-	if (shft > 14 && (shft & 1) != 0)
-		shft--;
+	if (pgshft > 14 && (pgshft & 1) != 0)
+		pgshft--;
 
 	pte = PTE_PRESENT | PTE_MA_WB | PTE_ACCESSED | PTE_DIRTY |
 	    PTE_PL_KERN | (acc & PTE_AR_MASK) | (pa & PTE_PPN_MASK);
 
 	__asm __volatile("mov cr.ifa=%0" :: "r"(va));
-	__asm __volatile("mov cr.itir=%0" :: "r"(shft << 2));
+	__asm __volatile("mov cr.itir=%0" :: "r"(pgshft << 2));
 	__asm __volatile("srlz.d;;");
 
-	__asm __volatile("ptr.d %0,%1" :: "r"(va), "r"(shft << 2));
+	__asm __volatile("ptr.d %0,%1" :: "r"(va), "r"(pgshft << 2));
 	__asm __volatile("srlz.d;;");
-	__asm __volatile("itr.d dtr[%0]=%1" :: "r"(didx), "r"(pte));
+	__asm __volatile("itr.d dtr[%0]=%1" :: "r"(dtr_idx), "r"(pte));
 	__asm __volatile("srlz.d;;");
-	didx++;
+	dtr_idx++;
 
-	if (acc == PTE_AR_RWX) {
-		__asm __volatile("ptr.i %0,%1;;" :: "r"(va), "r"(shft << 2));
+	if (acc == PTE_AR_RWX || acc == PTE_AR_RX) {
+		__asm __volatile("ptr.i %0,%1;;" :: "r"(va), "r"(pgshft << 2));
 		__asm __volatile("srlz.i;;");
-		__asm __volatile("itr.i itr[%0]=%1;;" :: "r"(iidx), "r"(pte));
+		__asm __volatile("itr.i itr[%0]=%1;;" :: "r"(itr_idx), "r"(pte));
 		__asm __volatile("srlz.i;;");
-		iidx++;
+		itr_idx++;
 	}
+
+	return (pgshft);
 }
 
 static void
@@ -143,28 +164,43 @@ mmu_setup_legacy(uint64_t entry)
 	ia64_set_rr(IA64_RR_BASE(7), (7 << 8) | (28 << 2));
 	__asm __volatile("srlz.i;;");
 
-	mmu_wire(entry, IA64_RR_MASK(entry), 1UL << 28, PTE_AR_RWX);
+	mmu_wire(entry, IA64_RR_MASK(entry), 28, PTE_AR_RWX);
 }
 
 static void
-mmu_setup_paged(vm_offset_t pbvm_top)
+mmu_setup_paged(struct bootinfo *bi)
 {
-	vm_size_t sz;
+	void *pa;
+	size_t sz;
+	u_int shft;
 
 	ia64_set_rr(IA64_RR_BASE(IA64_PBVM_RR),
 	    (IA64_PBVM_RR << 8) | (IA64_PBVM_PAGE_SHIFT << 2));
 	__asm __volatile("srlz.i;;");
 
 	/* Wire the PBVM page table. */
-	mmu_wire(IA64_PBVM_PGTBL, (uintptr_t)ia64_pgtbl, ia64_pgtblsz,
-	    PTE_AR_RW);
-
-	/* Wire as much of the PBVM we can. This must be a power of 2. */
-	sz = pbvm_top - IA64_PBVM_BASE;
-	sz = (sz + IA64_PBVM_PAGE_MASK) & ~IA64_PBVM_PAGE_MASK;
-	while (sz & (sz - 1))
-		sz -= IA64_PBVM_PAGE_SIZE;
-	mmu_wire(IA64_PBVM_BASE, ia64_pgtbl[0], sz, PTE_AR_RWX);
+	mmu_wire(IA64_PBVM_PGTBL, (uintptr_t)ia64_pgtbl,
+	    sz2shft(IA64_PBVM_PGTBL, ia64_pgtblsz), PTE_AR_RW);
+
+	/* Wire as much of the text segment as we can. */
+	sz = ia64_text_size;	/* XXX */
+	pa = ia64_va2pa(ia64_text_start, &ia64_text_size);
+	ia64_text_size = sz;	/* XXX */
+	shft = sz2shft(ia64_text_start, ia64_text_size);
+	shft = mmu_wire(ia64_text_start, (uintptr_t)pa, shft, PTE_AR_RX);
+	ia64_copyin(&shft, (uintptr_t)&bi->bi_text_mapped, 4);
+
+	/* Wire as much of the data segment as well. */
+	sz = ia64_data_size;	/* XXX */
+	pa = ia64_va2pa(ia64_data_start, &ia64_data_size);
+	ia64_data_size = sz;	/* XXX */
+	shft = sz2shft(ia64_data_start, ia64_data_size);
+	shft = mmu_wire(ia64_data_start, (uintptr_t)pa, shft, PTE_AR_RW);
+	ia64_copyin(&shft, (uintptr_t)&bi->bi_data_mapped, 4);
+
+	/* Update the bootinfo with the number of TRs used. */
+	ia64_copyin(&itr_idx, (uintptr_t)&bi->bi_itr_used, 4);
+	ia64_copyin(&dtr_idx, (uintptr_t)&bi->bi_dtr_used, 4);
 }
 
 static int
@@ -196,7 +232,7 @@ elf64_exec(struct preloaded_file *fp)
 	if (IS_LEGACY_KERNEL())
 		mmu_setup_legacy(hdr->e_entry);
 	else
-		mmu_setup_paged((uintptr_t)(bi + 1));
+		mmu_setup_paged(bi);
 
 	enter_kernel(hdr->e_entry, bi);
 	/* NOTREACHED */
@@ -211,3 +247,20 @@ elf64_obj_exec(struct preloaded_file *fp)
 	    fp->f_name);
 	return (ENOSYS);
 }
+
+void
+ia64_loadseg(Elf_Ehdr *eh, Elf_Phdr *ph, uint64_t delta)
+{
+
+	if (eh->e_type != ET_EXEC)
+		return;
+
+	if (ph->p_flags & PF_X) {
+		ia64_text_start = ph->p_vaddr + delta;
+		ia64_text_size = ph->p_memsz;
+	} else {
+		ia64_data_start = ph->p_vaddr + delta;
+		ia64_data_size = ph->p_memsz;
+	}
+}
+
diff --git a/sys/boot/ia64/common/libia64.h b/sys/boot/ia64/common/libia64.h
index 3a0119e..29912f5 100644
--- a/sys/boot/ia64/common/libia64.h
+++ b/sys/boot/ia64/common/libia64.h
@@ -31,9 +31,9 @@
 
 #include <bootstrap.h>
 #include <ia64/include/bootinfo.h>
-#include <ia64/include/vmparam.h>
+#include <machine/vmparam.h>
 
-#define	IS_LEGACY_KERNEL()	(ia64_pgtbl == NULL || ia64_pgtblsz == 0)
+#define	IS_LEGACY_KERNEL()	(ia64_legacy_kernel)
 
 /*
  * Portability functions provided by the loader
@@ -48,15 +48,24 @@ int ia64_platform_enter(const char *);
  * Functions and variables provided by the ia64 common code
  * and shared by all loader implementations.
  */
+extern u_int ia64_legacy_kernel;
+
 extern uint64_t *ia64_pgtbl;
 extern uint32_t ia64_pgtblsz;
 
 int ia64_autoload(void);
 int ia64_bootinfo(struct preloaded_file *, struct bootinfo **);
+uint64_t ia64_loadaddr(u_int, void *, uint64_t);
+#ifdef __elfN
+void ia64_loadseg(Elf_Ehdr *, Elf_Phdr *, uint64_t);
+#else
+void ia64_loadseg(void *, void *, uint64_t);
+#endif
 
 ssize_t ia64_copyin(const void *, vm_offset_t, size_t);
 ssize_t ia64_copyout(vm_offset_t, void *, size_t);
 ssize_t ia64_readin(int, vm_offset_t, size_t);
+void *ia64_va2pa(vm_offset_t, size_t *);
 
 char *ia64_fmtdev(struct devdesc *);
 int ia64_getdev(void **, const char *, const char **);
diff --git a/sys/boot/ia64/efi/main.c b/sys/boot/ia64/efi/main.c
index 3f0b071..485a26d 100644
--- a/sys/boot/ia64/efi/main.c
+++ b/sys/boot/ia64/efi/main.c
@@ -196,9 +196,11 @@ main(int argc, CHAR16 *argv[])
 	setenv("LINES", "24", 1);	/* optional */
 
 	archsw.arch_autoload = ia64_autoload;
-	archsw.arch_getdev = ia64_getdev;
 	archsw.arch_copyin = ia64_copyin;
 	archsw.arch_copyout = ia64_copyout;
+	archsw.arch_getdev = ia64_getdev;
+	archsw.arch_loadaddr = ia64_loadaddr;
+	archsw.arch_loadseg = ia64_loadseg;
 	archsw.arch_readin = ia64_readin;
 
 	interact();			/* doesn't return */
diff --git a/sys/boot/ia64/ski/efi_stub.c b/sys/boot/ia64/ski/efi_stub.c
index 53b3867..7236c69 100644
--- a/sys/boot/ia64/ski/efi_stub.c
+++ b/sys/boot/ia64/ski/efi_stub.c
@@ -28,8 +28,8 @@
 __FBSDID("$FreeBSD$");
 
 #include <sys/types.h>
+#include <machine/bootinfo.h>
 #include <machine/efi.h>
-#include <ia64/include/bootinfo.h>
 #include <stand.h>
 #include "libski.h"
 
diff --git a/sys/boot/ia64/ski/main.c b/sys/boot/ia64/ski/main.c
index 6cfa25b..5a00d1c 100644
--- a/sys/boot/ia64/ski/main.c
+++ b/sys/boot/ia64/ski/main.c
@@ -97,11 +97,13 @@ ski_main(void)
 	    env_nounset);
 
 	setenv("LINES", "24", 1);	/* optional */
-    
+
 	archsw.arch_autoload = ia64_autoload;
-	archsw.arch_getdev = ia64_getdev;
 	archsw.arch_copyin = ia64_copyin;
 	archsw.arch_copyout = ia64_copyout;
+	archsw.arch_getdev = ia64_getdev;
+	archsw.arch_loadaddr = ia64_loadaddr;
+	archsw.arch_loadseg = ia64_loadseg;
 	archsw.arch_readin = ia64_readin;
 
 	interact();			/* doesn't return */