MFaltix:

Add support for Pre-Boot Virtual Memory (PBVM) to the loader.

PBVM allows us to link the kernel at a fixed virtual address without
having to make any assumptions about the physical memory layout. On
the SGI Altix 350 for example, there's no usuable physical memory
below 192GB. Also, the PBVM allows us to control better where we're
going to physically load the kernel and its modules so that we can
make sure we load the kernel in memory that's close to the BSP.

The PBVM is managed by a simple page table. The minimum size of the
page table is 4KB (EFI page size) and the maximum is currently set
to 1MB. A page in the PBVM is 64KB, as that's the maximum alignment
one can specify in a linker script. The bottom line is that PBVM is
between 64KB and 8GB in size.

The loader maps the PBVM page table at a fixed virtual address and
using a single translations. The PBVM itself is also mapped using a
single translation for a maximum of 32MB.

While here, increase the heap in the EFI loader from 512KB to 2MB
and set the stage for supporting relocatable modules.

4 files changed, 250 insertions, 52 deletions

diff --git a/sys/boot/ia64/common/bootinfo.c b/sys/boot/ia64/common/bootinfo.c
index 6f892a9..076c953 100644
--- a/sys/boot/ia64/common/bootinfo.c
+++ b/sys/boot/ia64/common/bootinfo.c
@@ -226,7 +226,7 @@ bi_copymodules(vm_offset_t addr)
  * - Module metadata are formatted and placed in kernel space.
  */
 int
-bi_load(struct preloaded_file *fp, uint64_t *bi_addr)
+ia64_bootinfo(struct preloaded_file *fp, struct bootinfo **res)
 {
 	struct bootinfo bi;
 	struct preloaded_file *xp;
@@ -234,7 +234,9 @@ bi_load(struct preloaded_file *fp, uint64_t *bi_addr)
 	struct devdesc *rootdev;
 	char *rootdevname;
 	vm_offset_t addr, ssym, esym;
+	int error;
 
+	*res = NULL;
 	bzero(&bi, sizeof(struct bootinfo));
 	bi.bi_magic = BOOTINFO_MAGIC;
 	bi.bi_version = 1;
@@ -289,8 +291,28 @@ bi_load(struct preloaded_file *fp, uint64_t *bi_addr)
 		bi.bi_envp = 0;
 	}
 
-	addr = (addr + PAGE_MASK) & ~PAGE_MASK;
+	addr = (addr + 15) & ~15;
 	bi.bi_kernend = addr;
 
-	return (ldr_bootinfo(&bi, bi_addr));
+	error = ia64_platform_bootinfo(&bi, res);
+	if (error)
+		return (error);
+
+	if (IS_LEGACY_KERNEL()) {
+		if (*res == NULL)
+			return (EDOOFUS);
+
+		bcopy(&bi, *res, sizeof(bi));
+		return (0);
+	}
+
+	bi.bi_pbvm_pgtbl = (uintptr_t)ia64_pgtbl;
+	bi.bi_pbvm_pgtblsz = ia64_pgtblsz;
+	ia64_copyin((void *)bi.bi_memmap, addr, bi.bi_memmap_size);
+	bi.bi_memmap = addr;
+	addr = (addr + bi.bi_memmap_size + 15) & ~15;
+	bi.bi_kernend = addr + sizeof(bi);	
+	ia64_copyin(&bi, addr, sizeof(bi));
+	*res = (void *)addr;
+	return (0);
 }
diff --git a/sys/boot/ia64/common/copy.c b/sys/boot/ia64/common/copy.c
index 8d8dab6..93ef77b 100644
--- a/sys/boot/ia64/common/copy.c
+++ b/sys/boot/ia64/common/copy.c
@@ -32,17 +32,98 @@ __FBSDID("$FreeBSD$");
 
 #include "libia64.h"
 
+uint64_t *ia64_pgtbl;
+uint32_t ia64_pgtblsz;
+
+static int
+pgtbl_extend(u_int idx)
+{
+	uint64_t *pgtbl;
+	uint32_t pgtblsz;
+	u_int pot;
+
+	pgtblsz = (idx + 1) << 3;
+
+	/* The minimum size is 4KB. */
+	if (pgtblsz < 4096)
+		pgtblsz = 4096;
+
+	/* Find the next higher power of 2. */
+	pgtblsz--;
+	for (pot = 1; pot < 32; pot <<= 1)
+		pgtblsz = pgtblsz | (pgtblsz >> pot);
+	pgtblsz++;
+
+	/* The maximum size is 1MB. */
+	if (pgtblsz > 1048576)
+		return (ENOMEM);
+
+	/* Make sure the size is a valid (mappable) page size. */
+	if (pgtblsz == 32*1024 || pgtblsz == 128*1024 || pgtblsz == 512*1024)
+		pgtblsz <<= 1;
+
+	/* Allocate naturally aligned memory. */
+	pgtbl = (void *)ia64_platform_alloc(0, pgtblsz);
+	if (pgtbl == NULL)
+		return (ENOMEM);
+
+	/* Initialize new page table. */
+	if (ia64_pgtbl != NULL && ia64_pgtbl != pgtbl)
+		bcopy(ia64_pgtbl, pgtbl, ia64_pgtblsz);
+	bzero(pgtbl + (ia64_pgtblsz >> 3), pgtblsz - ia64_pgtblsz);
+
+	if (ia64_pgtbl != NULL && ia64_pgtbl != pgtbl)
+		ia64_platform_free(0, (uintptr_t)ia64_pgtbl, ia64_pgtblsz);
+
+	ia64_pgtbl = pgtbl;
+	ia64_pgtblsz = pgtblsz;
+	return (0);
+}
+
 static void *
 va2pa(vm_offset_t va, size_t *len)
 {
 	uint64_t pa;
+	u_int idx, ofs;
+	int error;
 
+	/* Backward compatibility. */
 	if (va >= IA64_RR_BASE(7)) {
 		pa = IA64_RR_MASK(va);
 		return ((void *)pa);
 	}
 
-	printf("\n%s: va=%lx, *len=%lx\n", __func__, va, *len);
+	if (va < IA64_PBVM_BASE) {
+		error = EINVAL;
+		goto fail;
+	}
+
+	idx = (va - IA64_PBVM_BASE) >> IA64_PBVM_PAGE_SHIFT;
+	if (idx >= (ia64_pgtblsz >> 3)) {
+		error = pgtbl_extend(idx);
+		if (error)
+			goto fail;
+	}
+
+	ofs = va & IA64_PBVM_PAGE_MASK;
+	pa = ia64_pgtbl[idx];
+	if (pa == 0) {
+		pa = ia64_platform_alloc(va - ofs, IA64_PBVM_PAGE_SIZE);
+		if (pa == 0) {
+			error = ENOMEM;
+			goto fail;
+		}
+		ia64_pgtbl[idx] = pa;
+	}
+	pa += ofs;
+
+	/* We can not cross page boundaries (in general). */
+	if (*len + ofs > IA64_PBVM_PAGE_SIZE)
+		*len = IA64_PBVM_PAGE_SIZE - ofs;
+
+	return ((void *)pa);
+
+ fail:
 	*len = 0;
 	return (NULL);
 }
diff --git a/sys/boot/ia64/common/exec.c b/sys/boot/ia64/common/exec.c
index 1a8094f..cb91141 100644
--- a/sys/boot/ia64/common/exec.c
+++ b/sys/boot/ia64/common/exec.c
@@ -36,25 +36,36 @@ __FBSDID("$FreeBSD$");
 #include <machine/ia64_cpu.h>
 #include <machine/pte.h>
 
-#include <ia64/include/bootinfo.h>
 #include <ia64/include/vmparam.h>
 
 #include <efi.h>
 #include <efilib.h>
 
-#include "bootstrap.h"
+#include "libia64.h"
 
-#define _KERNEL
+static int elf64_exec(struct preloaded_file *amp);
+static int elf64_obj_exec(struct preloaded_file *amp);
 
-static int	elf64_exec(struct preloaded_file *amp);
+static struct file_format ia64_elf = {
+	elf64_loadfile,
+	elf64_exec
+};
+static struct file_format ia64_elf_obj = {
+	elf64_obj_loadfile,
+	elf64_obj_exec
+};
 
-struct file_format ia64_elf = { elf64_loadfile, elf64_exec };
+struct file_format *file_formats[] = {
+	&ia64_elf,
+	&ia64_elf_obj,
+	NULL
+};
 
 /*
  * Entered with psr.ic and psr.i both zero.
  */
-void
-enter_kernel(uint64_t start, uint64_t bi)
+static void
+enter_kernel(uint64_t start, struct bootinfo *bi)
 {
 
 	__asm __volatile("srlz.i;;");
@@ -73,53 +84,130 @@ enter_kernel(uint64_t start, uint64_t bi)
 	/* NOTREACHED */
 }
 
-static int
-elf64_exec(struct preloaded_file *fp)
+static void
+mmu_wire(vm_offset_t va, vm_paddr_t pa, vm_size_t sz, u_int acc)
 {
-	struct file_metadata	*md;
-	Elf_Ehdr		*hdr;
-	pt_entry_t		pte;
-	uint64_t		bi_addr;
+	static u_int iidx = 0, didx = 0;
+	pt_entry_t pte;
+	u_int shft;
+
+	/* Round up to the smallest possible page size. */
+	if (sz < 4096)
+		sz = 4096;
+	/* Determine the exponent (base 2). */
+	shft = 0;
+	while (sz > 1) {
+		shft++;
+		sz >>= 1;
+	}
+	/* Truncate to the largest possible page size (256MB). */
+	if (shft > 28)
+		shft = 28;
+	/* Round down to a valid (mappable) page size. */
+	if (shft > 14 && (shft & 1) != 0)
+		shft--;
 
-	md = file_findmetadata(fp, MODINFOMD_ELFHDR);
-	if (md == NULL)
-		return (EINVAL);
-	hdr = (Elf_Ehdr *)&(md->md_data);
-
-	bi_load(fp, &bi_addr);
-
-	printf("Entering %s at 0x%lx...\n", fp->f_name, hdr->e_entry);
-
-	ldr_enter(fp->f_name);
+	pte = PTE_PRESENT | PTE_MA_WB | PTE_ACCESSED | PTE_DIRTY |
+	    PTE_PL_KERN | (acc & PTE_AR_MASK) | (pa & PTE_PPN_MASK);
+
+	__asm __volatile("mov cr.ifa=%0" :: "r"(va));
+	__asm __volatile("mov cr.itir=%0" :: "r"(shft << 2));
+	__asm __volatile("srlz.d;;");
+
+	__asm __volatile("ptr.d %0,%1" :: "r"(va), "r"(shft << 2));
+	__asm __volatile("srlz.d;;");
+	__asm __volatile("itr.d dtr[%0]=%1" :: "r"(didx), "r"(pte));
+	__asm __volatile("srlz.d;;");
+	didx++;
+
+	if (acc == PTE_AR_RWX) {
+		__asm __volatile("ptr.i %0,%1;;" :: "r"(va), "r"(shft << 2));
+		__asm __volatile("srlz.i;;");
+		__asm __volatile("itr.i itr[%0]=%1;;" :: "r"(iidx), "r"(pte));
+		__asm __volatile("srlz.i;;");
+		iidx++;
+	}
+}
 
-	__asm __volatile("rsm psr.ic|psr.i;;");
-	__asm __volatile("srlz.i;;");
+static void
+mmu_setup_legacy(uint64_t entry)
+{
 
 	/*
 	 * Region 6 is direct mapped UC and region 7 is direct mapped
 	 * WC. The details of this is controlled by the Alt {I,D}TLB
-	 * handlers. Here we just make sure that they have the largest 
+	 * handlers. Here we just make sure that they have the largest
 	 * possible page size to minimise TLB usage.
 	 */
 	ia64_set_rr(IA64_RR_BASE(6), (6 << 8) | (28 << 2));
 	ia64_set_rr(IA64_RR_BASE(7), (7 << 8) | (28 << 2));
+	__asm __volatile("srlz.i;;");
 
-	pte = PTE_PRESENT | PTE_MA_WB | PTE_ACCESSED | PTE_DIRTY |
-	    PTE_PL_KERN | PTE_AR_RWX | PTE_ED;
-	pte |= IA64_RR_MASK(hdr->e_entry) & PTE_PPN_MASK;
+	mmu_wire(entry, IA64_RR_MASK(entry), 1UL << 28, PTE_AR_RWX);
+}
 
-	__asm __volatile("mov cr.ifa=%0" :: "r"(hdr->e_entry));
-	__asm __volatile("mov cr.itir=%0" :: "r"(28 << 2));
-	__asm __volatile("ptr.i %0,%1" :: "r"(hdr->e_entry), "r"(28<<2));
-	__asm __volatile("ptr.d %0,%1" :: "r"(hdr->e_entry), "r"(28<<2));
-	__asm __volatile("srlz.i;;");
-	__asm __volatile("itr.i itr[%0]=%1;;" :: "r"(0), "r"(pte));
+static void
+mmu_setup_paged(vm_offset_t pbvm_top)
+{
+	vm_size_t sz;
+
+	ia64_set_rr(IA64_RR_BASE(IA64_PBVM_RR),
+	    (IA64_PBVM_RR << 8) | (IA64_PBVM_PAGE_SHIFT << 2));
 	__asm __volatile("srlz.i;;");
-	__asm __volatile("itr.d dtr[%0]=%1;;" :: "r"(0), "r"(pte));
+
+	/* Wire the PBVM page table. */
+	mmu_wire(IA64_PBVM_PGTBL, (uintptr_t)ia64_pgtbl, ia64_pgtblsz,
+	    PTE_AR_RW);
+
+	/* Wire as much of the PBVM we can. This must be a power of 2. */
+	sz = pbvm_top - IA64_PBVM_BASE;
+	sz = (sz + IA64_PBVM_PAGE_MASK) & ~IA64_PBVM_PAGE_MASK;
+	while (sz & (sz - 1))
+		sz -= IA64_PBVM_PAGE_SIZE;
+	mmu_wire(IA64_PBVM_BASE, ia64_pgtbl[0], sz, PTE_AR_RWX);
+}
+
+static int
+elf64_exec(struct preloaded_file *fp)
+{
+	struct bootinfo *bi;
+	struct file_metadata *md;
+	Elf_Ehdr *hdr;
+	int error;
+
+	md = file_findmetadata(fp, MODINFOMD_ELFHDR);
+	if (md == NULL)
+		return (EINVAL);
+
+	error = ia64_bootinfo(fp, &bi);
+	if (error)
+		return (error);
+
+	hdr = (Elf_Ehdr *)&(md->md_data);
+	printf("Entering %s at 0x%lx...\n", fp->f_name, hdr->e_entry);
+
+	error = ia64_platform_enter(fp->f_name);
+	if (error)
+		return (error);
+
+	__asm __volatile("rsm psr.ic|psr.i;;");
 	__asm __volatile("srlz.i;;");
 
-	enter_kernel(hdr->e_entry, bi_addr);
+	if (IS_LEGACY_KERNEL())
+		mmu_setup_legacy(hdr->e_entry);
+	else
+		mmu_setup_paged((uintptr_t)(bi + 1));
 
+	enter_kernel(hdr->e_entry, bi);
 	/* NOTREACHED */
-	return (0);
+	return (EDOOFUS);
+}
+
+static int
+elf64_obj_exec(struct preloaded_file *fp)
+{
+
+	printf("%s called for preloaded file %p (=%s):\n", __func__, fp,
+	    fp->f_name);
+	return (ENOSYS);
 }
diff --git a/sys/boot/ia64/common/libia64.h b/sys/boot/ia64/common/libia64.h
index a196002..3a0119e 100644
--- a/sys/boot/ia64/common/libia64.h
+++ b/sys/boot/ia64/common/libia64.h
@@ -31,28 +31,35 @@
 
 #include <bootstrap.h>
 #include <ia64/include/bootinfo.h>
+#include <ia64/include/vmparam.h>
+
+#define	IS_LEGACY_KERNEL()	(ia64_pgtbl == NULL || ia64_pgtblsz == 0)
 
 /*
  * Portability functions provided by the loader
  * implementation specific to the platform.
  */
-extern uint64_t ldr_alloc(vm_offset_t);
-extern int ldr_bootinfo(struct bootinfo *, uint64_t *);
-extern int ldr_enter(const char *);
+vm_paddr_t ia64_platform_alloc(vm_offset_t, vm_size_t);
+void ia64_platform_free(vm_offset_t, vm_paddr_t, vm_size_t);
+int ia64_platform_bootinfo(struct bootinfo *, struct bootinfo **);
+int ia64_platform_enter(const char *);
 
 /*
  * Functions and variables provided by the ia64 common code
  * and shared by all loader implementations.
  */
+extern uint64_t *ia64_pgtbl;
+extern uint32_t ia64_pgtblsz;
 
-extern int ia64_autoload(void);
+int ia64_autoload(void);
+int ia64_bootinfo(struct preloaded_file *, struct bootinfo **);
 
-extern ssize_t ia64_copyin(const void *, vm_offset_t, size_t);
-extern ssize_t ia64_copyout(vm_offset_t, void *, size_t);
-extern ssize_t ia64_readin(int, vm_offset_t, size_t);
+ssize_t ia64_copyin(const void *, vm_offset_t, size_t);
+ssize_t ia64_copyout(vm_offset_t, void *, size_t);
+ssize_t ia64_readin(int, vm_offset_t, size_t);
 
-extern char *ia64_fmtdev(struct devdesc *);
-extern int ia64_getdev(void **, const char *, const char **);
-extern int ia64_setcurrdev(struct env_var *, int, const void *);
+char *ia64_fmtdev(struct devdesc *);
+int ia64_getdev(void **, const char *, const char **);
+int ia64_setcurrdev(struct env_var *, int, const void *);
 
 #endif /* !_LIBIA64_H_ */