Kernel and modules have "set_vnet" linker set, where virtualized

global variables are placed. When a module is loaded by link_elf
linker its variables from "set_vnet" linker set are copied to the
kernel "set_vnet" ("modspace") and all references to these variables
inside the module are relocated accordingly.

The issue is when a module is loaded that has references to global
variables from another, previously loaded module: these references are
not relocated so an invalid address is used when the module tries to
access the variable. The example is V_layer3_chain, defined in ipfw
module and accessed from ipfw_nat.

The same issue is with DPCPU variables, which use "set_pcpu" linker
set.

Fix this making the link_elf linker on a module load recognize
"external" DPCPU/VNET variables defined in the previously loaded
modules and relocate them accordingly. For this set_pcpu_list and
set_vnet_list are used, where the addresses of modules' "set_pcpu" and
"set_vnet" linker sets are stored.

Note, archs that use link_elf_obj (amd64) were not affected by this
issue.

Reviewed by:	jhb, julian, zec (initial version)
MFC after:	1 month

1 files changed, 98 insertions, 1 deletions

diff --git a/sys/kern/link_elf.c b/sys/kern/link_elf.c
index 5dd0623..c37525d 100644
--- a/sys/kern/link_elf.c
+++ b/sys/kern/link_elf.c
@@ -123,6 +123,15 @@ typedef struct elf_file {
 #endif
 } *elf_file_t;
 
+struct elf_set {
+	Elf_Addr	es_start;
+	Elf_Addr	es_stop;
+	Elf_Addr	es_base;
+	TAILQ_ENTRY(elf_set)	es_link;
+};
+
+TAILQ_HEAD(elf_set_head, elf_set);
+
 #include <kern/kern_ctf.c>
 
 static int	link_elf_link_common_finish(linker_file_t);
@@ -181,6 +190,75 @@ static int	parse_dynamic(elf_file_t);
 static int	relocate_file(elf_file_t);
 static int	link_elf_preload_parse_symbols(elf_file_t);
 
+static struct elf_set_head set_pcpu_list;
+#ifdef VIMAGE
+static struct elf_set_head set_vnet_list;
+#endif
+
+static void
+elf_set_add(struct elf_set_head *list, Elf_Addr start, Elf_Addr stop, Elf_Addr base)
+{
+	struct elf_set *set, *iter;
+
+	set = malloc(sizeof(*set), M_LINKER, M_WAITOK);
+	set->es_start = start;
+	set->es_stop = stop;
+	set->es_base = base;
+
+	TAILQ_FOREACH(iter, list, es_link) {
+
+		KASSERT((set->es_start < iter->es_start && set->es_stop < iter->es_stop) ||
+		    (set->es_start > iter->es_start && set->es_stop > iter->es_stop),
+		    ("linker sets intersection: to insert: 0x%jx-0x%jx; inserted: 0x%jx-0x%jx",
+		    (uintmax_t)set->es_start, (uintmax_t)set->es_stop,
+		    (uintmax_t)iter->es_start, (uintmax_t)iter->es_stop));
+
+		if (iter->es_start > set->es_start) {
+			TAILQ_INSERT_BEFORE(iter, set, es_link);
+			break;
+		}
+	}
+
+	if (iter == NULL)
+		TAILQ_INSERT_TAIL(list, set, es_link);
+}
+
+static int
+elf_set_find(struct elf_set_head *list, Elf_Addr addr, Elf_Addr *start, Elf_Addr *base)
+{
+	struct elf_set *set;
+
+	TAILQ_FOREACH(set, list, es_link) {
+		if (addr < set->es_start)
+			return (0);
+		if (addr < set->es_stop) {
+			*start = set->es_start;
+			*base = set->es_base;
+			return (1);
+		}
+	}
+
+	return (0);
+}
+
+static void
+elf_set_delete(struct elf_set_head *list, Elf_Addr start)
+{
+	struct elf_set *set;
+
+	TAILQ_FOREACH(set, list, es_link) {
+		if (start < set->es_start)
+			break;
+		if (start == set->es_start) {
+			TAILQ_REMOVE(list, set, es_link);
+			free(set, M_LINKER);
+			return;
+		}
+	}
+	KASSERT(0, ("deleting unknown linker set (start = 0x%jx)",
+	    (uintmax_t)start));
+}
+
 #ifdef GDB
 static void	r_debug_state(struct r_debug *, struct link_map *);
 
@@ -345,6 +423,10 @@ link_elf_init(void* arg)
 
 	(void)link_elf_link_common_finish(linker_kernel_file);
 	linker_kernel_file->flags |= LINKER_FILE_LINKED;
+	TAILQ_INIT(&set_pcpu_list);
+#ifdef VIMAGE
+	TAILQ_INIT(&set_vnet_list);
+#endif
 }
 
 SYSINIT(link_elf, SI_SUB_KLD, SI_ORDER_THIRD, link_elf_init, 0);
@@ -515,6 +597,8 @@ parse_dpcpu(elf_file_t ef)
 		return (ENOSPC);
 	memcpy((void *)ef->pcpu_base, (void *)ef->pcpu_start, count);
 	dpcpu_copy((void *)ef->pcpu_base, count);
+	elf_set_add(&set_pcpu_list, ef->pcpu_start, ef->pcpu_stop,
+	    ef->pcpu_base);
 
 	return (0);
 }
@@ -544,6 +628,8 @@ parse_vnet(elf_file_t ef)
 		return (ENOSPC);
 	memcpy((void *)ef->vnet_base, (void *)ef->vnet_start, count);
 	vnet_data_copy((void *)ef->vnet_base, count);
+	elf_set_add(&set_vnet_list, ef->vnet_start, ef->vnet_stop,
+	    ef->vnet_base);
 
 	return (0);
 }
@@ -996,11 +1082,13 @@ link_elf_unload_file(linker_file_t file)
 	if (ef->pcpu_base != 0) {
 		dpcpu_free((void *)ef->pcpu_base,
 		    ef->pcpu_stop - ef->pcpu_start);
+		elf_set_delete(&set_pcpu_list, ef->pcpu_start);
 	}
 #ifdef VIMAGE
 	if (ef->vnet_base != 0) {
 		vnet_data_free((void *)ef->vnet_base,
 		    ef->vnet_stop - ef->vnet_start);
+		elf_set_delete(&set_vnet_list, ef->vnet_start);
 	}
 #endif
 #ifdef GDB
@@ -1439,6 +1527,7 @@ elf_lookup(linker_file_t lf, Elf_Size symidx, int deps)
 	elf_file_t ef = (elf_file_t)lf;
 	const Elf_Sym *sym;
 	const char *symbol;
+	Elf_Addr addr, start, base;
 
 	/* Don't even try to lookup the symbol if the index is bogus. */
 	if (symidx >= ef->nchains)
@@ -1470,7 +1559,15 @@ elf_lookup(linker_file_t lf, Elf_Size symidx, int deps)
 	if (*symbol == 0)
 		return (0);
 
-	return ((Elf_Addr)linker_file_lookup_symbol(lf, symbol, deps));
+	addr = ((Elf_Addr)linker_file_lookup_symbol(lf, symbol, deps));
+
+	if (elf_set_find(&set_pcpu_list, addr, &start, &base))
+		addr = addr - start + base;
+#ifdef VIMAGE
+	else if (elf_set_find(&set_vnet_list, addr, &start, &base))
+		addr = addr - start + base;
+#endif
+	return addr;
 }
 
 static void