6 files changed, 312 insertions, 72 deletions

diff --git a/sys/arm/arm/elf_machdep.c b/sys/arm/arm/elf_machdep.c
index eafd8a06e7c9..86eef70985ba 100644
--- a/sys/arm/arm/elf_machdep.c
+++ b/sys/arm/arm/elf_machdep.c
@@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$");
 
 #include <machine/elf.h>
 #include <machine/md_var.h>
+#include <machine/stack.h>
 #ifdef VFP
 #include <machine/vfp.h>
 #endif
@@ -309,12 +310,21 @@ elf_cpu_load_file(linker_file_t lf)
 	cpu_l2cache_wb_range((vm_offset_t)lf->address, (vm_size_t)lf->size);
 	cpu_icache_sync_range((vm_offset_t)lf->address, (vm_size_t)lf->size);
 #endif
+
+	/*
+	 * Inform the stack(9) code of the new module, so it can acquire its
+	 * per-module unwind data.
+	 */
+	unwind_module_loaded(lf);
+
 	return (0);
 }
 
 int
-elf_cpu_unload_file(linker_file_t lf __unused)
+elf_cpu_unload_file(linker_file_t lf)
 {
 
+	/* Inform the stack(9) code that this module is gone. */
+	unwind_module_unloaded(lf);
 	return (0);
 }
diff --git a/sys/arm/arm/unwind.c b/sys/arm/arm/unwind.c
index cba5f033ba1d..bf8ffddfd2c2 100644
--- a/sys/arm/arm/unwind.c
+++ b/sys/arm/arm/unwind.c
@@ -32,8 +32,11 @@
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
-#include <sys/systm.h>
+#include <sys/kernel.h>
 #include <sys/linker.h>
+#include <sys/malloc.h>
+#include <sys/queue.h>
+#include <sys/systm.h>
 
 #include <machine/machdep.h>
 #include <machine/stack.h>
@@ -94,79 +97,236 @@ struct unwind_idx {
 	uint32_t insn;
 };
 
-/* Expand a 31-bit signed value to a 32-bit signed value */
-static __inline int32_t
-expand_prel31(uint32_t prel31)
+/*
+ * Local cache of unwind info for loaded modules.
+ *
+ * To unwind the stack through the code in a loaded module, we need to access
+ * the module's exidx unwind data.  To locate that data, one must search the
+ * elf section headers for the SHT_ARM_EXIDX section.  Those headers are
+ * available at the time the module is being loaded, but are discarded by time
+ * the load process has completed.  Code in kern/link_elf.c locates the data we
+ * need and stores it into the linker_file structure before calling the arm
+ * machdep routine for handling loaded modules (in arm/elf_machdep.c).  That
+ * function calls into this code to pass along the unwind info, which we save
+ * into one of these module_info structures.
+ *
+ * Because we have to help stack(9) gather stack info at any time, including in
+ * contexts where sleeping is not allowed, we cannot use linker_file_foreach()
+ * to walk the kernel's list of linker_file structs, because doing so requires
+ * acquiring an exclusive sx_lock.  So instead, we keep a local list of these
+ * structures, one for each loaded module (and one for the kernel itself that we
+ * synthesize at init time).  New entries are added to the end of this list as
+ * needed, but entries are never deleted from the list.  Instead, they are
+ * cleared out in-place to mark them as unused.  That means the code doing stack
+ * unwinding can always safely walk the list without locking, because the
+ * structure of the list never changes in a way that would cause the walker to
+ * follow a bad link.
+ *
+ * A cleared-out entry on the list has module start=UINTPTR_MAX and end=0, so
+ * start <= addr < end cannot be true for any value of addr being searched for.
+ * We also don't have to worry about races where we look up the unwind info just
+ * before a module is unloaded and try to access it concurrently with or just
+ * after the unloading happens in another thread, because that means the path of
+ * execution leads through a now-unloaded module, and that's already well into
+ * undefined-behavior territory.
+ *
+ * List entries marked as unused get reused when new modules are loaded.  We
+ * don't worry about holding a few unused bytes of memory in the list after
+ * unloading a module.
+ */
+struct module_info {
+	uintptr_t	module_start;   /* Start of loaded module */
+	uintptr_t	module_end;     /* End of loaded module */
+	uintptr_t	exidx_start;    /* Start of unwind data */
+	uintptr_t	exidx_end;      /* End of unwind data */
+	STAILQ_ENTRY(module_info)
+			link;           /* Link to next entry */
+};
+static STAILQ_HEAD(, module_info) module_list;
+
+/*
+ * Hide ugly casting in somewhat-less-ugly macros.
+ *  CADDR - cast a pointer or number to caddr_t.
+ *  UADDR - cast a pointer or number to uintptr_t.
+ */
+#define	CADDR(addr)	((caddr_t)(void*)(uintptr_t)(addr))
+#define	UADDR(addr)	((uintptr_t)(addr))
+
+/*
+ * Clear the info in an existing module_info entry on the list.  The
+ * module_start/end addresses are set to values that cannot match any real
+ * memory address.  The entry remains on the list, but will be ignored until it
+ * is populated with new data.
+ */
+static void
+clear_module_info(struct module_info *info)
 {
+	info->module_start = UINTPTR_MAX;
+	info->module_end   = 0;
+}
 
-	return ((int32_t)(prel31 & 0x7fffffffu) << 1) / 2;
+/*
+ * Populate an existing module_info entry (which is already on the list) with
+ * the info for a new module.
+ */
+static void
+populate_module_info(struct module_info *info, linker_file_t lf)
+{
+
+	/*
+	 * Careful!  The module_start and module_end fields must not be set
+	 * until all other data in the structure is valid.
+	 */
+	info->exidx_start  = UADDR(lf->exidx_addr);
+	info->exidx_end    = UADDR(lf->exidx_addr) + lf->exidx_size;
+	info->module_start = UADDR(lf->address);
+	info->module_end   = UADDR(lf->address) + lf->size;
 }
 
-struct search_context {
-	uint32_t addr;
-	caddr_t exidx_start;
-	caddr_t exidx_end;
-};
+/*
+ * Create a new empty module_info entry and add it to the tail of the list.
+ */
+static struct module_info *
+create_module_info(void)
+{
+	struct module_info *info;
 
-static int
-module_search(linker_file_t lf, void *context)
+	info = malloc(sizeof(*info), M_CACHE, M_WAITOK | M_ZERO);
+	clear_module_info(info);
+	STAILQ_INSERT_TAIL(&module_list, info, link);
+	return (info);
+}
+
+/*
+ * Search for a module_info entry on the list whose address range contains the
+ * given address.  If the search address is zero (no module will be loaded at
+ * zero), then we're looking for an empty item to reuse, which is indicated by
+ * module_start being set to UINTPTR_MAX in the entry.
+ */
+static struct module_info *
+find_module_info(uintptr_t addr)
 {
-	struct search_context *sc = context;
-	linker_symval_t symval;
-	c_linker_sym_t sym;
-
-	if (lf->address <= (caddr_t)sc->addr &&
-	    (lf->address + lf->size) >= (caddr_t)sc->addr) {
-		if ((LINKER_LOOKUP_SYMBOL(lf, "__exidx_start", &sym) == 0 ||
-		    LINKER_LOOKUP_SYMBOL(lf, "exidx_start", &sym) == 0) &&
-		    LINKER_SYMBOL_VALUES(lf, sym, &symval) == 0)
-			sc->exidx_start = symval.value;
-
-		if ((LINKER_LOOKUP_SYMBOL(lf, "__exidx_end", &sym) == 0 ||
-		    LINKER_LOOKUP_SYMBOL(lf, "exidx_end", &sym) == 0) &&
-		    LINKER_SYMBOL_VALUES(lf, sym, &symval) == 0)
-			sc->exidx_end = symval.value;
-
-		if (sc->exidx_start != NULL && sc->exidx_end != NULL)
-			return (1);
-		panic("Invalid module %s, no unwind tables\n", lf->filename);
+	struct module_info *info;
+
+	STAILQ_FOREACH(info, &module_list, link) {
+		if ((addr >= info->module_start && addr < info->module_end) ||
+		    (addr == 0 && info->module_start == UINTPTR_MAX))
+			return (info);
 	}
+	return (NULL);
+}
+
+/*
+ * Handle the loading of a new module by populating a module_info for it.  This
+ * is called for both preloaded and dynamically loaded modules.
+ */
+void
+unwind_module_loaded(struct linker_file *lf)
+{
+	struct module_info *info;
+
+	/*
+	 * A module that contains only data may have no unwind info; don't
+	 * create any module info for it.
+	 */
+	if (lf->exidx_size == 0)
+		return;
+
+	/*
+	 * Find an unused entry in the existing list to reuse.  If we don't find
+	 * one, create a new one and link it into the list.  This is the only
+	 * place the module_list is modified.  Adding a new entry to the list
+	 * will not perturb any other threads currently walking the list.  This
+	 * function is invoked while kern_linker is still holding its lock
+	 * to prevent its module list from being modified, so we don't have to
+	 * worry about racing other threads doing an insert concurrently.
+	 */
+	if ((info = find_module_info(0)) == NULL) {
+		info = create_module_info();
+	}
+	populate_module_info(info, lf);
+}
+
+/* Handle the unloading of a module. */
+void
+unwind_module_unloaded(struct linker_file *lf)
+{
+	struct module_info *info;
+
+	/*
+	 * A module that contains only data may have no unwind info and there
+	 * won't be a list entry for it.
+	 */
+	if (lf->exidx_size == 0)
+		return;
+
+	/*
+	 * When a module is unloaded, we clear the info out of its entry in the
+	 * module list, making that entry available for later reuse.
+	 */
+	if ((info = find_module_info(UADDR(lf->address))) == NULL) {
+		printf("arm unwind: module '%s' not on list at unload time\n",
+		    lf->filename);
+		return;
+	}
+	clear_module_info(info);
+}
+
+/*
+ * Initialization must run fairly early, as soon as malloc(9) is available, and
+ * definitely before witness, which uses stack(9).  We synthesize a module_info
+ * entry for the kernel, because unwind_module_loaded() doesn't get called for
+ * it.  Also, it is unlike other modules in that the elf metadata for locating
+ * the unwind tables might be stripped, so instead we have to use the
+ * _exidx_start/end symbols created by ldscript.arm.
+ */
+static int
+module_info_init(void *arg __unused)
+{
+	struct linker_file thekernel;
+
+	STAILQ_INIT(&module_list);
+
+	thekernel.filename   = "kernel";
+	thekernel.address    = CADDR(&_start);
+	thekernel.size       = UADDR(&_end) - UADDR(&_start);
+	thekernel.exidx_addr = CADDR(&_exidx_start);
+	thekernel.exidx_size = UADDR(&_exidx_end) - UADDR(&_exidx_start);
+	populate_module_info(create_module_info(), &thekernel);
+
 	return (0);
 }
+SYSINIT(unwind_init, SI_SUB_KMEM, SI_ORDER_ANY, module_info_init, NULL);
+
+/* Expand a 31-bit signed value to a 32-bit signed value */
+static __inline int32_t
+expand_prel31(uint32_t prel31)
+{
+
+	return ((int32_t)(prel31 & 0x7fffffffu) << 1) / 2;
+}
 
 /*
  * Perform a binary search of the index table to find the function
  * with the largest address that doesn't exceed addr.
  */
 static struct unwind_idx *
-find_index(uint32_t addr, int search_modules)
+find_index(uint32_t addr)
 {
-	struct search_context sc;
-	caddr_t idx_start, idx_end;
+	struct module_info *info;
 	unsigned int min, mid, max;
 	struct unwind_idx *start;
 	struct unwind_idx *item;
 	int32_t prel31_addr;
 	uint32_t func_addr;
 
-	start = (struct unwind_idx *)&_exidx_start;
-	idx_start = (caddr_t)&_exidx_start;
-	idx_end = (caddr_t)&_exidx_end;
-
-	/* This may acquire a lock */
-	if (search_modules) {
-		bzero(&sc, sizeof(sc));
-		sc.addr = addr;
-		if (linker_file_foreach(module_search, &sc) != 0 &&
-		   sc.exidx_start != NULL && sc.exidx_end != NULL) {
-			start = (struct unwind_idx *)sc.exidx_start;
-			idx_start = sc.exidx_start;
-			idx_end = sc.exidx_end;
-		}
-	}
+	info = find_module_info(addr);
+	if (info == NULL)
+		return NULL;
 
 	min = 0;
-	max = (idx_end - idx_start) / sizeof(struct unwind_idx);
+	max = (info->exidx_end - info->exidx_start) / sizeof(struct unwind_idx);
+	start = (struct unwind_idx *)CADDR(info->exidx_start);
 
 	while (min != max) {
 		mid = min + (max - min + 1) / 2;
@@ -377,11 +537,17 @@ unwind_tab(struct unwind_state *state)
 	return 0;
 }
 
+/*
+ * Unwind a single stack frame.
+ * Return 0 on success or 1 if the stack cannot be unwound any further.
+ *
+ * XXX The can_lock argument is no longer germane; a sweep of callers should be
+ * made to remove it after this new code has proven itself for a while.
+ */
 int
-unwind_stack_one(struct unwind_state *state, int can_lock)
+unwind_stack_one(struct unwind_state *state, int can_lock __unused)
 {
 	struct unwind_idx *index;
-	int finished;
 
 	/* Reset the mask of updated registers */
 	state->update_mask = 0;
@@ -390,26 +556,20 @@ unwind_stack_one(struct unwind_state *state, int can_lock)
 	state->start_pc = state->registers[PC];
 
 	/* Find the item to run */
-	index = find_index(state->start_pc, can_lock);
+	index = find_index(state->start_pc);
+	if (index == NULL || index->insn == EXIDX_CANTUNWIND)
+		return 1;
 
-	finished = 0;
-	if (index->insn != EXIDX_CANTUNWIND) {
-		if (index->insn & (1U << 31)) {
-			/* The data is within the instruction */
-			state->insn = &index->insn;
-		} else {
-			/* A prel31 offset to the unwind table */
-			state->insn = (uint32_t *)
-			    ((uintptr_t)&index->insn +
-			     expand_prel31(index->insn));
-		}
-		/* Run the unwind function */
-		finished = unwind_tab(state);
+	if (index->insn & (1U << 31)) {
+		/* The data is within the instruction */
+		state->insn = &index->insn;
+	} else {
+		/* A prel31 offset to the unwind table */
+		state->insn = (uint32_t *)
+		    ((uintptr_t)&index->insn +
+		     expand_prel31(index->insn));
 	}
 
-	/* This is the top of the stack, finish */
-	if (index->insn == EXIDX_CANTUNWIND)
-		finished = 1;
-
-	return (finished);
+	/* Run the unwind function, return its finished/not-finished status. */
+	return (unwind_tab(state));
 }
diff --git a/sys/arm/include/stack.h b/sys/arm/include/stack.h
index 9721b0ca0403..4bc384f775bc 100644
--- a/sys/arm/include/stack.h
+++ b/sys/arm/include/stack.h
@@ -55,6 +55,14 @@ struct unwind_state {
 #define	LR	14
 #define	PC	15
 
+#ifdef _KERNEL
+
 int unwind_stack_one(struct unwind_state *, int);
 
+struct linker_file;
+void unwind_module_loaded(struct linker_file *);
+void unwind_module_unloaded(struct linker_file *);
+
+#endif
+
 #endif /* !_MACHINE_STACK_H_ */
diff --git a/sys/kern/kern_linker.c b/sys/kern/kern_linker.c
index 682bb9aae0d5..31934a679082 100644
--- a/sys/kern/kern_linker.c
+++ b/sys/kern/kern_linker.c
@@ -624,6 +624,10 @@ linker_make_file(const char *pathname, linker_class_t lc)
 	lf->ndeps = 0;
 	lf->deps = NULL;
 	lf->loadcnt = ++loadcnt;
+#ifdef __arm__
+	lf->exidx_addr = 0;
+	lf->exidx_size = 0;
+#endif
 	STAILQ_INIT(&lf->common);
 	TAILQ_INIT(&lf->modules);
 	TAILQ_INSERT_TAIL(&linker_files, lf, link);
diff --git a/sys/kern/link_elf.c b/sys/kern/link_elf.c
index 9568dbee068c..21c604d0bb2c 100644
--- a/sys/kern/link_elf.c
+++ b/sys/kern/link_elf.c
@@ -773,6 +773,50 @@ preload_protect(elf_file_t ef, vm_prot_t prot)
 #endif
 }
 
+#ifdef __arm__
+/*
+ * Locate the ARM exception/unwind table info for DDB and stack(9) use by
+ * searching for the section header that describes it.  There may be no unwind
+ * info, for example in a module containing only data.
+ */
+static void
+link_elf_locate_exidx(linker_file_t lf, Elf_Shdr *shdr, int nhdr)
+{
+	int i;
+
+	for (i = 0; i < nhdr; i++) {
+		if (shdr[i].sh_type == SHT_ARM_EXIDX) {
+			lf->exidx_addr = shdr[i].sh_addr + lf->address;
+			lf->exidx_size = shdr[i].sh_size;
+			break;
+		}
+	}
+}
+
+/*
+ * Locate the section headers metadata in a preloaded module, then use it to
+ * locate the exception/unwind table in the module.  The size of the metadata
+ * block is stored in a uint32 word immediately before the data itself, and a
+ * comment in preload_search_info() says it is safe to rely on that.
+ */
+static void
+link_elf_locate_exidx_preload(struct linker_file *lf, caddr_t modptr)
+{
+	uint32_t *modinfo;
+	Elf_Shdr *shdr;
+	uint32_t  nhdr;
+
+	modinfo = (uint32_t *)preload_search_info(modptr,
+	    MODINFO_METADATA | MODINFOMD_SHDR);
+	if (modinfo != NULL) {
+		shdr = (Elf_Shdr *)modinfo;
+		nhdr = modinfo[-1] / sizeof(Elf_Shdr);
+		link_elf_locate_exidx(lf, shdr, nhdr);
+	}
+}
+
+#endif /* __arm__ */
+
 static int
 link_elf_link_preload(linker_class_t cls, const char *filename,
     linker_file_t *result)
@@ -828,6 +872,10 @@ link_elf_link_preload(linker_class_t cls, const char *filename,
 		lf->ctors_size = *ctors_sizep;
 	}
 
+#ifdef __arm__
+	link_elf_locate_exidx_preload(lf, modptr);
+#endif
+
 	error = parse_dynamic(ef);
 	if (error == 0)
 		error = parse_dpcpu(ef);
@@ -1225,6 +1273,11 @@ link_elf_load_file(linker_class_t cls, const char* filename,
 	ef->ddbstrtab = ef->strbase;
 
 nosyms:
+
+#ifdef __arm__
+	link_elf_locate_exidx(lf, shdr, hdr->e_shnum);
+#endif
+
 	error = link_elf_link_common_finish(lf);
 	if (error != 0)
 		goto out;
diff --git a/sys/sys/linker.h b/sys/sys/linker.h
index bfb778dd4051..143d7473cf10 100644
--- a/sys/sys/linker.h
+++ b/sys/sys/linker.h
@@ -97,6 +97,11 @@ struct linker_file {
      */
     int			nenabled;	/* number of enabled probes. */
     int			fbt_nentries;	/* number of fbt entries created. */
+
+#ifdef __arm__
+    caddr_t		exidx_addr;	/* Unwind data index table start */
+    size_t		exidx_size;	/* Unwind data index table size */
+#endif
 };
 
 /*