diff options
26 files changed, 185 insertions, 451 deletions
diff --git a/.clang-format b/.clang-format index 480affab5722..78e005ff7d39 100644 --- a/.clang-format +++ b/.clang-format @@ -76,6 +76,7 @@ ForEachMacros: - TAILQ_FOREACH_REVERSE_SAFE - TAILQ_FOREACH_SAFE - VM_MAP_ENTRY_FOREACH + - VM_PAGE_DUMP_FOREACH IndentCaseLabels: false IndentPPDirectives: None Language: Cpp diff --git a/sys/amd64/amd64/minidump_machdep.c b/sys/amd64/amd64/minidump_machdep.c index 48a427869bab..69d12672e2f4 100644 --- a/sys/amd64/amd64/minidump_machdep.c +++ b/sys/amd64/amd64/minidump_machdep.c @@ -54,9 +54,6 @@ __FBSDID("$FreeBSD$"); CTASSERT(sizeof(struct kerneldumpheader) == 512); -uint64_t *vm_page_dump; -int vm_page_dump_size; - static struct kerneldumpheader kdh; /* Handle chunked writes. */ @@ -64,7 +61,6 @@ static size_t fragsz; static void *dump_va; static size_t counter, progress, dumpsize, wdog_next; -CTASSERT(sizeof(*vm_page_dump) == 8); static int dump_retry_count = 5; SYSCTL_INT(_machdep, OID_AUTO, dump_retry_count, CTLFLAG_RWTUN, &dump_retry_count, 0, "Number of times dump has to retry before bailing out"); @@ -223,9 +219,8 @@ minidumpsys(struct dumperinfo *di) uint32_t pmapsize; vm_offset_t va; int error; - uint64_t bits; uint64_t *pml4, *pdp, *pd, *pt, pa; - int i, ii, j, k, n, bit; + int i, ii, j, k, n; int retry_count; struct minidumphdr mdhdr; @@ -304,19 +299,13 @@ minidumpsys(struct dumperinfo *di) /* Calculate dump size. */ dumpsize = pmapsize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = bsfq(bits); - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) { - dumpsize += PAGE_SIZE; - } else { - dump_drop_page(pa); - } - bits &= ~(1ul << bit); + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) { + dumpsize += PAGE_SIZE; + } else { + dump_drop_page(pa); } } dumpsize += PAGE_SIZE; @@ -328,7 +317,7 @@ minidumpsys(struct dumperinfo *di) strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.pmapsize = pmapsize; mdhdr.kernbase = VM_MIN_KERNEL_ADDRESS; mdhdr.dmapbase = DMAP_MIN_ADDRESS; @@ -357,7 +346,8 @@ minidumpsys(struct dumperinfo *di) goto fail; /* Dump bitmap */ - error = blk_write(di, (char *)vm_page_dump, 0, round_page(vm_page_dump_size)); + error = blk_write(di, (char *)vm_page_dump, 0, + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -409,16 +399,10 @@ minidumpsys(struct dumperinfo *di) } /* Dump memory chunks */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = bsfq(bits); - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + bit) * PAGE_SIZE; - error = blk_write(di, 0, pa, PAGE_SIZE); - if (error) - goto fail; - bits &= ~(1ul << bit); - } + VM_PAGE_DUMP_FOREACH(pa) { + error = blk_write(di, 0, pa, PAGE_SIZE); + if (error) + goto fail; } error = blk_flush(di); @@ -454,25 +438,3 @@ minidumpsys(struct dumperinfo *di) printf("** DUMP FAILED (ERROR %d) **\n", error); return (error); } - -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_set_long(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_clear_long(&vm_page_dump[idx], 1ul << bit); -} diff --git a/sys/amd64/include/md_var.h b/sys/amd64/include/md_var.h index 9a550d7024fe..7169474b31db 100644 --- a/sys/amd64/include/md_var.h +++ b/sys/amd64/include/md_var.h @@ -36,7 +36,6 @@ #include <x86/x86_var.h> -extern uint64_t *vm_page_dump; extern int hw_lower_amd64_sharedpage; extern int hw_ibrs_disable; extern int hw_ssb_disable; diff --git a/sys/amd64/include/vmparam.h b/sys/amd64/include/vmparam.h index 0117b94b2f36..38074a647b7c 100644 --- a/sys/amd64/include/vmparam.h +++ b/sys/amd64/include/vmparam.h @@ -253,4 +253,9 @@ */ #define VM_BATCHQUEUE_SIZE 31 +/* + * Need a page dump array for minidump. + */ +#define MINIDUMP_PAGE_TRACKING 1 + #endif /* _MACHINE_VMPARAM_H_ */ diff --git a/sys/arm/arm/minidump_machdep.c b/sys/arm/arm/minidump_machdep.c index c8d55118ec74..4d4d565885b7 100644 --- a/sys/arm/arm/minidump_machdep.c +++ b/sys/arm/arm/minidump_machdep.c @@ -57,9 +57,6 @@ __FBSDID("$FreeBSD$"); CTASSERT(sizeof(struct kerneldumpheader) == 512); -uint32_t *vm_page_dump; -int vm_page_dump_size; - static struct kerneldumpheader kdh; /* Handle chunked writes. */ @@ -67,8 +64,6 @@ static size_t fragsz; static void *dump_va; static uint64_t counter, progress; -CTASSERT(sizeof(*vm_page_dump) == 4); - static int is_dumpable(vm_paddr_t pa) { @@ -182,10 +177,9 @@ minidumpsys(struct dumperinfo *di) struct minidumphdr mdhdr; uint64_t dumpsize; uint32_t ptesize; - uint32_t bits; uint32_t pa, prev_pa = 0, count = 0; vm_offset_t va; - int i, bit, error; + int error; char *addr; /* @@ -212,20 +206,13 @@ minidumpsys(struct dumperinfo *di) /* Calculate dump size. */ dumpsize = ptesize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffs(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) - dumpsize += PAGE_SIZE; - else - dump_drop_page(pa); - bits &= ~(1ul << bit); - } + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) + dumpsize += PAGE_SIZE; + else + dump_drop_page(pa); } dumpsize += PAGE_SIZE; @@ -236,7 +223,7 @@ minidumpsys(struct dumperinfo *di) strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.ptesize = ptesize; mdhdr.kernbase = KERNBASE; mdhdr.arch = __ARM_ARCH; @@ -270,7 +257,7 @@ minidumpsys(struct dumperinfo *di) /* Dump bitmap */ error = blk_write(di, (char *)vm_page_dump, 0, - round_page(vm_page_dump_size)); + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -293,28 +280,21 @@ minidumpsys(struct dumperinfo *di) } /* Dump memory chunks */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffs(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - if (!count) { - prev_pa = pa; + VM_PAGE_DUMP_FOREACH(pa) { + if (!count) { + prev_pa = pa; + count++; + } else { + if (pa == (prev_pa + count * PAGE_SIZE)) count++; - } else { - if (pa == (prev_pa + count * PAGE_SIZE)) - count++; - else { - error = blk_write(di, NULL, prev_pa, - count * PAGE_SIZE); - if (error) - goto fail; - count = 1; - prev_pa = pa; - } + else { + error = blk_write(di, NULL, prev_pa, + count * PAGE_SIZE); + if (error) + goto fail; + count = 1; + prev_pa = pa; } - bits &= ~(1ul << bit); } } if (count) { @@ -349,25 +329,3 @@ fail: printf("\n** DUMP FAILED (ERROR %d) **\n", error); return (error); } - -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_set_int(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_clear_int(&vm_page_dump[idx], 1ul << bit); -} diff --git a/sys/arm/include/md_var.h b/sys/arm/include/md_var.h index 6b58c96c1b8d..eb6e59794622 100644 --- a/sys/arm/include/md_var.h +++ b/sys/arm/include/md_var.h @@ -38,8 +38,6 @@ extern long Maxmem; extern char sigcode[]; extern int szsigcode; -extern uint32_t *vm_page_dump; -extern int vm_page_dump_size; extern u_long elf_hwcap; extern u_long elf_hwcap2; extern vm_paddr_t arm_physmem_kernaddr; @@ -72,8 +70,6 @@ extern enum cpu_class cpu_class; struct dumperinfo; extern int busdma_swi_pending; void busdma_swi(void); -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); int minidumpsys(struct dumperinfo *); extern uint32_t initial_fpscr; diff --git a/sys/arm/include/vmparam.h b/sys/arm/include/vmparam.h index b9b9163d8af7..28269074d139 100644 --- a/sys/arm/include/vmparam.h +++ b/sys/arm/include/vmparam.h @@ -193,4 +193,9 @@ extern vm_offset_t vm_max_kernel_address; #define DEVMAP_MAX_VADDR ARM_VECTORS_HIGH +/* + * Need a page dump array for minidump. + */ +#define MINIDUMP_PAGE_TRACKING 1 + #endif /* _MACHINE_VMPARAM_H_ */ diff --git a/sys/arm64/arm64/minidump_machdep.c b/sys/arm64/arm64/minidump_machdep.c index ba22f7dfc16f..758b7849a6b3 100644 --- a/sys/arm64/arm64/minidump_machdep.c +++ b/sys/arm64/arm64/minidump_machdep.c @@ -56,9 +56,6 @@ __FBSDID("$FreeBSD$"); CTASSERT(sizeof(struct kerneldumpheader) == 512); -uint64_t *vm_page_dump; -int vm_page_dump_size; - static struct kerneldumpheader kdh; /* Handle chunked writes. */ @@ -68,8 +65,6 @@ static size_t counter, progress, dumpsize; static uint64_t tmpbuffer[Ln_ENTRIES]; -CTASSERT(sizeof(*vm_page_dump) == 8); - static int is_dumpable(vm_paddr_t pa) { @@ -213,9 +208,8 @@ minidumpsys(struct dumperinfo *di) pt_entry_t *l3; vm_offset_t va; vm_paddr_t pa; - uint64_t bits; uint32_t pmapsize; - int bit, error, i, j, retry_count; + int error, i, j, retry_count; retry_count = 0; retry: @@ -255,20 +249,12 @@ minidumpsys(struct dumperinfo *di) /* Calculate dump size. */ dumpsize = pmapsize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffsl(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) - dumpsize += PAGE_SIZE; - else - dump_drop_page(pa); - bits &= ~(1ul << bit); - } + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(pa) { + if (is_dumpable(pa)) + dumpsize += PAGE_SIZE; + else + dump_drop_page(pa); } dumpsize += PAGE_SIZE; @@ -279,7 +265,7 @@ minidumpsys(struct dumperinfo *di) strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.pmapsize = pmapsize; mdhdr.kernbase = VM_MIN_KERNEL_ADDRESS; mdhdr.dmapphys = DMAP_MIN_PHYSADDR; @@ -311,7 +297,7 @@ minidumpsys(struct dumperinfo *di) /* Dump bitmap */ error = blk_write(di, (char *)vm_page_dump, 0, - round_page(vm_page_dump_size)); + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -378,17 +364,10 @@ minidumpsys(struct dumperinfo *di) } /* Dump memory chunks */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffsl(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - error = blk_write(di, 0, pa, PAGE_SIZE); - if (error) - goto fail; - bits &= ~(1ul << bit); - } + VM_PAGE_DUMP_FOREACH(pa) { + error = blk_write(di, 0, pa, PAGE_SIZE); + if (error) + goto fail; } error = blk_flush(di); @@ -424,25 +403,3 @@ fail: printf("** DUMP FAILED (ERROR %d) **\n", error); return (error); } - -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_set_long(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_clear_long(&vm_page_dump[idx], 1ul << bit); -} diff --git a/sys/arm64/include/md_var.h b/sys/arm64/include/md_var.h index cc790d984711..0132ab0dd8fd 100644 --- a/sys/arm64/include/md_var.h +++ b/sys/arm64/include/md_var.h @@ -36,8 +36,6 @@ extern long Maxmem; extern char sigcode[]; extern int szsigcode; -extern uint64_t *vm_page_dump; -extern int vm_page_dump_size; extern u_long elf_hwcap; extern u_long elf_hwcap2; @@ -45,8 +43,6 @@ struct dumperinfo; extern int busdma_swi_pending; void busdma_swi(void); -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); int minidumpsys(struct dumperinfo *); void generic_bs_fault(void) __asm(__STRING(generic_bs_fault)); void generic_bs_peek_1(void) __asm(__STRING(generic_bs_peek_1)); diff --git a/sys/arm64/include/vmparam.h b/sys/arm64/include/vmparam.h index 96b46e575483..aa5b68bd6d80 100644 --- a/sys/arm64/include/vmparam.h +++ b/sys/arm64/include/vmparam.h @@ -243,4 +243,9 @@ extern vm_offset_t init_pt_va; #define DEVMAP_MAX_VADDR VM_MAX_KERNEL_ADDRESS +/* + * Need a page dump array for minidump. + */ +#define MINIDUMP_PAGE_TRACKING 1 + #endif /* !_MACHINE_VMPARAM_H_ */ diff --git a/sys/i386/i386/minidump_machdep.c b/sys/i386/i386/minidump_machdep.c index 5ef23cbc34ec..d2a7999cb2c3 100644 --- a/sys/i386/i386/minidump_machdep.c +++ b/sys/i386/i386/minidump_machdep.c @@ -49,33 +49,6 @@ __FBSDID("$FreeBSD$"); CTASSERT(sizeof(struct kerneldumpheader) == 512); -uint32_t *vm_page_dump; -int vm_page_dump_size; - -CTASSERT(sizeof(*vm_page_dump) == 4); - -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_set_int(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_clear_int(&vm_page_dump[idx], 1ul << bit); -} - int minidumpsys(struct dumperinfo *di) { diff --git a/sys/i386/i386/minidump_machdep_base.c b/sys/i386/i386/minidump_machdep_base.c index 49400b0cc2ae..1c963907a430 100644 --- a/sys/i386/i386/minidump_machdep_base.c +++ b/sys/i386/i386/minidump_machdep_base.c @@ -62,8 +62,6 @@ static size_t fragsz; static void *dump_va; static uint64_t counter, progress; -CTASSERT(sizeof(*vm_page_dump) == 4); - static int is_dumpable(vm_paddr_t pa) { @@ -181,11 +179,10 @@ minidumpsys(struct dumperinfo *di) uint32_t ptesize; vm_offset_t va; int error; - uint32_t bits; uint64_t pa; pd_entry_t *pd; pt_entry_t *pt; - int i, j, k, bit; + int j, k; struct minidumphdr mdhdr; counter = 0; @@ -227,19 +224,13 @@ minidumpsys(struct dumperinfo *di) /* Calculate dump size. */ dumpsize = ptesize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = bsfl(bits); - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) { - dumpsize += PAGE_SIZE; - } else { - dump_drop_page(pa); - } - bits &= ~(1ul << bit); + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) { + dumpsize += PAGE_SIZE; + } else { + dump_drop_page(pa); } } dumpsize += PAGE_SIZE; @@ -251,7 +242,7 @@ minidumpsys(struct dumperinfo *di) strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.ptesize = ptesize; mdhdr.kernbase = KERNBASE; mdhdr.paemode = pae_mode; @@ -279,7 +270,8 @@ minidumpsys(struct dumperinfo *di) goto fail; /* Dump bitmap */ - error = blk_write(di, (char *)vm_page_dump, 0, round_page(vm_page_dump_size)); + error = blk_write(di, (char *)vm_page_dump, 0, + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -321,16 +313,10 @@ minidumpsys(struct dumperinfo *di) } /* Dump memory chunks */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = bsfl(bits); - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + bit) * PAGE_SIZE; - error = blk_write(di, 0, pa, PAGE_SIZE); - if (error) - goto fail; - bits &= ~(1ul << bit); - } + VM_PAGE_DUMP_FOREACH(pa) { + error = blk_write(di, 0, pa, PAGE_SIZE); + if (error) + goto fail; } error = blk_flush(di); diff --git a/sys/i386/include/md_var.h b/sys/i386/include/md_var.h index b20446dce12b..c41de85b9bc9 100644 --- a/sys/i386/include/md_var.h +++ b/sys/i386/include/md_var.h @@ -47,7 +47,6 @@ extern int szfreebsd4_sigcode; extern int szosigcode; extern int sz_lcall_tramp; #endif -extern uint32_t *vm_page_dump; extern vm_offset_t proc0kstack; extern uintptr_t setidt_disp; diff --git a/sys/i386/include/vmparam.h b/sys/i386/include/vmparam.h index c7cf4de8cf75..212ddf758d17 100644 --- a/sys/i386/include/vmparam.h +++ b/sys/i386/include/vmparam.h @@ -240,4 +240,9 @@ #define PHYS_TO_DMAP(x) ({ panic("No direct map exists"); 0; }) #define DMAP_TO_PHYS(x) ({ panic("No direct map exists"); 0; }) +/* + * Need a page dump array for minidump. + */ +#define MINIDUMP_PAGE_TRACKING 1 + #endif /* _MACHINE_VMPARAM_H_ */ diff --git a/sys/mips/include/md_var.h b/sys/mips/include/md_var.h index 4bf9c64a8f5b..8462b1beb22e 100644 --- a/sys/mips/include/md_var.h +++ b/sys/mips/include/md_var.h @@ -50,8 +50,6 @@ extern int szsigcode; extern char sigcode32[]; extern int szsigcode32; #endif -extern uint32_t *vm_page_dump; -extern int vm_page_dump_size; extern vm_offset_t kstack0; extern vm_offset_t kernel_kseg0_end; @@ -84,8 +82,6 @@ extern int busdma_swi_pending; void busdma_swi(void); struct dumperinfo; -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); int minidumpsys(struct dumperinfo *); #endif /* !_MACHINE_MD_VAR_H_ */ diff --git a/sys/mips/include/vmparam.h b/sys/mips/include/vmparam.h index 305224dc7b87..183efe1a7c0c 100644 --- a/sys/mips/include/vmparam.h +++ b/sys/mips/include/vmparam.h @@ -197,4 +197,9 @@ #define PHYS_TO_DMAP(x) MIPS_PHYS_TO_DIRECT(x) #define DMAP_TO_PHYS(x) MIPS_DIRECT_TO_PHYS(x) +/* + * Need a page dump array for minidump. + */ +#define MINIDUMP_PAGE_TRACKING 1 + #endif /* !_MACHINE_VMPARAM_H_ */ diff --git a/sys/mips/mips/minidump_machdep.c b/sys/mips/mips/minidump_machdep.c index c44ee8a77781..164e7a04204b 100644 --- a/sys/mips/mips/minidump_machdep.c +++ b/sys/mips/mips/minidump_machdep.c @@ -54,9 +54,6 @@ __FBSDID("$FreeBSD$"); CTASSERT(sizeof(struct kerneldumpheader) == 512); -uint32_t *vm_page_dump; -int vm_page_dump_size; - static struct kerneldumpheader kdh; /* Handle chunked writes. */ @@ -66,8 +63,6 @@ static char tmpbuffer[PAGE_SIZE]; extern pd_entry_t *kernel_segmap; -CTASSERT(sizeof(*vm_page_dump) == 4); - static int is_dumpable(vm_paddr_t pa) { @@ -83,28 +78,6 @@ is_dumpable(vm_paddr_t pa) return (0); } -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_set_int(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_clear_int(&vm_page_dump[idx], 1ul << bit); -} - static struct { int min_per; int max_per; @@ -192,13 +165,12 @@ minidumpsys(struct dumperinfo *di) { struct minidumphdr mdhdr; uint32_t ptesize; - uint32_t bits; vm_paddr_t pa; vm_offset_t prev_pte = 0; uint32_t count = 0; vm_offset_t va; pt_entry_t *pte; - int i, bit, error; + int i, error; void *dump_va; /* Flush cache */ @@ -233,20 +205,13 @@ minidumpsys(struct dumperinfo *di) /* Calculate dump size. */ dumpsize = ptesize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffs(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) - dumpsize += PAGE_SIZE; - else - dump_drop_page(pa); - bits &= ~(1ul << bit); - } + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) + dumpsize += PAGE_SIZE; + else + dump_drop_page(pa); } dumpsize += PAGE_SIZE; @@ -257,7 +222,7 @@ minidumpsys(struct dumperinfo *di) strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.ptesize = ptesize; mdhdr.kernbase = VM_MIN_KERNEL_ADDRESS; @@ -286,7 +251,7 @@ minidumpsys(struct dumperinfo *di) /* Dump bitmap */ error = write_buffer(di, (char *)vm_page_dump, - round_page(vm_page_dump_size)); + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -320,19 +285,12 @@ minidumpsys(struct dumperinfo *di) } /* Dump memory chunks page by page*/ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffs(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - dump_va = pmap_kenter_temporary(pa, 0); - error = write_buffer(di, dump_va, PAGE_SIZE); - if (error) - goto fail; - pmap_kenter_temporary_free(pa); - bits &= ~(1ul << bit); - } + VM_PAGE_DUMP_FOREACH(pa) { + dump_va = pmap_kenter_temporary(pa, 0); + error = write_buffer(di, dump_va, PAGE_SIZE); + if (error) + goto fail; + pmap_kenter_temporary_free(pa); } error = dump_finish(di, &kdh); diff --git a/sys/powerpc/include/md_var.h b/sys/powerpc/include/md_var.h index 7db9d3d3bf37..da9232313ffb 100644 --- a/sys/powerpc/include/md_var.h +++ b/sys/powerpc/include/md_var.h @@ -42,14 +42,9 @@ extern int szsigcode32; extern char sigcode64[], sigcode64_elfv2[]; extern int szsigcode64, szsigcode64_elfv2; -extern uint64_t *vm_page_dump; -extern int vm_page_dump_size; - struct dumperinfo; int minidumpsys(struct dumperinfo *); int is_dumpable(vm_paddr_t); -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); #endif extern long Maxmem; diff --git a/sys/powerpc/include/vmparam.h b/sys/powerpc/include/vmparam.h index f46b058e7266..03ea95c86d05 100644 --- a/sys/powerpc/include/vmparam.h +++ b/sys/powerpc/include/vmparam.h @@ -307,6 +307,18 @@ struct pmap_physseg { #define PMAP_HAS_PAGE_ARRAY 1 #endif +#if defined(__powerpc64__) +/* + * Need a page dump array for minidump. + */ +#define MINIDUMP_PAGE_TRACKING 1 +#else +/* + * No minidump with 32-bit powerpc. + */ +#define MINIDUMP_PAGE_TRACKING 0 +#endif + #define PMAP_HAS_DMAP (hw_direct_map) #define PHYS_TO_DMAP(x) ({ \ KASSERT(hw_direct_map, ("Direct map not provided by PMAP")); \ diff --git a/sys/powerpc/powerpc/minidump_machdep.c b/sys/powerpc/powerpc/minidump_machdep.c index e93b11379db0..74b790537e38 100644 --- a/sys/powerpc/powerpc/minidump_machdep.c +++ b/sys/powerpc/powerpc/minidump_machdep.c @@ -45,16 +45,6 @@ #include <machine/md_var.h> #include <machine/minidump.h> -/* - * bit to physical address - * - * bm - bitmap - * i - bitmap entry index - * bit - bit number - */ -#define BTOP(bm, i, bit) \ - (((uint64_t)(i) * sizeof(*(bm)) * NBBY + (bit)) * PAGE_SIZE) - /* Debugging stuff */ #define MINIDUMP_DEBUG 0 #if MINIDUMP_DEBUG @@ -70,9 +60,6 @@ static void dump_total(const char *id, size_t sz); extern vm_offset_t __startkernel, __endkernel; -int vm_page_dump_size; -uint64_t *vm_page_dump; - static int dump_retry_count = 5; SYSCTL_INT(_machdep, OID_AUTO, dump_retry_count, CTLFLAG_RWTUN, &dump_retry_count, 0, @@ -103,28 +90,6 @@ static size_t counter, dumpsize, progress; /* Handle chunked writes. */ static size_t fragsz; -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_set_long(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_clear_long(&vm_page_dump[idx], 1ul << bit); -} - int is_dumpable(vm_paddr_t pa) { @@ -280,9 +245,8 @@ int minidumpsys(struct dumperinfo *di) { vm_paddr_t pa; - int bit, error, i, retry_count; + int error, i, retry_count; uint32_t pmapsize; - uint64_t bits; struct minidumphdr mdhdr; retry_count = 0; @@ -306,24 +270,14 @@ retry: /* Calculate dump size */ dumpsize = PAGE_SIZE; /* header */ dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); dumpsize += pmapsize; - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - /* TODO optimize with bit manipulation instructions */ - if (bits == 0) - continue; - for (bit = 0; bit < 64; bit++) { - if ((bits & (1ul<<bit)) == 0) - continue; - - pa = BTOP(vm_page_dump, i, bit); - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) - dumpsize += PAGE_SIZE; - else - dump_drop_page(pa); - } + VM_PAGE_DUMP_FOREACH(pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) + dumpsize += PAGE_SIZE; + else + dump_drop_page(pa); } progress = dumpsize; @@ -333,7 +287,7 @@ retry: strncpy(mdhdr.mmu_name, pmap_mmu_name(), sizeof(mdhdr.mmu_name) - 1); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.pmapsize = pmapsize; mdhdr.kernbase = VM_MIN_KERNEL_ADDRESS; mdhdr.kernend = VM_MAX_SAFE_KERNEL_ADDRESS; @@ -368,10 +322,10 @@ retry: /* Dump bitmap */ error = blk_write(di, (char *)vm_page_dump, 0, - round_page(vm_page_dump_size)); + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; - dump_total("bitmap", round_page(vm_page_dump_size)); + dump_total("bitmap", round_page(BITSET_SIZE(vm_page_dump_pages))); /* Dump kernel page directory pages */ error = dump_pmap(di); @@ -380,20 +334,10 @@ retry: dump_total("pmap", pmapsize); /* Dump memory chunks */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - /* TODO optimize with bit manipulation instructions */ - if (bits == 0) - continue; - for (bit = 0; bit < 64; bit++) { - if ((bits & (1ul<<bit)) == 0) - continue; - - pa = BTOP(vm_page_dump, i, bit); - error = blk_write(di, 0, pa, PAGE_SIZE); - if (error) - goto fail; - } + VM_PAGE_DUMP_FOREACH(pa) { + error = blk_write(di, 0, pa, PAGE_SIZE); + if (error) + goto fail; } error = blk_flush(di); diff --git a/sys/riscv/include/md_var.h b/sys/riscv/include/md_var.h index 4a0b554dfa13..b36a44d7d023 100644 --- a/sys/riscv/include/md_var.h +++ b/sys/riscv/include/md_var.h @@ -36,8 +36,6 @@ extern long Maxmem; extern char sigcode[]; extern int szsigcode; -extern uint64_t *vm_page_dump; -extern int vm_page_dump_size; extern u_long elf_hwcap; extern register_t mvendorid; extern register_t marchid; @@ -46,8 +44,6 @@ extern register_t mimpid; struct dumperinfo; void busdma_swi(void); -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); int minidumpsys(struct dumperinfo *); #endif /* !_MACHINE_MD_VAR_H_ */ diff --git a/sys/riscv/include/vmparam.h b/sys/riscv/include/vmparam.h index 989895dafef9..00ee1c3ed989 100644 --- a/sys/riscv/include/vmparam.h +++ b/sys/riscv/include/vmparam.h @@ -236,4 +236,9 @@ extern vm_offset_t init_pt_va; #define DEVMAP_MAX_VADDR VM_MAX_KERNEL_ADDRESS +/* + * Need a page dump array for minidump. + */ +#define MINIDUMP_PAGE_TRACKING 1 + #endif /* !_MACHINE_VMPARAM_H_ */ diff --git a/sys/riscv/riscv/minidump_machdep.c b/sys/riscv/riscv/minidump_machdep.c index 58cf2f9d72e5..3631db224071 100644 --- a/sys/riscv/riscv/minidump_machdep.c +++ b/sys/riscv/riscv/minidump_machdep.c @@ -53,10 +53,6 @@ __FBSDID("$FreeBSD$"); #include <machine/minidump.h> CTASSERT(sizeof(struct kerneldumpheader) == 512); -CTASSERT(sizeof(*vm_page_dump) == 8); - -uint64_t *vm_page_dump; -int vm_page_dump_size; static struct kerneldumpheader kdh; @@ -221,8 +217,7 @@ minidumpsys(struct dumperinfo *di) vm_offset_t va; vm_paddr_t pa; int error; - uint64_t bits; - int i, bit; + int i; int retry_count; retry_count = 0; @@ -262,20 +257,13 @@ retry: /* Calculate dump size */ dumpsize = pmapsize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffsl(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) - dumpsize += PAGE_SIZE; - else - dump_drop_page(pa); - bits &= ~(1ul << bit); - } + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) + dumpsize += PAGE_SIZE; + else + dump_drop_page(pa); } dumpsize += PAGE_SIZE; @@ -286,7 +274,7 @@ retry: strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.pmapsize = pmapsize; mdhdr.kernbase = KERNBASE; mdhdr.dmapphys = DMAP_MIN_PHYSADDR; @@ -318,7 +306,7 @@ retry: /* Dump bitmap */ error = blk_write(di, (char *)vm_page_dump, 0, - round_page(vm_page_dump_size)); + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -360,17 +348,10 @@ retry: /* Dump memory chunks */ /* XXX cluster it up and use blk_dump() */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffsl(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - error = blk_write(di, 0, pa, PAGE_SIZE); - if (error) - goto fail; - bits &= ~(1ul << bit); - } + VM_PAGE_DUMP_FOREACH(pa) { + error = blk_write(di, 0, pa, PAGE_SIZE); + if (error) + goto fail; } error = blk_flush(di); @@ -406,31 +387,3 @@ fail: printf("** DUMP FAILED (ERROR %d) **\n", error); return (error); } - -/* - * Add a page to the minidump bitmap. - */ -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_set_long(&vm_page_dump[idx], 1ul << bit); -} - -/* - * Remove page from the minidump bitmap. - */ -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_clear_long(&vm_page_dump[idx], 1ul << bit); -} diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index e120c0d29099..140711a00227 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -155,6 +155,9 @@ vm_page_t vm_page_array; long vm_page_array_size; long first_page; +struct bitset *vm_page_dump; +long vm_page_dump_pages; + static TAILQ_HEAD(, vm_page) blacklist_head; static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING | CTLFLAG_RD | @@ -554,6 +557,9 @@ vm_page_startup(vm_offset_t vaddr) vm_paddr_t page_range __unused; vm_paddr_t last_pa, pa; u_long pagecount; +#if MINIDUMP_PAGE_TRACKING + u_long vm_page_dump_size; +#endif int biggestone, i, segind; #ifdef WITNESS vm_offset_t mapped; @@ -588,9 +594,7 @@ vm_page_startup(vm_offset_t vaddr) witness_startup((void *)mapped); #endif -#if defined(__aarch64__) || defined(__amd64__) || defined(__arm__) || \ - defined(__i386__) || defined(__mips__) || defined(__riscv) || \ - defined(__powerpc64__) +#if MINIDUMP_PAGE_TRACKING /* * Allocate a bitmap to indicate that a random physical page * needs to be included in a minidump. @@ -606,8 +610,8 @@ vm_page_startup(vm_offset_t vaddr) for (i = 0; dump_avail[i + 1] != 0; i += 2) if (dump_avail[i + 1] > last_pa) last_pa = dump_avail[i + 1]; - page_range = last_pa / PAGE_SIZE; - vm_page_dump_size = round_page(roundup2(page_range, NBBY) / NBBY); + vm_page_dump_pages = last_pa / PAGE_SIZE; + vm_page_dump_size = round_page(BITSET_SIZE(vm_page_dump_pages)); new_end -= vm_page_dump_size; vm_page_dump = (void *)(uintptr_t)pmap_map(&vaddr, new_end, new_end + vm_page_dump_size, VM_PROT_READ | VM_PROT_WRITE); diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h index a44e31f506d0..ec9a539b8f05 100644 --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -69,6 +69,8 @@ #ifndef _VM_PAGE_ #define _VM_PAGE_ +#include <sys/_bitset.h> +#include <sys/bitset.h> #include <vm/pmap.h> /* @@ -585,6 +587,26 @@ malloc2vm_flags(int malloc_flags) #define PS_ALL_VALID 0x2 #define PS_NONE_BUSY 0x4 +extern struct bitset *vm_page_dump; +extern long vm_page_dump_pages; + +static inline void +dump_add_page(vm_paddr_t pa) +{ + BIT_SET_ATOMIC(vm_page_dump_pages, pa >> PAGE_SHIFT, vm_page_dump); +} + +static inline void +dump_drop_page(vm_paddr_t pa) +{ + BIT_CLR_ATOMIC(vm_page_dump_pages, pa >> PAGE_SHIFT, vm_page_dump); +} + +#define VM_PAGE_DUMP_FOREACH(pa) \ + for (vm_pindex_t __b = BIT_FFS(vm_page_dump_pages, vm_page_dump); \ + (pa) = (__b - 1) * PAGE_SIZE, __b != 0; \ + __b = BIT_FFS_AT(vm_page_dump_pages, vm_page_dump, __b)) + bool vm_page_busy_acquire(vm_page_t m, int allocflags); void vm_page_busy_downgrade(vm_page_t m); int vm_page_busy_tryupgrade(vm_page_t m); diff --git a/sys/x86/include/x86_var.h b/sys/x86/include/x86_var.h index 789b8a2cbc48..a1ce8f9e3113 100644 --- a/sys/x86/include/x86_var.h +++ b/sys/x86/include/x86_var.h @@ -78,7 +78,6 @@ extern char hv_vendor[]; extern char kstack[]; extern char sigcode[]; extern int szsigcode; -extern int vm_page_dump_size; extern int workaround_erratum383; extern int _udatasel; extern int _ucodesel; @@ -122,8 +121,6 @@ void cpu_probe_amdc1e(void); void cpu_setregs(void); bool disable_wp(void); void restore_wp(bool old_wp); -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); void finishidentcpu(void); void identify_cpu1(void); void identify_cpu2(void); |