diff options
56 files changed, 376 insertions, 323 deletions
diff --git a/libexec/rpc.rstatd/rstat_proc.c b/libexec/rpc.rstatd/rstat_proc.c index 18fe27e2064f..f0f2ffd65b57 100644 --- a/libexec/rpc.rstatd/rstat_proc.c +++ b/libexec/rpc.rstatd/rstat_proc.c @@ -170,6 +170,7 @@ updatestat(void) struct timeval tm, btm; int mib[6]; size_t len; + uint64_t val; int ifcount; #ifdef DEBUG @@ -229,11 +230,12 @@ updatestat(void) #endif #define FETCH_CNT(stat, cnt) do { \ - len = sizeof((stat)); \ - if (sysctlbyname("vm.stats." #cnt , &(stat), &len, 0, 0) < 0) { \ - syslog(LOG_ERR, "sysctl(vm.stats." #cnt "): %m"); \ + len = sizeof(uint64_t); \ + if (sysctlbyname("vm.stats." #cnt , &val, &len, NULL, 0) < 0) { \ + syslog(LOG_ERR, "sysctl(vm.stats." #cnt "): %m"); \ exit(1); \ } \ + stat = val; \ } while (0) FETCH_CNT(stats_all.s1.v_pgpgin, vm.v_vnodepgsin); diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c index e779ef21c4ab..d77cbedd383c 100644 --- a/sys/amd64/amd64/trap.c +++ b/sys/amd64/amd64/trap.c @@ -176,7 +176,7 @@ trap(struct trapframe *frame) register_t addr = 0; ksiginfo_t ksi; - PCPU_INC(cnt.v_trap); + VM_CNT_INC(v_trap); type = frame->tf_trapno; #ifdef SMP diff --git a/sys/amd64/include/atomic.h b/sys/amd64/include/atomic.h index aa1946279c71..e3cfff7cd47a 100644 --- a/sys/amd64/include/atomic.h +++ b/sys/amd64/include/atomic.h @@ -348,7 +348,7 @@ atomic_testandclear_long(volatile u_long *p, u_int v) * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. * An assertion in amd64/vm_machdep.c ensures that the value is correct. */ -#define OFFSETOF_MONITORBUF 0x180 +#define OFFSETOF_MONITORBUF 0x100 #if defined(SMP) static __inline void diff --git a/sys/amd64/include/counter.h b/sys/amd64/include/counter.h index 74ed1ef4d1f1..0e90e05c7253 100644 --- a/sys/amd64/include/counter.h +++ b/sys/amd64/include/counter.h @@ -31,7 +31,9 @@ #include <sys/pcpu.h> -extern struct pcpu __pcpu[1]; +extern struct pcpu __pcpu[]; + +#define EARLY_COUNTER &__pcpu[0].pc_early_dummy_counter #define counter_enter() do {} while (0) #define counter_exit() do {} while (0) diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h index 7e3c1b0bbb27..af5e889f2107 100644 --- a/sys/amd64/include/pcpu.h +++ b/sys/amd64/include/pcpu.h @@ -66,7 +66,7 @@ uint32_t pc_pcid_next; \ uint32_t pc_pcid_gen; \ uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \ - char __pad[145] /* be divisor of PAGE_SIZE \ + char __pad[384] /* be divisor of PAGE_SIZE \ after cache alignment */ #define PC_DBREG_CMD_NONE 0 diff --git a/sys/arm/arm/intr.c b/sys/arm/arm/intr.c index e4dbccd01ed9..42ec4f2833e0 100644 --- a/sys/arm/arm/intr.c +++ b/sys/arm/arm/intr.c @@ -185,7 +185,7 @@ intr_irq_handler(struct trapframe *frame) struct intr_event *event; int i; - PCPU_INC(cnt.v_intr); + VM_CNT_INC(v_intr); i = -1; while ((i = arm_get_next_irq(i)) != -1) { intrcnt[i]++; diff --git a/sys/arm/arm/trap-v4.c b/sys/arm/arm/trap-v4.c index 67e8ad2f521f..bec22abf0dcd 100644 --- a/sys/arm/arm/trap-v4.c +++ b/sys/arm/arm/trap-v4.c @@ -204,7 +204,7 @@ abort_handler(struct trapframe *tf, int type) td = curthread; p = td->td_proc; - PCPU_INC(cnt.v_trap); + VM_CNT_INC(v_trap); /* Data abort came from user mode? */ user = TRAP_USERMODE(tf); @@ -615,7 +615,7 @@ prefetch_abort_handler(struct trapframe *tf) td = curthread; p = td->td_proc; - PCPU_INC(cnt.v_trap); + VM_CNT_INC(v_trap); if (TRAP_USERMODE(tf)) { td->td_frame = tf; diff --git a/sys/arm/arm/trap-v6.c b/sys/arm/arm/trap-v6.c index de707b5eed36..a6db6dca1412 100644 --- a/sys/arm/arm/trap-v6.c +++ b/sys/arm/arm/trap-v6.c @@ -291,7 +291,7 @@ abort_handler(struct trapframe *tf, int prefetch) void *onfault; #endif - PCPU_INC(cnt.v_trap); + VM_CNT_INC(v_trap); td = curthread; fsr = (prefetch) ? cp15_ifsr_get(): cp15_dfsr_get(); diff --git a/sys/arm/arm/undefined.c b/sys/arm/arm/undefined.c index 1bac76de4ca6..2f385b9987d8 100644 --- a/sys/arm/arm/undefined.c +++ b/sys/arm/arm/undefined.c @@ -201,7 +201,7 @@ undefinedinstruction(struct trapframe *frame) if (__predict_true(frame->tf_spsr & PSR_F) == 0) enable_interrupts(PSR_F); - PCPU_INC(cnt.v_trap); + VM_CNT_INC(v_trap); fault_pc = frame->tf_pc; diff --git a/sys/arm/include/counter.h b/sys/arm/include/counter.h index b2ddabd3f702..924f128cc3cd 100644 --- a/sys/arm/include/counter.h +++ b/sys/arm/include/counter.h @@ -32,6 +32,10 @@ #include <sys/pcpu.h> #include <machine/atomic.h> +extern struct pcpu __pcpu[]; + +#define EARLY_COUNTER &__pcpu[0].pc_early_dummy_counter + #define counter_enter() do {} while (0) #define counter_exit() do {} while (0) diff --git a/sys/arm/include/pcpu.h b/sys/arm/include/pcpu.h index 3136f97c0e26..6a23ae3fdb4d 100644 --- a/sys/arm/include/pcpu.h +++ b/sys/arm/include/pcpu.h @@ -57,10 +57,10 @@ struct vmspace; void *pc_qmap_pte2p; \ unsigned int pc_dbreg[32]; \ int pc_dbreg_cmd; \ - char __pad[27] + char __pad[155] #else #define PCPU_MD_FIELDS \ - char __pad[157] + char __pad[93] #endif #ifdef _KERNEL diff --git a/sys/arm64/include/counter.h b/sys/arm64/include/counter.h index 2df08dd107e4..e37c34f463af 100644 --- a/sys/arm64/include/counter.h +++ b/sys/arm64/include/counter.h @@ -32,6 +32,10 @@ #include <sys/pcpu.h> #include <machine/atomic.h> +extern struct pcpu __pcpu[]; + +#define EARLY_COUNTER &__pcpu[0].pc_early_dummy_counter + #define counter_enter() do {} while (0) #define counter_exit() do {} while (0) diff --git a/sys/arm64/include/pcpu.h b/sys/arm64/include/pcpu.h index 53f4e7b9cce1..53311663f67c 100644 --- a/sys/arm64/include/pcpu.h +++ b/sys/arm64/include/pcpu.h @@ -39,7 +39,7 @@ u_int pc_acpi_id; /* ACPI CPU id */ \ u_int pc_midr; /* stored MIDR value */ \ uint64_t pc_clock; \ - char __pad[113] + char __pad[241] #ifdef _KERNEL diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c index c1f12dd57deb..2a58ef655b52 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c @@ -4575,8 +4575,8 @@ zfs_getpages(struct vnode *vp, vm_page_t *m, int count, int *rbehind, return (zfs_vm_pagerret_bad); } - PCPU_INC(cnt.v_vnodein); - PCPU_ADD(cnt.v_vnodepgsin, count); + VM_CNT_INC(v_vnodein); + VM_CNT_ADD(v_vnodepgsin, count); lsize = PAGE_SIZE; if (IDX_TO_OFF(mlast->pindex) + lsize > object->un_pager.vnp.vnp_size) @@ -4758,8 +4758,8 @@ zfs_putpages(struct vnode *vp, vm_page_t *ma, size_t len, int flags, vm_page_undirty(ma[i]); } zfs_vmobject_wunlock(object); - PCPU_INC(cnt.v_vnodeout); - PCPU_ADD(cnt.v_vnodepgsout, ncount); + VM_CNT_INC(v_vnodeout); + VM_CNT_ADD(v_vnodepgsout, ncount); } dmu_tx_commit(tx); diff --git a/sys/compat/linprocfs/linprocfs.c b/sys/compat/linprocfs/linprocfs.c index 05258b223f54..09f015ea5514 100644 --- a/sys/compat/linprocfs/linprocfs.c +++ b/sys/compat/linprocfs/linprocfs.c @@ -425,17 +425,17 @@ linprocfs_dostat(PFS_FILL_ARGS) } sbuf_printf(sb, "disk 0 0 0 0\n" - "page %u %u\n" - "swap %u %u\n" - "intr %u\n" - "ctxt %u\n" + "page %ju %ju\n" + "swap %ju %ju\n" + "intr %ju\n" + "ctxt %ju\n" "btime %lld\n", - vm_cnt.v_vnodepgsin, - vm_cnt.v_vnodepgsout, - vm_cnt.v_swappgsin, - vm_cnt.v_swappgsout, - vm_cnt.v_intr, - vm_cnt.v_swtch, + (uintmax_t)VM_CNT_FETCH(v_vnodepgsin), + (uintmax_t)VM_CNT_FETCH(v_vnodepgsout), + (uintmax_t)VM_CNT_FETCH(v_swappgsin), + (uintmax_t)VM_CNT_FETCH(v_swappgsout), + (uintmax_t)VM_CNT_FETCH(v_intr), + (uintmax_t)VM_CNT_FETCH(v_swtch), (long long)boottime.tv_sec); return (0); } diff --git a/sys/fs/fuse/fuse_vnops.c b/sys/fs/fuse/fuse_vnops.c index 483f2941133c..0a12e192d36c 100644 --- a/sys/fs/fuse/fuse_vnops.c +++ b/sys/fs/fuse/fuse_vnops.c @@ -1793,8 +1793,8 @@ fuse_vnop_getpages(struct vop_getpages_args *ap) kva = (vm_offset_t)bp->b_data; pmap_qenter(kva, pages, npages); - PCPU_INC(cnt.v_vnodein); - PCPU_ADD(cnt.v_vnodepgsin, npages); + VM_CNT_INC(v_vnodein); + VM_CNT_ADD(v_vnodepgsin, npages); count = npages << PAGE_SHIFT; iov.iov_base = (caddr_t)kva; @@ -1927,8 +1927,8 @@ fuse_vnop_putpages(struct vop_putpages_args *ap) kva = (vm_offset_t)bp->b_data; pmap_qenter(kva, pages, npages); - PCPU_INC(cnt.v_vnodeout); - PCPU_ADD(cnt.v_vnodepgsout, count); + VM_CNT_INC(v_vnodeout); + VM_CNT_ADD(v_vnodepgsout, count); iov.iov_base = (caddr_t)kva; iov.iov_len = count; diff --git a/sys/fs/nfsclient/nfs_clbio.c b/sys/fs/nfsclient/nfs_clbio.c index df3aea9681bb..2b22ef9bdf3d 100644 --- a/sys/fs/nfsclient/nfs_clbio.c +++ b/sys/fs/nfsclient/nfs_clbio.c @@ -184,8 +184,8 @@ ncl_getpages(struct vop_getpages_args *ap) kva = (vm_offset_t) bp->b_data; pmap_qenter(kva, pages, npages); - PCPU_INC(cnt.v_vnodein); - PCPU_ADD(cnt.v_vnodepgsin, npages); + VM_CNT_INC(v_vnodein); + VM_CNT_ADD(v_vnodepgsin, npages); count = npages << PAGE_SHIFT; iov.iov_base = (caddr_t) kva; @@ -320,8 +320,8 @@ ncl_putpages(struct vop_putpages_args *ap) } mtx_unlock(&np->n_mtx); - PCPU_INC(cnt.v_vnodeout); - PCPU_ADD(cnt.v_vnodepgsout, count); + VM_CNT_INC(v_vnodeout); + VM_CNT_ADD(v_vnodepgsout, count); iov.iov_base = unmapped_buf; iov.iov_len = count; diff --git a/sys/fs/smbfs/smbfs_io.c b/sys/fs/smbfs/smbfs_io.c index 498d0637ff99..990336db6bcf 100644 --- a/sys/fs/smbfs/smbfs_io.c +++ b/sys/fs/smbfs/smbfs_io.c @@ -470,8 +470,8 @@ smbfs_getpages(ap) kva = (vm_offset_t) bp->b_data; pmap_qenter(kva, pages, npages); - PCPU_INC(cnt.v_vnodein); - PCPU_ADD(cnt.v_vnodepgsin, npages); + VM_CNT_INC(v_vnodein); + VM_CNT_ADD(v_vnodepgsin, npages); count = npages << PAGE_SHIFT; iov.iov_base = (caddr_t) kva; @@ -595,8 +595,8 @@ smbfs_putpages(ap) kva = (vm_offset_t) bp->b_data; pmap_qenter(kva, pages, npages); - PCPU_INC(cnt.v_vnodeout); - PCPU_ADD(cnt.v_vnodepgsout, count); + VM_CNT_INC(v_vnodeout); + VM_CNT_ADD(v_vnodepgsout, count); iov.iov_base = (caddr_t) kva; iov.iov_len = count; diff --git a/sys/i386/i386/trap.c b/sys/i386/i386/trap.c index 7c16b241605c..ef0a61c13382 100644 --- a/sys/i386/i386/trap.c +++ b/sys/i386/i386/trap.c @@ -192,7 +192,7 @@ trap(struct trapframe *frame) static int lastalert = 0; #endif - PCPU_INC(cnt.v_trap); + VM_CNT_INC(v_trap); type = frame->tf_trapno; #ifdef SMP diff --git a/sys/i386/include/atomic.h b/sys/i386/include/atomic.h index ccd1dbfff968..c167cf5c0fae 100644 --- a/sys/i386/include/atomic.h +++ b/sys/i386/include/atomic.h @@ -45,7 +45,7 @@ * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. * An assertion in i386/vm_machdep.c ensures that the value is correct. */ -#define __OFFSETOF_MONITORBUF 0x180 +#define __OFFSETOF_MONITORBUF 0x80 static __inline void __mbk(void) diff --git a/sys/i386/include/counter.h b/sys/i386/include/counter.h index 73b49cefc5be..bc4827e64fac 100644 --- a/sys/i386/include/counter.h +++ b/sys/i386/include/counter.h @@ -36,6 +36,10 @@ #include <machine/md_var.h> #include <machine/specialreg.h> +extern struct pcpu __pcpu[]; + +#define EARLY_COUNTER &__pcpu[0].pc_early_dummy_counter + #define counter_enter() do { \ if ((cpu_feature & CPUID_CX8) == 0) \ critical_enter(); \ @@ -46,8 +50,6 @@ critical_exit(); \ } while (0) -extern struct pcpu __pcpu[MAXCPU]; - static inline void counter_64_inc_8b(uint64_t *p, int64_t inc) { diff --git a/sys/i386/include/pcpu.h b/sys/i386/include/pcpu.h index 1ec6f580ea0f..30a06badc8ba 100644 --- a/sys/i386/include/pcpu.h +++ b/sys/i386/include/pcpu.h @@ -68,7 +68,7 @@ caddr_t pc_cmap_addr2; \ vm_offset_t pc_qmap_addr; /* KVA for temporary mappings */\ uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \ - char __pad[189] + char __pad[445] #ifdef _KERNEL diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c index 92bbcd72cea3..5931df8ba2d6 100644 --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -664,20 +664,20 @@ do_fork(struct thread *td, struct fork_req *fr, struct proc *p2, struct thread * vm_forkproc(td, p2, td2, vm2, fr->fr_flags); if (fr->fr_flags == (RFFDG | RFPROC)) { - PCPU_INC(cnt.v_forks); - PCPU_ADD(cnt.v_forkpages, p2->p_vmspace->vm_dsize + + VM_CNT_INC(v_forks); + VM_CNT_ADD(v_forkpages, p2->p_vmspace->vm_dsize + p2->p_vmspace->vm_ssize); } else if (fr->fr_flags == (RFFDG | RFPROC | RFPPWAIT | RFMEM)) { - PCPU_INC(cnt.v_vforks); - PCPU_ADD(cnt.v_vforkpages, p2->p_vmspace->vm_dsize + + VM_CNT_INC(v_vforks); + VM_CNT_ADD(v_vforkpages, p2->p_vmspace->vm_dsize + p2->p_vmspace->vm_ssize); } else if (p1 == &proc0) { - PCPU_INC(cnt.v_kthreads); - PCPU_ADD(cnt.v_kthreadpages, p2->p_vmspace->vm_dsize + + VM_CNT_INC(v_kthreads); + VM_CNT_ADD(v_kthreadpages, p2->p_vmspace->vm_dsize + p2->p_vmspace->vm_ssize); } else { - PCPU_INC(cnt.v_rforks); - PCPU_ADD(cnt.v_rforkpages, p2->p_vmspace->vm_dsize + + VM_CNT_INC(v_rforks); + VM_CNT_ADD(v_rforkpages, p2->p_vmspace->vm_dsize + p2->p_vmspace->vm_ssize); } diff --git a/sys/kern/kern_intr.c b/sys/kern/kern_intr.c index d7ff94ec2608..c9944089ef17 100644 --- a/sys/kern/kern_intr.c +++ b/sys/kern/kern_intr.c @@ -1156,7 +1156,7 @@ swi_sched(void *cookie, int flags) ih->ih_need = 1; if (!(flags & SWI_DELAY)) { - PCPU_INC(cnt.v_soft); + VM_CNT_INC(v_soft); #ifdef INTR_FILTER error = intr_event_schedule_thread(ie, ie->ie_thread); #else diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c index 19c9e17ce1e8..52c3b891df78 100644 --- a/sys/kern/kern_synch.c +++ b/sys/kern/kern_synch.c @@ -426,7 +426,7 @@ mi_switch(int flags, struct thread *newtd) td->td_incruntime += runtime; PCPU_SET(switchtime, new_switchtime); td->td_generation++; /* bump preempt-detect counter */ - PCPU_INC(cnt.v_swtch); + VM_CNT_INC(v_swtch); PCPU_SET(switchticks, ticks); CTR4(KTR_PROC, "mi_switch: old thread %ld (td_sched %p, pid %ld, %s)", td->td_tid, td_get_sched(td), td->td_proc->p_pid, td->td_name); diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c index a83fb78c0ae5..f7a354eb7305 100644 --- a/sys/kern/kern_thread.c +++ b/sys/kern/kern_thread.c @@ -546,7 +546,7 @@ thread_exit(void) td->td_incruntime += runtime; PCPU_SET(switchtime, new_switchtime); PCPU_SET(switchticks, ticks); - PCPU_INC(cnt.v_swtch); + VM_CNT_INC(v_swtch); /* Save our resource usage in our process. */ td->td_ru.ru_nvcsw++; diff --git a/sys/kern/subr_intr.c b/sys/kern/subr_intr.c index ca912458bf59..f856da261892 100644 --- a/sys/kern/subr_intr.c +++ b/sys/kern/subr_intr.c @@ -292,7 +292,7 @@ intr_irq_handler(struct trapframe *tf) KASSERT(irq_root_filter != NULL, ("%s: no filter", __func__)); - PCPU_INC(cnt.v_intr); + VM_CNT_INC(v_intr); critical_enter(); td = curthread; oldframe = td->td_intr_frame; diff --git a/sys/kern/subr_syscall.c b/sys/kern/subr_syscall.c index 8eeb4d87c60a..3c0a9323677b 100644 --- a/sys/kern/subr_syscall.c +++ b/sys/kern/subr_syscall.c @@ -58,7 +58,7 @@ syscallenter(struct thread *td, struct syscall_args *sa) struct proc *p; int error, traced; - PCPU_INC(cnt.v_syscall); + VM_CNT_INC(v_syscall); p = td->td_proc; td->td_pticks = 0; diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c index d86d5db2ceb7..d382628c6e7b 100644 --- a/sys/kern/subr_trap.c +++ b/sys/kern/subr_trap.c @@ -234,7 +234,7 @@ ast(struct trapframe *framep) td->td_flags &= ~(TDF_ASTPENDING | TDF_NEEDSIGCHK | TDF_NEEDSUSPCHK | TDF_NEEDRESCHED | TDF_ALRMPEND | TDF_PROFPEND | TDF_MACPEND); thread_unlock(td); - PCPU_INC(cnt.v_trap); + VM_CNT_INC(v_trap); if (td->td_cowgen != p->p_cowgen) thread_cow_update(td); diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c index 30cf49995d93..41b4828af192 100644 --- a/sys/kern/vfs_bio.c +++ b/sys/kern/vfs_bio.c @@ -4769,8 +4769,8 @@ vfs_bio_getpages(struct vnode *vp, vm_page_t *ma, int count, pgsin += pgsin_a; if (rahead != NULL) *rahead = pgsin_a; - PCPU_INC(cnt.v_vnodein); - PCPU_ADD(cnt.v_vnodepgsin, pgsin); + VM_CNT_INC(v_vnodein); + VM_CNT_ADD(v_vnodepgsin, pgsin); br_flags = (mp != NULL && (mp->mnt_kern_flag & MNTK_UNMAPPED_BUFS) != 0) ? GB_UNMAPPED : 0; diff --git a/sys/mips/include/counter.h b/sys/mips/include/counter.h index b293b296360c..c9ccf4c13c6d 100644 --- a/sys/mips/include/counter.h +++ b/sys/mips/include/counter.h @@ -34,6 +34,8 @@ #include <sys/proc.h> #endif +#define EARLY_COUNTER &((struct pcpu *)pcpu_space)->pc_early_dummy_counter + #define counter_enter() critical_enter() #define counter_exit() critical_exit() diff --git a/sys/mips/include/intr_machdep.h b/sys/mips/include/intr_machdep.h index 7c1d97636b73..07cb218aeade 100644 --- a/sys/mips/include/intr_machdep.h +++ b/sys/mips/include/intr_machdep.h @@ -71,6 +71,6 @@ mips_intrcnt_inc(mips_intrcnt_t counter) { if (counter) atomic_add_long(counter, 1); - PCPU_INC(cnt.v_intr); + VM_CNT_INC(v_intr); } #endif /* !_MACHINE_INTR_MACHDEP_H_ */ diff --git a/sys/mips/include/pcpu.h b/sys/mips/include/pcpu.h index 77a3983c8872..292e07b98c88 100644 --- a/sys/mips/include/pcpu.h +++ b/sys/mips/include/pcpu.h @@ -45,11 +45,11 @@ #ifdef __mips_n64 #define PCPU_MD_MIPS64_FIELDS \ PCPU_MD_COMMON_FIELDS \ - char __pad[53] + char __pad[245] #else #define PCPU_MD_MIPS32_FIELDS \ PCPU_MD_COMMON_FIELDS \ - char __pad[189] + char __pad[125] #endif #ifdef __mips_n64 diff --git a/sys/powerpc/include/counter.h b/sys/powerpc/include/counter.h index a38ff336770b..73dda27a78c4 100644 --- a/sys/powerpc/include/counter.h +++ b/sys/powerpc/include/counter.h @@ -34,6 +34,10 @@ #include <sys/proc.h> #endif +extern struct pcpu __pcpu[]; + +#define EARLY_COUNTER &__pcpu[0].pc_early_dummy_counter + #ifdef __powerpc64__ #define counter_enter() do {} while (0) @@ -79,8 +83,6 @@ counter_u64_zero_inline(counter_u64_t c) #define counter_u64_add_protected(c, i) counter_u64_add(c, i) -extern struct pcpu __pcpu[MAXCPU]; - static inline void counter_u64_add(counter_u64_t c, int64_t inc) { diff --git a/sys/powerpc/include/pcpu.h b/sys/powerpc/include/pcpu.h index 1a30e1efc053..7cf559c1303f 100644 --- a/sys/powerpc/include/pcpu.h +++ b/sys/powerpc/include/pcpu.h @@ -57,7 +57,7 @@ struct pvo_entry; vm_offset_t pc_qmap_addr; \ struct pvo_entry *pc_qmap_pvo; \ struct mtx pc_qmap_lock; \ - /* char __pad[0] */ + char __pad[128] #define PCPU_MD_AIM64_FIELDS \ struct slb pc_slb[64]; \ @@ -67,7 +67,7 @@ struct pvo_entry; vm_offset_t pc_qmap_addr; \ struct pvo_entry *pc_qmap_pvo; \ struct mtx pc_qmap_lock; \ - char __pad[1121 - sizeof(struct mtx)] + char __pad[1345] #ifdef __powerpc64__ #define PCPU_MD_AIM_FIELDS PCPU_MD_AIM64_FIELDS @@ -81,9 +81,9 @@ struct pvo_entry; #define BOOKE_TLBSAVE_LEN (BOOKE_TLB_SAVELEN * BOOKE_TLB_MAXNEST) #ifdef __powerpc64__ -#define BOOKE_PCPU_PAD 773 +#define BOOKE_PCPU_PAD 901 #else -#define BOOKE_PCPU_PAD 173 +#define BOOKE_PCPU_PAD 429 #endif #define PCPU_MD_BOOKE_FIELDS \ register_t pc_booke_critsave[BOOKE_CRITSAVE_LEN]; \ diff --git a/sys/powerpc/powerpc/trap.c b/sys/powerpc/powerpc/trap.c index 9002b7ee5802..526042580eab 100644 --- a/sys/powerpc/powerpc/trap.c +++ b/sys/powerpc/powerpc/trap.c @@ -171,7 +171,7 @@ trap(struct trapframe *frame) u_int ucode; ksiginfo_t ksi; - PCPU_INC(cnt.v_trap); + VM_CNT_INC(v_trap); td = curthread; p = td->td_proc; diff --git a/sys/sparc64/include/counter.h b/sys/sparc64/include/counter.h index 6beb1c51ef03..c725073dd0f6 100644 --- a/sys/sparc64/include/counter.h +++ b/sys/sparc64/include/counter.h @@ -34,6 +34,9 @@ #include <sys/proc.h> #endif +extern struct pcpu dummy_pcpu[]; +#define EARLY_COUNTER &dummy_pcpu[0].pc_early_dummy_counter + #define counter_enter() critical_enter() #define counter_exit() critical_exit() diff --git a/sys/sparc64/include/pcpu.h b/sys/sparc64/include/pcpu.h index 51524fba4a0a..c54c45b2443f 100644 --- a/sys/sparc64/include/pcpu.h +++ b/sys/sparc64/include/pcpu.h @@ -62,7 +62,7 @@ struct pmap; u_int pc_tlb_ctx; \ u_int pc_tlb_ctx_max; \ u_int pc_tlb_ctx_min; \ - char __pad[397] + char __pad[653] #ifdef _KERNEL diff --git a/sys/sparc64/sparc64/exception.S b/sys/sparc64/sparc64/exception.S index 8a1160ba80aa..dc5a7a0001e5 100644 --- a/sys/sparc64/sparc64/exception.S +++ b/sys/sparc64/sparc64/exception.S @@ -2476,9 +2476,8 @@ ENTRY(tl0_intr) inc %l1 stx %l1, [%l0] - lduw [PCPU(CNT) + V_INTR], %l0 - inc %l0 - stw %l0, [PCPU(CNT) + V_INTR] + call counter_intr_inc + nop ba,a %xcc, tl0_ret nop @@ -2989,11 +2988,8 @@ ENTRY(tl1_intr) add %l5, %l4, %l4 ldx [%l4], %l5 inc %l5 - stx %l5, [%l4] - - lduw [PCPU(CNT) + V_INTR], %l4 - inc %l4 - stw %l4, [PCPU(CNT) + V_INTR] + call counter_intr_inc + stx %l5, [%l4] ldx [%sp + SPOFF + CCFSZ + TF_Y], %l4 diff --git a/sys/sparc64/sparc64/genassym.c b/sys/sparc64/sparc64/genassym.c index 0b4a10ce8c51..164e37ad482b 100644 --- a/sys/sparc64/sparc64/genassym.c +++ b/sys/sparc64/sparc64/genassym.c @@ -134,7 +134,6 @@ ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid)); ASSYM(PC_IRHEAD, offsetof(struct pcpu, pc_irhead)); ASSYM(PC_IRTAIL, offsetof(struct pcpu, pc_irtail)); ASSYM(PC_IRFREE, offsetof(struct pcpu, pc_irfree)); -ASSYM(PC_CNT, offsetof(struct pcpu, pc_cnt)); ASSYM(PC_SIZEOF, sizeof(struct pcpu)); ASSYM(PC_CACHE, offsetof(struct pcpu, pc_cache)); diff --git a/sys/sparc64/sparc64/intr_machdep.c b/sys/sparc64/sparc64/intr_machdep.c index 6497a0516663..b31ce4c43d31 100644 --- a/sys/sparc64/sparc64/intr_machdep.c +++ b/sys/sparc64/sparc64/intr_machdep.c @@ -123,6 +123,7 @@ static void intr_stray_level(struct trapframe *); static void intr_stray_vector(void *); static int intrcnt_setname(const char *, int); static void intrcnt_updatename(int, const char *, int); +void counter_intr_inc(void); static void intrcnt_updatename(int vec, const char *name, int ispil) @@ -451,6 +452,19 @@ intr_describe(int vec, void *ih, const char *descr) return (error); } +/* + * Do VM_CNT_INC(intr), being in the interrupt context already. This is + * called from assembly. + * To avoid counter_enter() and appropriate assertion, unwrap VM_CNT_INC() + * and hardcode the actual increment. + */ +void +counter_intr_inc(void) +{ + + *(uint64_t *)zpcpu_get(vm_cnt.v_intr) += 1; +} + #ifdef SMP /* * Support for balancing interrupt sources across CPUs. For now we just diff --git a/sys/sparc64/sparc64/machdep.c b/sys/sparc64/sparc64/machdep.c index 7660a1e8b26b..e5f215b42d7e 100644 --- a/sys/sparc64/sparc64/machdep.c +++ b/sys/sparc64/sparc64/machdep.c @@ -126,6 +126,7 @@ long realmem; void *dpcpu0; char pcpu0[PCPU_PAGES * PAGE_SIZE]; +struct pcpu dummy_pcpu[MAXCPU]; struct trapframe frame0; vm_offset_t kstack0; diff --git a/sys/sparc64/sparc64/trap.c b/sys/sparc64/sparc64/trap.c index 83e6f9a3697e..d2281b11e133 100644 --- a/sys/sparc64/sparc64/trap.c +++ b/sys/sparc64/sparc64/trap.c @@ -267,7 +267,7 @@ trap(struct trapframe *tf) trap_msg[tf->tf_type & ~T_KERNEL], (TRAPF_USERMODE(tf) ? "user" : "kernel"), rdpr(pil)); - PCPU_INC(cnt.v_trap); + VM_CNT_INC(v_trap); if ((tf->tf_tstate & TSTATE_PRIV) == 0) { KASSERT(td != NULL, ("trap: curthread NULL")); diff --git a/sys/sys/pcpu.h b/sys/sys/pcpu.h index 4c92a00c8444..41756a84eddc 100644 --- a/sys/sys/pcpu.h +++ b/sys/sys/pcpu.h @@ -43,7 +43,6 @@ #include <sys/_sx.h> #include <sys/queue.h> #include <sys/_rmlock.h> -#include <sys/vmmeter.h> #include <sys/resource.h> #include <machine/pcpu.h> @@ -158,7 +157,6 @@ struct pcpu { u_int pc_cpuid; /* This cpu number */ STAILQ_ENTRY(pcpu) pc_allcpu; struct lock_list_entry *pc_spinlocks; - struct vmmeter pc_cnt; /* VM stats counters */ long pc_cp_time[CPUSTATES]; /* statclock ticks */ struct device *pc_device; void *pc_netisr; /* netisr SWI cookie */ @@ -166,6 +164,7 @@ struct pcpu { int pc_domain; /* Memory domain. */ struct rm_queue pc_rm_queue; /* rmlock list of trackers */ uintptr_t pc_dynamic; /* Dynamic per-cpu data area */ + uint64_t pc_early_dummy_counter; /* Startup time counter(9) */ /* * Keep MD fields last, so that CPU-specific variations on a diff --git a/sys/sys/vmmeter.h b/sys/sys/vmmeter.h index b5d0ef148dc8..d769258360d8 100644 --- a/sys/sys/vmmeter.h +++ b/sys/sys/vmmeter.h @@ -39,50 +39,84 @@ */ #define MAXSLP 20 +/* Systemwide totals computed every five seconds. */ +struct vmtotal { + int16_t t_rq; /* length of the run queue */ + int16_t t_dw; /* jobs in ``disk wait'' (neg priority) */ + int16_t t_pw; /* jobs in page wait */ + int16_t t_sl; /* jobs sleeping in core */ + int16_t t_sw; /* swapped out runnable/short block jobs */ + int32_t t_vm; /* total virtual memory */ + int32_t t_avm; /* active virtual memory */ + int32_t t_rm; /* total real memory in use */ + int32_t t_arm; /* active real memory */ + int32_t t_vmshr; /* shared virtual memory */ + int32_t t_avmshr; /* active shared virtual memory */ + int32_t t_rmshr; /* shared real memory */ + int32_t t_armshr; /* active shared real memory */ + int32_t t_free; /* free memory pages */ +}; + +#if defined(_KERNEL) || defined(_WANT_VMMETER) +#include <sys/counter.h> + /* * System wide statistics counters. * Locking: * a - locked by atomic operations * c - constant after initialization * f - locked by vm_page_queue_free_mtx - * p - locked by being in the PCPU and atomicity respect to interrupts + * p - uses counter(9) * q - changes are synchronized by the corresponding vm_pagequeue lock */ struct vmmeter { /* * General system activity. */ - u_int v_swtch; /* (p) context switches */ - u_int v_trap; /* (p) calls to trap */ - u_int v_syscall; /* (p) calls to syscall() */ - u_int v_intr; /* (p) device interrupts */ - u_int v_soft; /* (p) software interrupts */ + counter_u64_t v_swtch; /* (p) context switches */ + counter_u64_t v_trap; /* (p) calls to trap */ + counter_u64_t v_syscall; /* (p) calls to syscall() */ + counter_u64_t v_intr; /* (p) device interrupts */ + counter_u64_t v_soft; /* (p) software interrupts */ /* * Virtual memory activity. */ - u_int v_vm_faults; /* (p) address memory faults */ - u_int v_io_faults; /* (p) page faults requiring I/O */ - u_int v_cow_faults; /* (p) copy-on-writes faults */ - u_int v_cow_optim; /* (p) optimized copy-on-writes faults */ - u_int v_zfod; /* (p) pages zero filled on demand */ - u_int v_ozfod; /* (p) optimized zero fill pages */ - u_int v_swapin; /* (p) swap pager pageins */ - u_int v_swapout; /* (p) swap pager pageouts */ - u_int v_swappgsin; /* (p) swap pager pages paged in */ - u_int v_swappgsout; /* (p) swap pager pages paged out */ - u_int v_vnodein; /* (p) vnode pager pageins */ - u_int v_vnodeout; /* (p) vnode pager pageouts */ - u_int v_vnodepgsin; /* (p) vnode_pager pages paged in */ - u_int v_vnodepgsout; /* (p) vnode pager pages paged out */ - u_int v_intrans; /* (p) intransit blocking page faults */ - u_int v_reactivated; /* (p) pages reactivated by the pagedaemon */ - u_int v_pdwakeups; /* (p) times daemon has awaken from sleep */ - u_int v_pdpages; /* (p) pages analyzed by daemon */ - u_int v_pdshortfalls; /* (p) page reclamation shortfalls */ - - u_int v_dfree; /* (p) pages freed by daemon */ - u_int v_pfree; /* (p) pages freed by exiting processes */ - u_int v_tfree; /* (p) total pages freed */ + counter_u64_t v_vm_faults; /* (p) address memory faults */ + counter_u64_t v_io_faults; /* (p) page faults requiring I/O */ + counter_u64_t v_cow_faults; /* (p) copy-on-writes faults */ + counter_u64_t v_cow_optim; /* (p) optimized COW faults */ + counter_u64_t v_zfod; /* (p) pages zero filled on demand */ + counter_u64_t v_ozfod; /* (p) optimized zero fill pages */ + counter_u64_t v_swapin; /* (p) swap pager pageins */ + counter_u64_t v_swapout; /* (p) swap pager pageouts */ + counter_u64_t v_swappgsin; /* (p) swap pager pages paged in */ + counter_u64_t v_swappgsout; /* (p) swap pager pages paged out */ + counter_u64_t v_vnodein; /* (p) vnode pager pageins */ + counter_u64_t v_vnodeout; /* (p) vnode pager pageouts */ + counter_u64_t v_vnodepgsin; /* (p) vnode_pager pages paged in */ + counter_u64_t v_vnodepgsout; /* (p) vnode pager pages paged out */ + counter_u64_t v_intrans; /* (p) intransit blocking page faults */ + counter_u64_t v_reactivated; /* (p) reactivated by the pagedaemon */ + counter_u64_t v_pdwakeups; /* (p) times daemon has awaken */ + counter_u64_t v_pdpages; /* (p) pages analyzed by daemon */ + counter_u64_t v_pdshortfalls; /* (p) page reclamation shortfalls */ + + counter_u64_t v_dfree; /* (p) pages freed by daemon */ + counter_u64_t v_pfree; /* (p) pages freed by processes */ + counter_u64_t v_tfree; /* (p) total pages freed */ + /* + * Fork/vfork/rfork activity. + */ + counter_u64_t v_forks; /* (p) fork() calls */ + counter_u64_t v_vforks; /* (p) vfork() calls */ + counter_u64_t v_rforks; /* (p) rfork() calls */ + counter_u64_t v_kthreads; /* (p) fork() calls by kernel */ + counter_u64_t v_forkpages; /* (p) pages affected by fork() */ + counter_u64_t v_vforkpages; /* (p) pages affected by vfork() */ + counter_u64_t v_rforkpages; /* (p) pages affected by rfork() */ + counter_u64_t v_kthreadpages; /* (p) ... and by kernel fork() */ +#define VM_METER_NCOUNTERS \ + (offsetof(struct vmmeter, v_page_size) / sizeof(counter_u64_t)) /* * Distribution of page usages. */ @@ -100,24 +134,18 @@ struct vmmeter { u_int v_pageout_free_min; /* (c) min pages reserved for kernel */ u_int v_interrupt_free_min; /* (c) reserved pages for int code */ u_int v_free_severe; /* (c) severe page depletion point */ - /* - * Fork/vfork/rfork activity. - */ - u_int v_forks; /* (p) fork() calls */ - u_int v_vforks; /* (p) vfork() calls */ - u_int v_rforks; /* (p) rfork() calls */ - u_int v_kthreads; /* (p) fork() calls by kernel */ - u_int v_forkpages; /* (p) VM pages affected by fork() */ - u_int v_vforkpages; /* (p) VM pages affected by vfork() */ - u_int v_rforkpages; /* (p) VM pages affected by rfork() */ - u_int v_kthreadpages; /* (p) VM pages affected by fork() by kernel */ }; +#endif /* _KERNEL || _WANT_VMMETER */ + #ifdef _KERNEL extern struct vmmeter vm_cnt; - extern u_int vm_pageout_wakeup_thresh; +#define VM_CNT_ADD(var, x) counter_u64_add(vm_cnt.var, x) +#define VM_CNT_INC(var) VM_CNT_ADD(var, 1) +#define VM_CNT_FETCH(var) counter_u64_fetch(vm_cnt.var) + /* * Return TRUE if we are under our severe low-free-pages threshold * @@ -189,33 +217,5 @@ vm_laundry_target(void) return (vm_paging_target()); } - -/* - * Obtain the value of a per-CPU counter. - */ -#define VM_METER_PCPU_CNT(member) \ - vm_meter_cnt(__offsetof(struct vmmeter, member)) - -u_int vm_meter_cnt(size_t); - -#endif - -/* systemwide totals computed every five seconds */ -struct vmtotal { - int16_t t_rq; /* length of the run queue */ - int16_t t_dw; /* jobs in ``disk wait'' (neg priority) */ - int16_t t_pw; /* jobs in page wait */ - int16_t t_sl; /* jobs sleeping in core */ - int16_t t_sw; /* swapped out runnable/short block jobs */ - int32_t t_vm; /* total virtual memory */ - int32_t t_avm; /* active virtual memory */ - int32_t t_rm; /* total real memory in use */ - int32_t t_arm; /* active real memory */ - int32_t t_vmshr; /* shared virtual memory */ - int32_t t_avmshr; /* active shared virtual memory */ - int32_t t_rmshr; /* shared real memory */ - int32_t t_armshr; /* active shared real memory */ - int32_t t_free; /* free memory pages */ -}; - -#endif +#endif /* _KERNEL */ +#endif /* _SYS_VMMETER_H_ */ diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c index 33b3ab1359a9..76b69e5053e9 100644 --- a/sys/vm/swap_pager.c +++ b/sys/vm/swap_pager.c @@ -1180,8 +1180,8 @@ swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind, bp->b_pgbefore = rbehind != NULL ? *rbehind : 0; bp->b_pgafter = rahead != NULL ? *rahead : 0; - PCPU_INC(cnt.v_swapin); - PCPU_ADD(cnt.v_swappgsin, count); + VM_CNT_INC(v_swapin); + VM_CNT_ADD(v_swappgsin, count); /* * perform the I/O. NOTE!!! bp cannot be considered valid after @@ -1205,7 +1205,7 @@ swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind, VM_OBJECT_WLOCK(object); while ((m[0]->oflags & VPO_SWAPINPROG) != 0) { m[0]->oflags |= VPO_SWAPSLEEP; - PCPU_INC(cnt.v_intrans); + VM_CNT_INC(v_intrans); if (VM_OBJECT_SLEEP(object, &object->paging_in_progress, PSWP, "swread", hz * 20)) { printf( @@ -1393,8 +1393,8 @@ swap_pager_putpages(vm_object_t object, vm_page_t *m, int count, bp->b_dirtyoff = 0; bp->b_dirtyend = bp->b_bcount; - PCPU_INC(cnt.v_swapout); - PCPU_ADD(cnt.v_swappgsout, bp->b_npages); + VM_CNT_INC(v_swapout); + VM_CNT_ADD(v_swappgsout, bp->b_npages); /* * We unconditionally set rtvals[] to VM_PAGER_PEND so that we diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c index 2349e4938c01..0e584e5f0cb7 100644 --- a/sys/vm/vm_fault.c +++ b/sys/vm/vm_fault.c @@ -497,7 +497,7 @@ vm_fault_hold(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, boolean_t wired; /* Passed by reference. */ bool dead, growstack, hardfault, is_first_object_locked; - PCPU_INC(cnt.v_vm_faults); + VM_CNT_INC(v_vm_faults); fs.vp = NULL; faultcount = 0; nera = -1; @@ -673,7 +673,7 @@ RetryFault:; } vm_object_pip_wakeup(fs.object); VM_OBJECT_WUNLOCK(fs.object); - PCPU_INC(cnt.v_intrans); + VM_CNT_INC(v_intrans); vm_object_deallocate(fs.first_object); goto RetryFault; } @@ -999,9 +999,9 @@ readrest: if ((fs.m->flags & PG_ZERO) == 0) { pmap_zero_page(fs.m); } else { - PCPU_INC(cnt.v_ozfod); + VM_CNT_INC(v_ozfod); } - PCPU_INC(cnt.v_zfod); + VM_CNT_INC(v_zfod); fs.m->valid = VM_PAGE_BITS_ALL; /* Don't try to prefault neighboring pages. */ faultcount = 1; @@ -1095,7 +1095,7 @@ readrest: vm_page_xbusy(fs.m); fs.first_m = fs.m; fs.m = NULL; - PCPU_INC(cnt.v_cow_optim); + VM_CNT_INC(v_cow_optim); } else { /* * Oh, well, lets copy it. @@ -1131,7 +1131,7 @@ readrest: fs.m = fs.first_m; if (!is_first_object_locked) VM_OBJECT_WLOCK(fs.object); - PCPU_INC(cnt.v_cow_faults); + VM_CNT_INC(v_cow_faults); curthread->td_cow++; } else { prot &= ~VM_PROT_WRITE; @@ -1246,7 +1246,7 @@ readrest: */ unlock_and_deallocate(&fs); if (hardfault) { - PCPU_INC(cnt.v_io_faults); + VM_CNT_INC(v_io_faults); curthread->td_ru.ru_majflt++; #ifdef RACCT if (racct_enable && fs.object->type == OBJT_VNODE) { diff --git a/sys/vm/vm_meter.c b/sys/vm/vm_meter.c index 5f1789de50e5..5f4cd46ab1e4 100644 --- a/sys/vm/vm_meter.c +++ b/sys/vm/vm_meter.c @@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$"); #include <sys/systm.h> #include <sys/kernel.h> #include <sys/lock.h> +#include <sys/malloc.h> #include <sys/mutex.h> #include <sys/proc.h> #include <sys/resource.h> @@ -55,7 +56,52 @@ __FBSDID("$FreeBSD$"); #include <vm/vm_object.h> #include <sys/sysctl.h> -struct vmmeter vm_cnt; +struct vmmeter vm_cnt = { + .v_swtch = EARLY_COUNTER, + .v_trap = EARLY_COUNTER, + .v_syscall = EARLY_COUNTER, + .v_intr = EARLY_COUNTER, + .v_soft = EARLY_COUNTER, + .v_vm_faults = EARLY_COUNTER, + .v_io_faults = EARLY_COUNTER, + .v_cow_faults = EARLY_COUNTER, + .v_cow_optim = EARLY_COUNTER, + .v_zfod = EARLY_COUNTER, + .v_ozfod = EARLY_COUNTER, + .v_swapin = EARLY_COUNTER, + .v_swapout = EARLY_COUNTER, + .v_swappgsin = EARLY_COUNTER, + .v_swappgsout = EARLY_COUNTER, + .v_vnodein = EARLY_COUNTER, + .v_vnodeout = EARLY_COUNTER, + .v_vnodepgsin = EARLY_COUNTER, + .v_vnodepgsout = EARLY_COUNTER, + .v_intrans = EARLY_COUNTER, + .v_reactivated = EARLY_COUNTER, + .v_pdwakeups = EARLY_COUNTER, + .v_pdpages = EARLY_COUNTER, + .v_pdshortfalls = EARLY_COUNTER, + .v_dfree = EARLY_COUNTER, + .v_pfree = EARLY_COUNTER, + .v_tfree = EARLY_COUNTER, + .v_forks = EARLY_COUNTER, + .v_vforks = EARLY_COUNTER, + .v_rforks = EARLY_COUNTER, + .v_kthreads = EARLY_COUNTER, + .v_forkpages = EARLY_COUNTER, + .v_vforkpages = EARLY_COUNTER, + .v_rforkpages = EARLY_COUNTER, + .v_kthreadpages = EARLY_COUNTER, +}; + +static void +vmcounter_startup(void) +{ + counter_u64_t *cnt = (counter_u64_t *)&vm_cnt; + + COUNTER_ARRAY_ALLOC(cnt, VM_METER_NCOUNTERS, M_WAITOK); +} +SYSINIT(counter, SI_SUB_CPU, SI_ORDER_FOURTH + 1, vmcounter_startup, NULL); SYSCTL_UINT(_vm, VM_V_FREE_MIN, v_free_min, CTLFLAG_RW, &vm_cnt.v_free_min, 0, "Minimum low-free-pages threshold"); @@ -210,40 +256,6 @@ vmtotal(SYSCTL_HANDLER_ARGS) return (sysctl_handle_opaque(oidp, &total, sizeof(total), req)); } -/* - * vm_meter_cnt() - accumulate statistics from all cpus and the global cnt - * structure. - * - * The vmmeter structure is now per-cpu as well as global. Those - * statistics which can be kept on a per-cpu basis (to avoid cache - * stalls between cpus) can be moved to the per-cpu vmmeter. Remaining - * statistics, such as v_free_reserved, are left in the global - * structure. - */ -u_int -vm_meter_cnt(size_t offset) -{ - struct pcpu *pcpu; - u_int count; - int i; - - count = *(u_int *)((char *)&vm_cnt + offset); - CPU_FOREACH(i) { - pcpu = pcpu_find(i); - count += *(u_int *)((char *)&pcpu->pc_cnt + offset); - } - return (count); -} - -static int -cnt_sysctl(SYSCTL_HANDLER_ARGS) -{ - u_int count; - - count = vm_meter_cnt((char *)arg1 - (char *)&vm_cnt); - return (SYSCTL_OUT(req, &count, sizeof(count))); -} - SYSCTL_PROC(_vm, VM_TOTAL, vmtotal, CTLTYPE_OPAQUE|CTLFLAG_RD|CTLFLAG_MPSAFE, 0, sizeof(struct vmtotal), vmtotal, "S,vmtotal", "System virtual memory statistics"); @@ -255,9 +267,7 @@ static SYSCTL_NODE(_vm_stats, OID_AUTO, vm, CTLFLAG_RW, 0, SYSCTL_NODE(_vm_stats, OID_AUTO, misc, CTLFLAG_RW, 0, "VM meter misc stats"); #define VM_STATS(parent, var, descr) \ - SYSCTL_PROC(parent, OID_AUTO, var, \ - CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, &vm_cnt.var, 0, \ - cnt_sysctl, "IU", descr) + SYSCTL_COUNTER_U64(parent, OID_AUTO, var, CTLFLAG_RD, &vm_cnt.var, descr) #define VM_STATS_VM(var, descr) VM_STATS(_vm_stats_vm, var, descr) #define VM_STATS_SYS(var, descr) VM_STATS(_vm_stats_sys, var, descr) @@ -288,19 +298,6 @@ VM_STATS_VM(v_pdshortfalls, "Page reclamation shortfalls"); VM_STATS_VM(v_dfree, "Pages freed by pagedaemon"); VM_STATS_VM(v_pfree, "Pages freed by exiting processes"); VM_STATS_VM(v_tfree, "Total pages freed"); -VM_STATS_VM(v_page_size, "Page size in bytes"); -VM_STATS_VM(v_page_count, "Total number of pages in system"); -VM_STATS_VM(v_free_reserved, "Pages reserved for deadlock"); -VM_STATS_VM(v_free_target, "Pages desired free"); -VM_STATS_VM(v_free_min, "Minimum low-free-pages threshold"); -VM_STATS_VM(v_free_count, "Free pages"); -VM_STATS_VM(v_wire_count, "Wired pages"); -VM_STATS_VM(v_active_count, "Active pages"); -VM_STATS_VM(v_inactive_target, "Desired inactive pages"); -VM_STATS_VM(v_inactive_count, "Inactive pages"); -VM_STATS_VM(v_laundry_count, "Pages eligible for laundering"); -VM_STATS_VM(v_pageout_free_min, "Min pages reserved for kernel"); -VM_STATS_VM(v_interrupt_free_min, "Reserved pages for interrupt code"); VM_STATS_VM(v_forks, "Number of fork() calls"); VM_STATS_VM(v_vforks, "Number of vfork() calls"); VM_STATS_VM(v_rforks, "Number of rfork() calls"); @@ -310,6 +307,23 @@ VM_STATS_VM(v_vforkpages, "VM pages affected by vfork()"); VM_STATS_VM(v_rforkpages, "VM pages affected by rfork()"); VM_STATS_VM(v_kthreadpages, "VM pages affected by fork() by kernel"); +#define VM_STATS_UINT(var, descr) \ + SYSCTL_UINT(_vm_stats_vm, OID_AUTO, var, CTLFLAG_RD, &vm_cnt.var, 0, descr) +VM_STATS_UINT(v_page_size, "Page size in bytes"); +VM_STATS_UINT(v_page_count, "Total number of pages in system"); +VM_STATS_UINT(v_free_reserved, "Pages reserved for deadlock"); +VM_STATS_UINT(v_free_target, "Pages desired free"); +VM_STATS_UINT(v_free_min, "Minimum low-free-pages threshold"); +VM_STATS_UINT(v_free_count, "Free pages"); +VM_STATS_UINT(v_wire_count, "Wired pages"); +VM_STATS_UINT(v_active_count, "Active pages"); +VM_STATS_UINT(v_inactive_target, "Desired inactive pages"); +VM_STATS_UINT(v_inactive_count, "Inactive pages"); +VM_STATS_UINT(v_laundry_count, "Pages eligible for laundering"); +VM_STATS_UINT(v_pageout_free_min, "Min pages reserved for kernel"); +VM_STATS_UINT(v_interrupt_free_min, "Reserved pages for interrupt code"); +VM_STATS_UINT(v_free_severe, "Severe page depletion point"); + #ifdef COMPAT_FREEBSD11 /* * Provide compatibility sysctls for the benefit of old utilities which exit diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c index d57984b38826..022fbd57c882 100644 --- a/sys/vm/vm_object.c +++ b/sys/vm/vm_object.c @@ -765,7 +765,7 @@ vm_object_terminate(vm_object_t object) p->object = NULL; if (p->wire_count == 0) { vm_page_free(p); - PCPU_INC(cnt.v_pfree); + VM_CNT_INC(v_pfree); } vm_page_unlock(p); } diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index 5aeefbf1c09a..e6befe15b915 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -2748,7 +2748,7 @@ vm_page_free_toq(vm_page_t m) } else KASSERT(m->queue == PQ_NONE, ("vm_page_free_toq: unmanaged page %p is queued", m)); - PCPU_INC(cnt.v_tfree); + VM_CNT_INC(v_tfree); if (vm_page_sbusied(m)) panic("vm_page_free: freeing busy page %p", m); diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index 6c94f370971c..2e08fdf9d403 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -667,7 +667,7 @@ vm_pageout_object_deactivate_pages(pmap_t pmap, vm_object_t first_object, goto unlock_return; if (vm_page_busied(p)) continue; - PCPU_INC(cnt.v_pdpages); + VM_CNT_INC(v_pdpages); vm_page_lock(p); if (p->wire_count != 0 || p->hold_count != 0 || !pmap_page_exists_quick(pmap, p)) { @@ -1003,7 +1003,7 @@ scan: } if (act_delta != 0) { if (object->ref_count != 0) { - PCPU_INC(cnt.v_reactivated); + VM_CNT_INC(v_reactivated); vm_page_activate(m); /* @@ -1052,7 +1052,7 @@ scan: if (m->dirty == 0) { free_page: vm_page_free(m); - PCPU_INC(cnt.v_dfree); + VM_CNT_INC(v_dfree); } else if ((object->flags & OBJ_DEAD) == 0) { if (object->type != OBJT_SWAP && object->type != OBJT_DEFAULT) @@ -1187,7 +1187,7 @@ vm_pageout_laundry_worker(void *arg) KASSERT(shortfall_cycle >= 0, ("negative cycle %d", shortfall_cycle)); launder = 0; - wakeups = VM_METER_PCPU_CNT(v_pdwakeups); + wakeups = VM_CNT_FETCH(v_pdwakeups); /* * First determine whether we need to launder pages to meet a @@ -1378,7 +1378,7 @@ vm_pageout_scan(struct vm_domain *vmd, int pass) KASSERT(queue_locked, ("unlocked inactive queue")); KASSERT(vm_page_inactive(m), ("Inactive queue %p", m)); - PCPU_INC(cnt.v_pdpages); + VM_CNT_INC(v_pdpages); next = TAILQ_NEXT(m, plinks.q); /* @@ -1476,7 +1476,7 @@ unlock_page: } if (act_delta != 0) { if (object->ref_count != 0) { - PCPU_INC(cnt.v_reactivated); + VM_CNT_INC(v_reactivated); vm_page_activate(m); /* @@ -1521,7 +1521,7 @@ unlock_page: if (m->dirty == 0) { free_page: vm_page_free(m); - PCPU_INC(cnt.v_dfree); + VM_CNT_INC(v_dfree); --page_shortage; } else if ((object->flags & OBJ_DEAD) == 0) vm_page_launder(m); @@ -1551,7 +1551,7 @@ drop_page: if (pq->pq_cnt > 0 || atomic_load_acq_int(&swapdev_enabled)) { if (page_shortage > 0) { vm_laundry_request = VM_LAUNDRY_SHORTFALL; - PCPU_INC(cnt.v_pdshortfalls); + VM_CNT_INC(v_pdshortfalls); } else if (vm_laundry_request != VM_LAUNDRY_SHORTFALL) vm_laundry_request = VM_LAUNDRY_BACKGROUND; wakeup(&vm_laundry_request); @@ -1635,7 +1635,7 @@ drop_page: * The count for page daemon pages is updated after checking * the page for eligibility. */ - PCPU_INC(cnt.v_pdpages); + VM_CNT_INC(v_pdpages); /* * Check to see "how much" the page has been used. @@ -2036,7 +2036,7 @@ vm_pageout_worker(void *arg) if (mtx_sleep(&vm_pageout_wanted, &vm_page_queue_free_mtx, PDROP | PVM, "psleep", hz) == 0) { - PCPU_INC(cnt.v_pdwakeups); + VM_CNT_INC(v_pdwakeups); pass = 1; } else pass = 0; diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c index f52016d68f00..aba50c6a5b58 100644 --- a/sys/vm/vnode_pager.c +++ b/sys/vm/vnode_pager.c @@ -799,8 +799,8 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int count, relpbuf(bp, freecnt); VM_OBJECT_WLOCK(object); for (i = 0; i < count; i++) { - PCPU_INC(cnt.v_vnodein); - PCPU_INC(cnt.v_vnodepgsin); + VM_CNT_INC(v_vnodein); + VM_CNT_INC(v_vnodepgsin); error = vnode_pager_input_old(object, m[i]); if (error) break; @@ -819,8 +819,8 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int count, if (pagesperblock == 0) { relpbuf(bp, freecnt); for (i = 0; i < count; i++) { - PCPU_INC(cnt.v_vnodein); - PCPU_INC(cnt.v_vnodepgsin); + VM_CNT_INC(v_vnodein); + VM_CNT_INC(v_vnodepgsin); error = vnode_pager_input_smlfs(object, m[i]); if (error) break; @@ -1030,8 +1030,8 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int count, (uintmax_t)blkno0, (uintmax_t)bp->b_blkno)); atomic_add_long(&runningbufspace, bp->b_runningbufspace); - PCPU_INC(cnt.v_vnodein); - PCPU_ADD(cnt.v_vnodepgsin, bp->b_npages); + VM_CNT_INC(v_vnodein); + VM_CNT_ADD(v_vnodepgsin, bp->b_npages); if (iodone != NULL) { /* async */ bp->b_pgiodone = iodone; @@ -1276,8 +1276,8 @@ vnode_pager_generic_putpages(struct vnode *vp, vm_page_t *ma, int bytecount, auio.uio_td = (struct thread *) 0; error = VOP_WRITE(vp, &auio, vnode_pager_putpages_ioflags(flags), curthread->td_ucred); - PCPU_INC(cnt.v_vnodeout); - PCPU_ADD(cnt.v_vnodepgsout, ncount); + VM_CNT_INC(v_vnodeout); + VM_CNT_ADD(v_vnodepgsout, ncount); ppscheck = 0; if (error) { diff --git a/sys/x86/acpica/srat.c b/sys/x86/acpica/srat.c index 39210e971b6c..9af9643594e4 100644 --- a/sys/x86/acpica/srat.c +++ b/sys/x86/acpica/srat.c @@ -31,6 +31,7 @@ __FBSDID("$FreeBSD$"); #include "opt_vm.h" #include <sys/param.h> +#include <sys/systm.h> #include <sys/bus.h> #include <sys/kernel.h> #include <sys/lock.h> diff --git a/sys/x86/x86/intr_machdep.c b/sys/x86/x86/intr_machdep.c index 7392a4646768..bb47a3e9aaaf 100644 --- a/sys/x86/x86/intr_machdep.c +++ b/sys/x86/x86/intr_machdep.c @@ -246,7 +246,7 @@ intr_execute_handlers(struct intsrc *isrc, struct trapframe *frame) * processed too. */ (*isrc->is_count)++; - PCPU_INC(cnt.v_intr); + VM_CNT_INC(v_intr); ie = isrc->is_event; diff --git a/usr.bin/top/machine.c b/usr.bin/top/machine.c index f1af866fd3e9..df36c45f8bbb 100644 --- a/usr.bin/top/machine.c +++ b/usr.bin/top/machine.c @@ -516,7 +516,7 @@ get_system_info(struct system_info *si) static int swapavail = 0; static int swapfree = 0; static long bufspace = 0; - static int nspgsin, nspgsout; + static uint64_t nspgsin, nspgsout; GETSYSCTL("vfs.bufspace", bufspace); GETSYSCTL("vm.stats.vm.v_active_count", memory_stats[0]); diff --git a/usr.bin/vmstat/vmstat.c b/usr.bin/vmstat/vmstat.c index bc734f045a12..58e96b6f33b5 100644 --- a/usr.bin/vmstat/vmstat.c +++ b/usr.bin/vmstat/vmstat.c @@ -54,6 +54,7 @@ __FBSDID("$FreeBSD$"); #include <sys/sysctl.h> #include <sys/time.h> #include <sys/user.h> +#define _WANT_VMMETER #include <sys/vmmeter.h> #include <sys/pcpu.h> @@ -127,7 +128,57 @@ static long select_generation; static char **specified_devices; static devstat_select_mode select_mode; -static struct vmmeter sum, osum; +static struct __vmmeter { + uint64_t v_swtch; + uint64_t v_trap; + uint64_t v_syscall; + uint64_t v_intr; + uint64_t v_soft; + uint64_t v_vm_faults; + uint64_t v_io_faults; + uint64_t v_cow_faults; + uint64_t v_cow_optim; + uint64_t v_zfod; + uint64_t v_ozfod; + uint64_t v_swapin; + uint64_t v_swapout; + uint64_t v_swappgsin; + uint64_t v_swappgsout; + uint64_t v_vnodein; + uint64_t v_vnodeout; + uint64_t v_vnodepgsin; + uint64_t v_vnodepgsout; + uint64_t v_intrans; + uint64_t v_reactivated; + uint64_t v_pdwakeups; + uint64_t v_pdpages; + uint64_t v_pdshortfalls; + uint64_t v_dfree; + uint64_t v_pfree; + uint64_t v_tfree; + uint64_t v_forks; + uint64_t v_vforks; + uint64_t v_rforks; + uint64_t v_kthreads; + uint64_t v_forkpages; + uint64_t v_vforkpages; + uint64_t v_rforkpages; + uint64_t v_kthreadpages; + u_int v_page_size; + u_int v_page_count; + u_int v_free_reserved; + u_int v_free_target; + u_int v_free_min; + u_int v_free_count; + u_int v_wire_count; + u_int v_active_count; + u_int v_inactive_target; + u_int v_inactive_count; + u_int v_laundry_count; + u_int v_pageout_free_min; + u_int v_interrupt_free_min; + u_int v_free_severe; +} sum, osum; #define VMSTAT_DEFAULT_LINES 20 /* Default number of `winlines'. */ volatile sig_atomic_t wresized; /* Tty resized, when non-zero. */ @@ -448,93 +499,50 @@ getuptime(void) } static void -fill_pcpu(struct pcpu ***pcpup, int* maxcpup) -{ - struct pcpu **pcpu; - - int maxcpu, i; - - *pcpup = NULL; - - if (kd == NULL) - return; - - maxcpu = kvm_getmaxcpu(kd); - if (maxcpu < 0) - xo_errx(1, "kvm_getmaxcpu: %s", kvm_geterr(kd)); - - pcpu = calloc(maxcpu, sizeof(struct pcpu *)); - if (pcpu == NULL) - xo_err(1, "calloc"); - - for (i = 0; i < maxcpu; i++) { - pcpu[i] = kvm_getpcpu(kd, i); - if (pcpu[i] == (struct pcpu *)-1) - xo_errx(1, "kvm_getpcpu: %s", kvm_geterr(kd)); - } - - *maxcpup = maxcpu; - *pcpup = pcpu; -} - -static void -free_pcpu(struct pcpu **pcpu, int maxcpu) -{ - int i; - - for (i = 0; i < maxcpu; i++) - free(pcpu[i]); - free(pcpu); -} - -static void -fill_vmmeter(struct vmmeter *vmmp) +fill_vmmeter(struct __vmmeter *vmmp) { struct pcpu **pcpu; int maxcpu, i; if (kd != NULL) { - kread(X_SUM, vmmp, sizeof(*vmmp)); - fill_pcpu(&pcpu, &maxcpu); - for (i = 0; i < maxcpu; i++) { - if (pcpu[i] == NULL) - continue; -#define ADD_FROM_PCPU(i, name) \ - vmmp->name += pcpu[i]->pc_cnt.name - ADD_FROM_PCPU(i, v_swtch); - ADD_FROM_PCPU(i, v_trap); - ADD_FROM_PCPU(i, v_syscall); - ADD_FROM_PCPU(i, v_intr); - ADD_FROM_PCPU(i, v_soft); - ADD_FROM_PCPU(i, v_vm_faults); - ADD_FROM_PCPU(i, v_io_faults); - ADD_FROM_PCPU(i, v_cow_faults); - ADD_FROM_PCPU(i, v_cow_optim); - ADD_FROM_PCPU(i, v_zfod); - ADD_FROM_PCPU(i, v_ozfod); - ADD_FROM_PCPU(i, v_swapin); - ADD_FROM_PCPU(i, v_swapout); - ADD_FROM_PCPU(i, v_swappgsin); - ADD_FROM_PCPU(i, v_swappgsout); - ADD_FROM_PCPU(i, v_vnodein); - ADD_FROM_PCPU(i, v_vnodeout); - ADD_FROM_PCPU(i, v_vnodepgsin); - ADD_FROM_PCPU(i, v_vnodepgsout); - ADD_FROM_PCPU(i, v_intrans); - ADD_FROM_PCPU(i, v_tfree); - ADD_FROM_PCPU(i, v_forks); - ADD_FROM_PCPU(i, v_vforks); - ADD_FROM_PCPU(i, v_rforks); - ADD_FROM_PCPU(i, v_kthreads); - ADD_FROM_PCPU(i, v_forkpages); - ADD_FROM_PCPU(i, v_vforkpages); - ADD_FROM_PCPU(i, v_rforkpages); - ADD_FROM_PCPU(i, v_kthreadpages); -#undef ADD_FROM_PCPU - } - free_pcpu(pcpu, maxcpu); + struct vmmeter vm_cnt; + + kread(X_SUM, &vm_cnt, sizeof(vm_cnt)); +#define GET_COUNTER(name) \ + vmmp->name = kvm_counter_u64_fetch(kd, (u_long)vm_cnt.name) + GET_COUNTER(v_swtch); + GET_COUNTER(v_trap); + GET_COUNTER(v_syscall); + GET_COUNTER(v_intr); + GET_COUNTER(v_soft); + GET_COUNTER(v_vm_faults); + GET_COUNTER(v_io_faults); + GET_COUNTER(v_cow_faults); + GET_COUNTER(v_cow_optim); + GET_COUNTER(v_zfod); + GET_COUNTER(v_ozfod); + GET_COUNTER(v_swapin); + GET_COUNTER(v_swapout); + GET_COUNTER(v_swappgsin); + GET_COUNTER(v_swappgsout); + GET_COUNTER(v_vnodein); + GET_COUNTER(v_vnodeout); + GET_COUNTER(v_vnodepgsin); + GET_COUNTER(v_vnodepgsout); + GET_COUNTER(v_intrans); + GET_COUNTER(v_tfree); + GET_COUNTER(v_forks); + GET_COUNTER(v_vforks); + GET_COUNTER(v_rforks); + GET_COUNTER(v_kthreads); + GET_COUNTER(v_forkpages); + GET_COUNTER(v_vforkpages); + GET_COUNTER(v_rforkpages); + GET_COUNTER(v_kthreadpages); +#undef GET_COUNTER } else { - size_t size = sizeof(unsigned int); + size_t size = sizeof(uint64_t); + #define GET_VM_STATS(cat, name) \ mysysctl("vm.stats." #cat "." #name, &vmmp->name, &size, NULL, 0) /* sys */ |