4 files changed, 76 insertions, 26 deletions

diff --git a/sys/vm/vm_domainset.c b/sys/vm/vm_domainset.c
index deb2f5162b38..e9473f302668 100644
--- a/sys/vm/vm_domainset.c
+++ b/sys/vm/vm_domainset.c
@@ -56,11 +56,14 @@ __FBSDID("$FreeBSD$");
  * assumed that most allocations are successful.
  */
 
+static int vm_domainset_default_stride = 64;
+
 /*
  * Determine which policy is to be used for this allocation.
  */
 static void
-vm_domainset_iter_domain(struct vm_domainset_iter *di, struct vm_object *obj)
+vm_domainset_iter_init(struct vm_domainset_iter *di, struct vm_object *obj,
+    vm_pindex_t pindex)
 {
 	struct domainset *domain;
 
@@ -76,18 +79,33 @@ vm_domainset_iter_domain(struct vm_domainset_iter *di, struct vm_object *obj)
 		di->di_domain = curthread->td_domain.dr_policy;
 		di->di_iter = &curthread->td_domain.dr_iterator;
 	}
+	di->di_policy = di->di_domain->ds_policy;
+	if (di->di_policy == DOMAINSET_POLICY_INTERLEAVE) {
+		if (vm_object_reserv(obj)) {
+			/*
+			 * Color the pindex so we end up on the correct
+			 * reservation boundary.
+			 */
+			pindex += obj->pg_color;
+			pindex >>= VM_LEVEL_0_ORDER;
+		} else
+			pindex /= vm_domainset_default_stride;
+		/*
+		 * Offset pindex so the first page of each object does
+		 * not end up in domain 0.
+		 */
+		if (obj != NULL)
+			pindex += (((uintptr_t)obj) / sizeof(*obj));
+		di->di_offset = pindex;
+	}
 }
 
 static void
 vm_domainset_iter_rr(struct vm_domainset_iter *di, int *domain)
 {
-	int d;
 
-	d = *di->di_iter;
-	do {
-		d = (d + 1) % di->di_domain->ds_max;
-	} while (!DOMAINSET_ISSET(d, &di->di_domain->ds_mask));
-	*di->di_iter = *domain = d;
+	*domain = di->di_domain->ds_order[
+	    ++(*di->di_iter) % di->di_domain->ds_cnt];
 }
 
 static void
@@ -95,12 +113,21 @@ vm_domainset_iter_prefer(struct vm_domainset_iter *di, int *domain)
 {
 	int d;
 
-	d = *di->di_iter;
 	do {
-		d = (d + 1) % di->di_domain->ds_max;
-	} while (!DOMAINSET_ISSET(d, &di->di_domain->ds_mask) || 
-	    d == di->di_domain->ds_prefer);
-	*di->di_iter = *domain = d;
+		d = di->di_domain->ds_order[
+		    ++(*di->di_iter) % di->di_domain->ds_cnt];
+	} while (d == di->di_domain->ds_prefer);
+	*domain = d;
+}
+
+static void
+vm_domainset_iter_interleave(struct vm_domainset_iter *di, int *domain)
+{
+	int d;
+
+	d = di->di_offset % di->di_domain->ds_cnt;
+	*di->di_iter = d;
+	*domain = di->di_domain->ds_order[d];
 }
 
 static void
@@ -109,13 +136,15 @@ vm_domainset_iter_next(struct vm_domainset_iter *di, int *domain)
 
 	KASSERT(di->di_n > 0,
 	    ("vm_domainset_iter_first: Invalid n %d", di->di_n));
-	switch (di->di_domain->ds_policy) {
+	switch (di->di_policy) {
 	case DOMAINSET_POLICY_FIRSTTOUCH:
 		/*
 		 * To prevent impossible allocations we convert an invalid
 		 * first-touch to round-robin.
 		 */
 		/* FALLTHROUGH */
+	case DOMAINSET_POLICY_INTERLEAVE:
+		/* FALLTHROUGH */
 	case DOMAINSET_POLICY_ROUNDROBIN:
 		vm_domainset_iter_rr(di, domain);
 		break;
@@ -124,7 +153,7 @@ vm_domainset_iter_next(struct vm_domainset_iter *di, int *domain)
 		break;
 	default:
 		panic("vm_domainset_iter_first: Unknown policy %d",
-		    di->di_domain->ds_policy);
+		    di->di_policy);
 	}
 	KASSERT(*domain < vm_ndomains,
 	    ("vm_domainset_iter_next: Invalid domain %d", *domain));
@@ -134,11 +163,15 @@ static void
 vm_domainset_iter_first(struct vm_domainset_iter *di, int *domain)
 {
 
-	switch (di->di_domain->ds_policy) {
+	switch (di->di_policy) {
 	case DOMAINSET_POLICY_FIRSTTOUCH:
 		*domain = PCPU_GET(domain);
 		if (DOMAINSET_ISSET(*domain, &di->di_domain->ds_mask)) {
-			di->di_n = 1;
+			/*
+			 * Add an extra iteration because we will visit the
+			 * current domain a second time in the rr iterator.
+			 */
+			di->di_n = di->di_domain->ds_cnt + 1;
 			break;
 		}
 		/*
@@ -154,9 +187,13 @@ vm_domainset_iter_first(struct vm_domainset_iter *di, int *domain)
 		*domain = di->di_domain->ds_prefer;
 		di->di_n = di->di_domain->ds_cnt;
 		break;
+	case DOMAINSET_POLICY_INTERLEAVE:
+		vm_domainset_iter_interleave(di, domain);
+		di->di_n = di->di_domain->ds_cnt;
+		break;
 	default:
 		panic("vm_domainset_iter_first: Unknown policy %d",
-		    di->di_domain->ds_policy);
+		    di->di_policy);
 	}
 	KASSERT(di->di_n > 0,
 	    ("vm_domainset_iter_first: Invalid n %d", di->di_n));
@@ -166,10 +203,10 @@ vm_domainset_iter_first(struct vm_domainset_iter *di, int *domain)
 
 void
 vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj,
-    int *domain, int *req)
+    vm_pindex_t pindex, int *domain, int *req)
 {
 
-	vm_domainset_iter_domain(di, obj);
+	vm_domainset_iter_init(di, obj, pindex);
 	di->di_flags = *req;
 	*req = (di->di_flags & ~(VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) |
 	    VM_ALLOC_NOWAIT;
@@ -213,7 +250,9 @@ vm_domainset_iter_malloc_init(struct vm_domainset_iter *di,
     struct vm_object *obj, int *domain, int *flags)
 {
 
-	vm_domainset_iter_domain(di, obj);
+	vm_domainset_iter_init(di, obj, 0);
+	if (di->di_policy == DOMAINSET_POLICY_INTERLEAVE)
+		di->di_policy = DOMAINSET_POLICY_ROUNDROBIN;
 	di->di_flags = *flags;
 	*flags = (di->di_flags & ~M_WAITOK) | M_NOWAIT;
 	vm_domainset_iter_first(di, domain);
@@ -253,7 +292,7 @@ vm_domainset_iter_page(struct vm_domainset_iter *di, int *domain, int *flags)
 
 void
 vm_domainset_iter_page_init(struct vm_domainset_iter *di,
-            struct vm_object *obj, int *domain, int *flags)
+            struct vm_object *obj, vm_pindex_t pindex, int *domain, int *flags)
 {
 
 	*domain = 0;
diff --git a/sys/vm/vm_domainset.h b/sys/vm/vm_domainset.h
index fa188f51db65..542fe47da677 100644
--- a/sys/vm/vm_domainset.h
+++ b/sys/vm/vm_domainset.h
@@ -33,13 +33,15 @@
 struct vm_domainset_iter {
 	struct domainset	*di_domain;
 	int			*di_iter;
+	vm_pindex_t		di_offset;
+	int			di_policy;
 	int			di_flags;
 	int			di_n;
 };
 
 int	vm_domainset_iter_page(struct vm_domainset_iter *, int *, int *);
 void	vm_domainset_iter_page_init(struct vm_domainset_iter *,
-	    struct vm_object *, int *, int *);
+	    struct vm_object *, vm_pindex_t, int *, int *);
 int	vm_domainset_iter_malloc(struct vm_domainset_iter *, int *, int *);
 void	vm_domainset_iter_malloc_init(struct vm_domainset_iter *,
 	    struct vm_object *, int *, int *);
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index 873a96be3517..f4de17703706 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -1660,7 +1660,7 @@ vm_page_alloc_after(vm_object_t object, vm_pindex_t pindex,
 	vm_page_t m;
 	int domain;
 
-	vm_domainset_iter_page_init(&di, object, &domain, &req);
+	vm_domainset_iter_page_init(&di, object, pindex, &domain, &req);
 	do {
 		m = vm_page_alloc_domain_after(object, pindex, domain, req,
 		    mpred);
@@ -1893,7 +1893,7 @@ vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
 	vm_page_t m;
 	int domain;
 
-	vm_domainset_iter_page_init(&di, object, &domain, &req);
+	vm_domainset_iter_page_init(&di, object, pindex, &domain, &req);
 	do {
 		m = vm_page_alloc_contig_domain(object, pindex, domain, req,
 		    npages, low, high, alignment, boundary, memattr);
@@ -2092,7 +2092,7 @@ vm_page_alloc_freelist(int freelist, int req)
 	vm_page_t m;
 	int domain;
 
-	vm_domainset_iter_page_init(&di, kernel_object, &domain, &req);
+	vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req);
 	do {
 		m = vm_page_alloc_freelist_domain(domain, freelist, req);
 		if (m != NULL)
@@ -2691,7 +2691,7 @@ vm_page_reclaim_contig(int req, u_long npages, vm_paddr_t low, vm_paddr_t high,
 	int domain;
 	bool ret;
 
-	vm_domainset_iter_page_init(&di, kernel_object, &domain, &req);
+	vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req);
 	do {
 		ret = vm_page_reclaim_contig_domain(domain, req, npages, low,
 		    high, alignment, boundary);
diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c
index ca982eb52fc2..52bac7ce373d 100644
--- a/sys/vm/vnode_pager.c
+++ b/sys/vm/vnode_pager.c
@@ -59,6 +59,7 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/sysctl.h>
 #include <sys/proc.h>
 #include <sys/vnode.h>
 #include <sys/mount.h>
@@ -69,6 +70,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/conf.h>
 #include <sys/rwlock.h>
 #include <sys/sf_buf.h>
+#include <sys/domainset.h>
 
 #include <machine/atomic.h>
 
@@ -108,6 +110,12 @@ struct pagerops vnodepagerops = {
 int vnode_pbuf_freecnt;
 int vnode_async_pbuf_freecnt;
 
+static struct domainset *vnode_domainset = NULL;
+
+SYSCTL_PROC(_debug, OID_AUTO, vnode_domainset, CTLTYPE_STRING | CTLFLAG_RW,
+    &vnode_domainset, 0, sysctl_handle_domainset, "A",
+    "Default vnode NUMA policy");
+
 /* Create the VM system backing object for this vnode */
 int
 vnode_create_vobject(struct vnode *vp, off_t isize, struct thread *td)
@@ -241,6 +249,7 @@ retry:
 
 		object->un_pager.vnp.vnp_size = size;
 		object->un_pager.vnp.writemappings = 0;
+		object->domain.dr_policy = vnode_domainset;
 
 		object->handle = handle;
 		VI_LOCK(vp);