2 files changed, 41 insertions, 13 deletions

diff --git a/sys/netgraph/netgraph.h b/sys/netgraph/netgraph.h
index ae4bf70dc080..5e26266098b0 100644
--- a/sys/netgraph/netgraph.h
+++ b/sys/netgraph/netgraph.h
@@ -411,7 +411,6 @@ static __inline void _chknode(node_p node, char *file, int line);
 static __inline char * _ng_node_name(node_p node, char *file, int line);
 static __inline int _ng_node_has_name(node_p node, char *file, int line);
 static __inline ng_ID_t _ng_node_id(node_p node, char *file, int line);
-void ng_ref_node(node_p node);
 static __inline void _ng_node_ref(node_p node, char *file, int line);
 static __inline int _ng_node_unref(node_p node, char *file, int line);
 static __inline void _ng_node_set_private(node_p node, void * val,
@@ -461,8 +460,7 @@ static __inline void
 _ng_node_ref(node_p node, char *file, int line)
 {
 	_chknode(node, file, line);
-	/*_NG_NODE_REF(node);*/
-	ng_ref_node(node);
+	_NG_NODE_REF(node);
 }
 
 static __inline int
diff --git a/sys/netgraph/ng_base.c b/sys/netgraph/ng_base.c
index ddc8f294cea2..c020d2cabda2 100644
--- a/sys/netgraph/ng_base.c
+++ b/sys/netgraph/ng_base.c
@@ -52,6 +52,7 @@
 #include <sys/errno.h>
 #include <sys/kdb.h>
 #include <sys/kernel.h>
+#include <sys/ktr.h>
 #include <sys/limits.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
@@ -732,14 +733,6 @@ ng_rmnode(node_p node, hook_p dummy1, void *dummy2, int dummy3)
 	NG_NODE_UNREF(node);
 }
 
-#ifdef	NETGRAPH_DEBUG
-void
-ng_ref_node(node_p node)
-{
-	_NG_NODE_REF(node);
-}
-#endif
-
 /*
  * Remove a reference to the node, possibly the last.
  * deadnode always acts as it it were the last.
@@ -1828,6 +1821,9 @@ ng_dequeue(struct ng_queue *ngq, int *rw)
 	 * XXXGL: assert this?
 	 */
 	if (!QUEUE_ACTIVE(ngq)) {
+		CTR4(KTR_NET, "%20s: node [%x] (%p) queue empty; "
+		    "queue flags 0x%lx", __func__,
+		    ngq->q_node->nd_ID, ngq->q_node, ngq->q_flags);
 		return (NULL);
 	}
 
@@ -1844,6 +1840,9 @@ ng_dequeue(struct ng_queue *ngq, int *rw)
 			 * get called again until something changes.
 			 */
 			ng_worklist_remove(ngq->q_node);
+			CTR4(KTR_NET, "%20s: node [%x] (%p) queued reader "
+			    "can't proceed; queue flags 0x%lx", __func__,
+			    ngq->q_node->nd_ID, ngq->q_node, ngq->q_flags);
 			return (NULL);
 		}
 		/*
@@ -1910,6 +1909,9 @@ ng_dequeue(struct ng_queue *ngq, int *rw)
 		 * would be a waste of effort to do all this again.
 		 */
 		ng_worklist_remove(ngq->q_node);
+		CTR4(KTR_NET, "%20s: node [%x] (%p) can't dequeue anything; "
+		    "queue flags 0x%lx", __func__,
+		    ngq->q_node->nd_ID, ngq->q_node, ngq->q_flags);
 		return (NULL);
 	}
 
@@ -1919,6 +1921,9 @@ ng_dequeue(struct ng_queue *ngq, int *rw)
 	 */
 	item = ngq->queue;
 	ngq->queue = item->el_next;
+	CTR6(KTR_NET, "%20s: node [%x] (%p) dequeued item %p with flags 0x%lx; "
+	    "queue flags 0x%lx", __func__,
+	    ngq->q_node->nd_ID,ngq->q_node, item, item->el_flags, ngq->q_flags);
 	if (ngq->last == &(item->el_next)) {
 		/*
 		 * that was the last entry in the queue so set the 'last
@@ -1949,6 +1954,10 @@ ng_dequeue(struct ng_queue *ngq, int *rw)
 			ng_setisr(ngq->q_node);
 		}
 	}
+	CTR6(KTR_NET, "%20s: node [%x] (%p) returning item %p as %s; "
+	    "queue flags 0x%lx", __func__,
+	    ngq->q_node->nd_ID, ngq->q_node, item, *rw ? "WRITER" : "READER" ,
+	    ngq->q_flags);
 	return (item);
 }
 
@@ -1969,12 +1978,17 @@ ng_queue_rw(struct ng_queue * ngq, item_p  item, int rw)
 		NGI_SET_READER(item);
 	item->el_next = NULL;	/* maybe not needed */
 	*ngq->last = item;
+	CTR5(KTR_NET, "%20s: node [%x] (%p) queued item %p as %s", __func__,
+	    ngq->q_node->nd_ID, ngq->q_node, item, rw ? "WRITER" : "READER" );
 	/*
 	 * If it was the first item in the queue then we need to
 	 * set the last pointer and the type flags.
 	 */
-	if (ngq->last == &(ngq->queue))
+	if (ngq->last == &(ngq->queue)) {
 		atomic_add_long(&ngq->q_flags, OP_PENDING);
+		CTR3(KTR_NET, "%20s: node [%x] (%p) set OP_PENDING", __func__,
+		    ngq->q_node->nd_ID, ngq->q_node);
+	}
 
 	ngq->last = &(item->el_next);
 	/*
@@ -2014,6 +2028,8 @@ ng_acquire_read(struct ng_queue *ngq, item_p item)
 	atomic_add_long(&ngq->q_flags, READER_INCREMENT);
 	if ((ngq->q_flags & NGQ_RMASK) == 0) {
 		/* Successfully grabbed node */
+		CTR4(KTR_NET, "%20s: node [%x] (%p) fast acquired item %p",
+		    __func__, ngq->q_node->nd_ID, ngq->q_node, item);
 		return (item);
 	}
 	/* undo the damage if we didn't succeed */
@@ -2032,6 +2048,8 @@ ng_acquire_read(struct ng_queue *ngq, item_p item)
 	if ((ngq->q_flags & NGQ_RMASK) == 0) {
 		atomic_add_long(&ngq->q_flags, READER_INCREMENT);
 		mtx_unlock_spin((&ngq->q_mtx));
+		CTR4(KTR_NET, "%20s: node [%x] (%p) slow acquired item %p",
+		    __func__, ngq->q_node->nd_ID, ngq->q_node, item);
 		return (item);
 	}
 
@@ -2066,6 +2084,8 @@ restart:
 			atomic_subtract_long(&ngq->q_flags, WRITER_ACTIVE);
 			goto restart;
 		}
+		CTR4(KTR_NET, "%20s: node [%x] (%p) acquired item %p",
+		    __func__, ngq->q_node->nd_ID, ngq->q_node, item);
 		return (item);
 	}
 
@@ -3228,6 +3248,8 @@ ngintr(void)
 		node->nd_flags &= ~NGF_WORKQ;	
 		TAILQ_REMOVE(&ng_worklist, node, nd_work);
 		mtx_unlock_spin(&ng_worklist_mtx);
+		CTR3(KTR_NET, "%20s: node [%x] (%p) taken off worklist",
+		    __func__, node->nd_ID, node);
 		/*
 		 * We have the node. We also take over the reference
 		 * that the list had on it.
@@ -3263,12 +3285,16 @@ ngintr(void)
 static void
 ng_worklist_remove(node_p node)
 {
+	mtx_assert(&node->nd_input_queue.q_mtx, MA_OWNED);
+
 	mtx_lock_spin(&ng_worklist_mtx);
 	if (node->nd_flags & NGF_WORKQ) {
 		node->nd_flags &= ~NGF_WORKQ;
 		TAILQ_REMOVE(&ng_worklist, node, nd_work);
 		mtx_unlock_spin(&ng_worklist_mtx);
 		NG_NODE_UNREF(node);
+		CTR3(KTR_NET, "%20s: node [%x] (%p) removed from worklist",
+		    __func__, node->nd_ID, node);
 	} else {
 		mtx_unlock_spin(&ng_worklist_mtx);
 	}
@@ -3295,7 +3321,11 @@ ng_setisr(node_p node)
 		TAILQ_INSERT_TAIL(&ng_worklist, node, nd_work);
 		mtx_unlock_spin(&ng_worklist_mtx);
 		NG_NODE_REF(node); /* XXX fafe in mutex? */
-	}
+		CTR3(KTR_NET, "%20s: node [%x] (%p) put on worklist", __func__,
+		    node->nd_ID, node);
+	} else
+		CTR3(KTR_NET, "%20s: node [%x] (%p) already on worklist",
+		    __func__, node->nd_ID, node);
 	schednetisr(NETISR_NETGRAPH);
 }