7 files changed, 221 insertions, 64 deletions

diff --git a/sys/dev/xdma/xdma.c b/sys/dev/xdma/xdma.c
index 3e0a0f05f704..68aac5d5b7c8 100644
--- a/sys/dev/xdma/xdma.c
+++ b/sys/dev/xdma/xdma.c
@@ -1,6 +1,7 @@
 /*-
- * Copyright (c) 2016-2018 Ruslan Bukin <br@bsdpad.com>
- * All rights reserved.
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2016-2019 Ruslan Bukin <br@bsdpad.com>
  *
  * This software was developed by SRI International and the University of
  * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237
@@ -43,7 +44,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/sysctl.h>
 #include <sys/systm.h>
-#include <sys/sx.h>
 
 #include <machine/bus.h>
 
@@ -61,11 +61,13 @@ __FBSDID("$FreeBSD$");
  * Multiple xDMA controllers may work with single DMA device,
  * so we have global lock for physical channel management.
  */
-static struct sx xdma_sx;
+static struct mtx xdma_mtx;
+
+#define	XDMA_LOCK()			mtx_lock(&xdma_mtx)
+#define	XDMA_UNLOCK()			mtx_unlock(&xdma_mtx)
+#define	XDMA_ASSERT_LOCKED()		mtx_assert(&xdma_mtx, MA_OWNED)
 
-#define	XDMA_LOCK()			sx_xlock(&xdma_sx)
-#define	XDMA_UNLOCK()			sx_xunlock(&xdma_sx)
-#define	XDMA_ASSERT_LOCKED()		sx_xassert(&xdma_sx, MA_OWNED)
+#define	FDT_REG_CELLS	4
 
 /*
  * Allocate virtual xDMA channel.
@@ -95,11 +97,11 @@ xdma_channel_alloc(xdma_controller_t *xdma, uint32_t caps)
 
 	TAILQ_INIT(&xchan->ie_handlers);
 
-	sx_init(&xchan->sx_lock, "xDMA chan");
-	sx_init(&xchan->sx_qin_lock, "xDMA qin");
-	sx_init(&xchan->sx_qout_lock, "xDMA qout");
-	sx_init(&xchan->sx_bank_lock, "xDMA bank");
-	sx_init(&xchan->sx_proc_lock, "xDMA proc");
+	mtx_init(&xchan->mtx_lock, "xDMA chan", NULL, MTX_DEF);
+	mtx_init(&xchan->mtx_qin_lock, "xDMA qin", NULL, MTX_DEF);
+	mtx_init(&xchan->mtx_qout_lock, "xDMA qout", NULL, MTX_DEF);
+	mtx_init(&xchan->mtx_bank_lock, "xDMA bank", NULL, MTX_DEF);
+	mtx_init(&xchan->mtx_proc_lock, "xDMA proc", NULL, MTX_DEF);
 
 	TAILQ_INIT(&xchan->bank);
 	TAILQ_INIT(&xchan->queue_in);
@@ -138,11 +140,11 @@ xdma_channel_free(xdma_channel_t *xchan)
 
 	xdma_teardown_all_intr(xchan);
 
-	sx_destroy(&xchan->sx_lock);
-	sx_destroy(&xchan->sx_qin_lock);
-	sx_destroy(&xchan->sx_qout_lock);
-	sx_destroy(&xchan->sx_bank_lock);
-	sx_destroy(&xchan->sx_proc_lock);
+	mtx_destroy(&xchan->mtx_lock);
+	mtx_destroy(&xchan->mtx_qin_lock);
+	mtx_destroy(&xchan->mtx_qout_lock);
+	mtx_destroy(&xchan->mtx_bank_lock);
+	mtx_destroy(&xchan->mtx_proc_lock);
 
 	TAILQ_REMOVE(&xdma->channels, xchan, xchan_next);
 
@@ -303,6 +305,95 @@ xdma_ofw_md_data(xdma_controller_t *xdma, pcell_t *cells, int ncells)
 	return (ret);
 }
 
+static int
+xdma_handle_mem_node(vmem_t *vmem, phandle_t memory)
+{
+	pcell_t reg[FDT_REG_CELLS * FDT_MEM_REGIONS];
+	pcell_t *regp;
+	int addr_cells, size_cells;
+	int i, reg_len, ret, tuple_size, tuples;
+	vmem_addr_t mem_start;
+	vmem_size_t mem_size;
+
+	if ((ret = fdt_addrsize_cells(OF_parent(memory), &addr_cells,
+	    &size_cells)) != 0)
+		return (ret);
+
+	if (addr_cells > 2)
+		return (ERANGE);
+
+	tuple_size = sizeof(pcell_t) * (addr_cells + size_cells);
+	reg_len = OF_getproplen(memory, "reg");
+	if (reg_len <= 0 || reg_len > sizeof(reg))
+		return (ERANGE);
+
+	if (OF_getprop(memory, "reg", reg, reg_len) <= 0)
+		return (ENXIO);
+
+	tuples = reg_len / tuple_size;
+	regp = (pcell_t *)&reg;
+	for (i = 0; i < tuples; i++) {
+		ret = fdt_data_to_res(regp, addr_cells, size_cells,
+		    &mem_start, &mem_size);
+		if (ret != 0)
+			return (ret);
+
+		vmem_add(vmem, mem_start, mem_size, 0);
+		regp += addr_cells + size_cells;
+	}
+
+	return (0);
+}
+
+vmem_t *
+xdma_get_memory(device_t dev)
+{
+	phandle_t mem_node, node;
+	pcell_t mem_handle;
+	vmem_t *vmem;
+
+	node = ofw_bus_get_node(dev);
+	if (node <= 0) {
+		device_printf(dev,
+		    "%s called on not ofw based device.\n", __func__);
+		return (NULL);
+	}
+
+	if (!OF_hasprop(node, "memory-region"))
+		return (NULL);
+
+	if (OF_getencprop(node, "memory-region", (void *)&mem_handle,
+	    sizeof(mem_handle)) <= 0)
+		return (NULL);
+
+	vmem = vmem_create("xDMA vmem", 0, 0, PAGE_SIZE,
+	    PAGE_SIZE, M_BESTFIT | M_WAITOK);
+	if (vmem == NULL)
+		return (NULL);
+
+	mem_node = OF_node_from_xref(mem_handle);
+	if (xdma_handle_mem_node(vmem, mem_node) != 0) {
+		vmem_destroy(vmem);
+		return (NULL);
+	}
+
+	return (vmem);
+}
+
+void
+xdma_put_memory(vmem_t *vmem)
+{
+
+	vmem_destroy(vmem);
+}
+
+void
+xchan_set_memory(xdma_channel_t *xchan, vmem_t *vmem)
+{
+
+	xchan->vmem = vmem;
+}
+
 /*
  * Allocate xdma controller.
  */
@@ -400,7 +491,7 @@ static void
 xdma_init(void)
 {
 
-	sx_init(&xdma_sx, "xDMA");
+	mtx_init(&xdma_mtx, "xDMA", NULL, MTX_DEF);
 }
 
 SYSINIT(xdma, SI_SUB_DRIVERS, SI_ORDER_FIRST, xdma_init, NULL);
diff --git a/sys/dev/xdma/xdma.h b/sys/dev/xdma/xdma.h
index 36a773b85f41..583ad63e4e0f 100644
--- a/sys/dev/xdma/xdma.h
+++ b/sys/dev/xdma/xdma.h
@@ -1,6 +1,7 @@
 /*-
- * Copyright (c) 2016-2018 Ruslan Bukin <br@bsdpad.com>
- * All rights reserved.
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2016-2019 Ruslan Bukin <br@bsdpad.com>
  *
  * This software was developed by SRI International and the University of
  * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237
@@ -34,6 +35,7 @@
 #define _DEV_XDMA_XDMA_H_
 
 #include <sys/proc.h>
+#include <sys/vmem.h>
 
 enum xdma_direction {
 	XDMA_MEM_TO_MEM,
@@ -73,6 +75,7 @@ struct xdma_controller {
 	device_t dev;		/* DMA consumer device_t. */
 	device_t dma_dev;	/* A real DMA device_t. */
 	void *data;		/* OFW MD part. */
+	vmem_t *vmem;		/* Bounce memory. */
 
 	/* List of virtual channels allocated. */
 	TAILQ_HEAD(xdma_channel_list, xdma_channel)	channels;
@@ -84,6 +87,9 @@ struct xchan_buf {
 	bus_dmamap_t			map;
 	uint32_t			nsegs;
 	uint32_t			nsegs_left;
+	vm_offset_t			vaddr;
+	vm_offset_t			paddr;
+	vm_size_t			size;
 };
 
 struct xdma_request {
@@ -117,6 +123,7 @@ struct xdma_sglist {
 
 struct xdma_channel {
 	xdma_controller_t		*xdma;
+	vmem_t				*vmem;
 
 	uint32_t			flags;
 #define	XCHAN_BUFS_ALLOCATED		(1 << 0)
@@ -139,11 +146,11 @@ struct xdma_channel {
 	TAILQ_HEAD(, xdma_intr_handler)	ie_handlers;
 	TAILQ_ENTRY(xdma_channel)	xchan_next;
 
-	struct sx			sx_lock;
-	struct sx			sx_qin_lock;
-	struct sx			sx_qout_lock;
-	struct sx			sx_bank_lock;
-	struct sx			sx_proc_lock;
+	struct mtx			mtx_lock;
+	struct mtx			mtx_qin_lock;
+	struct mtx			mtx_qout_lock;
+	struct mtx			mtx_bank_lock;
+	struct mtx			mtx_proc_lock;
 
 	/* Request queue. */
 	bus_dma_tag_t			dma_tag_bufs;
@@ -176,30 +183,30 @@ struct xdma_intr_handler {
 
 static MALLOC_DEFINE(M_XDMA, "xdma", "xDMA framework");
 
-#define	XCHAN_LOCK(xchan)		sx_xlock(&(xchan)->sx_lock)
-#define	XCHAN_UNLOCK(xchan)		sx_xunlock(&(xchan)->sx_lock)
+#define	XCHAN_LOCK(xchan)		mtx_lock(&(xchan)->mtx_lock)
+#define	XCHAN_UNLOCK(xchan)		mtx_unlock(&(xchan)->mtx_lock)
 #define	XCHAN_ASSERT_LOCKED(xchan)	\
-    sx_assert(&(xchan)->sx_lock, SX_XLOCKED)
+    mtx_assert(&(xchan)->mtx_lock, MA_OWNED)
 
-#define	QUEUE_IN_LOCK(xchan)		sx_xlock(&(xchan)->sx_qin_lock)
-#define	QUEUE_IN_UNLOCK(xchan)		sx_xunlock(&(xchan)->sx_qin_lock)
+#define	QUEUE_IN_LOCK(xchan)		mtx_lock(&(xchan)->mtx_qin_lock)
+#define	QUEUE_IN_UNLOCK(xchan)		mtx_unlock(&(xchan)->mtx_qin_lock)
 #define	QUEUE_IN_ASSERT_LOCKED(xchan)	\
-    sx_assert(&(xchan)->sx_qin_lock, SX_XLOCKED)
+    mtx_assert(&(xchan)->mtx_qin_lock, MA_OWNED)
 
-#define	QUEUE_OUT_LOCK(xchan)		sx_xlock(&(xchan)->sx_qout_lock)
-#define	QUEUE_OUT_UNLOCK(xchan)		sx_xunlock(&(xchan)->sx_qout_lock)
+#define	QUEUE_OUT_LOCK(xchan)		mtx_lock(&(xchan)->mtx_qout_lock)
+#define	QUEUE_OUT_UNLOCK(xchan)		mtx_unlock(&(xchan)->mtx_qout_lock)
 #define	QUEUE_OUT_ASSERT_LOCKED(xchan)	\
-    sx_assert(&(xchan)->sx_qout_lock, SX_XLOCKED)
+    mtx_assert(&(xchan)->mtx_qout_lock, MA_OWNED)
 
-#define	QUEUE_BANK_LOCK(xchan)		sx_xlock(&(xchan)->sx_bank_lock)
-#define	QUEUE_BANK_UNLOCK(xchan)	sx_xunlock(&(xchan)->sx_bank_lock)
+#define	QUEUE_BANK_LOCK(xchan)		mtx_lock(&(xchan)->mtx_bank_lock)
+#define	QUEUE_BANK_UNLOCK(xchan)	mtx_unlock(&(xchan)->mtx_bank_lock)
 #define	QUEUE_BANK_ASSERT_LOCKED(xchan)	\
-    sx_assert(&(xchan)->sx_bank_lock, SX_XLOCKED)
+    mtx_assert(&(xchan)->mtx_bank_lock, MA_OWNED)
 
-#define	QUEUE_PROC_LOCK(xchan)		sx_xlock(&(xchan)->sx_proc_lock)
-#define	QUEUE_PROC_UNLOCK(xchan)	sx_xunlock(&(xchan)->sx_proc_lock)
+#define	QUEUE_PROC_LOCK(xchan)		mtx_lock(&(xchan)->mtx_proc_lock)
+#define	QUEUE_PROC_UNLOCK(xchan)	mtx_unlock(&(xchan)->mtx_proc_lock)
 #define	QUEUE_PROC_ASSERT_LOCKED(xchan)	\
-    sx_assert(&(xchan)->sx_proc_lock, SX_XLOCKED)
+    mtx_assert(&(xchan)->mtx_proc_lock, MA_OWNED)
 
 #define	XDMA_SGLIST_MAXLEN	2048
 #define	XDMA_MAX_SEG		128
@@ -207,11 +214,14 @@ static MALLOC_DEFINE(M_XDMA, "xdma", "xDMA framework");
 /* xDMA controller ops */
 xdma_controller_t *xdma_ofw_get(device_t dev, const char *prop);
 int xdma_put(xdma_controller_t *xdma);
+vmem_t * xdma_get_memory(device_t dev);
+void xdma_put_memory(vmem_t *vmem);
 
 /* xDMA channel ops */
 xdma_channel_t * xdma_channel_alloc(xdma_controller_t *, uint32_t caps);
 int xdma_channel_free(xdma_channel_t *);
 int xdma_request(xdma_channel_t *xchan, struct xdma_request *r);
+void xchan_set_memory(xdma_channel_t *xchan, vmem_t *vmem);
 
 /* SG interface */
 int xdma_prep_sg(xdma_channel_t *, uint32_t,
diff --git a/sys/dev/xdma/xdma_bank.c b/sys/dev/xdma/xdma_bank.c
index ac665410037f..dc59ba8a7d44 100644
--- a/sys/dev/xdma/xdma_bank.c
+++ b/sys/dev/xdma/xdma_bank.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2018 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2018-2019 Ruslan Bukin <br@bsdpad.com>
  * All rights reserved.
  *
  * This software was developed by SRI International and the University of
@@ -37,7 +37,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/bus.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
-#include <sys/sx.h>
 
 #include <machine/bus.h>
 
diff --git a/sys/dev/xdma/xdma_bio.c b/sys/dev/xdma/xdma_bio.c
index c158882706a1..dd8fd1d06b83 100644
--- a/sys/dev/xdma/xdma_bio.c
+++ b/sys/dev/xdma/xdma_bio.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2017-2018 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2017-2019 Ruslan Bukin <br@bsdpad.com>
  * All rights reserved.
  *
  * This software was developed by SRI International and the University of
@@ -37,7 +37,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/bus.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
-#include <sys/sx.h>
 
 #include <machine/bus.h>
 
diff --git a/sys/dev/xdma/xdma_mbuf.c b/sys/dev/xdma/xdma_mbuf.c
index ee5f564c1269..a8035aeee56d 100644
--- a/sys/dev/xdma/xdma_mbuf.c
+++ b/sys/dev/xdma/xdma_mbuf.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2017-2018 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2017-2019 Ruslan Bukin <br@bsdpad.com>
  * All rights reserved.
  *
  * This software was developed by SRI International and the University of
@@ -36,7 +36,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/conf.h>
 #include <sys/bus.h>
 #include <sys/kernel.h>
-#include <sys/sx.h>
 #include <sys/mbuf.h>
 
 #include <machine/bus.h>
diff --git a/sys/dev/xdma/xdma_queue.c b/sys/dev/xdma/xdma_queue.c
index 3d6a931be510..82ba831533e7 100644
--- a/sys/dev/xdma/xdma_queue.c
+++ b/sys/dev/xdma/xdma_queue.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2018 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2018-2019 Ruslan Bukin <br@bsdpad.com>
  * All rights reserved.
  *
  * This software was developed by SRI International and the University of
@@ -37,7 +37,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/bus.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
-#include <sys/sx.h>
 
 #include <machine/bus.h>
 
diff --git a/sys/dev/xdma/xdma_sg.c b/sys/dev/xdma/xdma_sg.c
index 6a6652679eb7..b8e050536313 100644
--- a/sys/dev/xdma/xdma_sg.c
+++ b/sys/dev/xdma/xdma_sg.c
@@ -1,6 +1,7 @@
 /*-
- * Copyright (c) 2018 Ruslan Bukin <br@bsdpad.com>
- * All rights reserved.
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2018-2019 Ruslan Bukin <br@bsdpad.com>
  *
  * This software was developed by SRI International and the University of
  * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237
@@ -38,10 +39,14 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
-#include <sys/sx.h>
+#include <sys/rwlock.h>
 
 #include <machine/bus.h>
 
+#include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_page.h>
+
 #ifdef FDT
 #include <dev/fdt/fdt_common.h>
 #include <dev/ofw/ofw_bus.h>
@@ -58,25 +63,71 @@ struct seg_load_request {
 	uint32_t error;
 };
 
+static void
+xchan_bufs_free_reserved(xdma_channel_t *xchan)
+{
+	struct xdma_request *xr;
+	vm_size_t size;
+	int i;
+
+	for (i = 0; i < xchan->xr_num; i++) {
+		xr = &xchan->xr_mem[i];
+		size = xr->buf.size;
+		if (xr->buf.vaddr) {
+			pmap_kremove_device(xr->buf.vaddr, size);
+			kva_free(xr->buf.vaddr, size);
+			xr->buf.vaddr = 0;
+		}
+		if (xr->buf.paddr) {
+			vmem_free(xchan->vmem, xr->buf.paddr, size);
+			xr->buf.paddr = 0;
+		}
+		xr->buf.size = 0;
+	}
+}
+
 static int
-_xchan_bufs_alloc(xdma_channel_t *xchan)
+xchan_bufs_alloc_reserved(xdma_channel_t *xchan)
 {
 	xdma_controller_t *xdma;
 	struct xdma_request *xr;
+	vmem_addr_t addr;
+	vm_size_t size;
 	int i;
 
 	xdma = xchan->xdma;
 
+	if (xchan->vmem == NULL)
+		return (ENOBUFS);
+
 	for (i = 0; i < xchan->xr_num; i++) {
 		xr = &xchan->xr_mem[i];
-		/* TODO: bounce buffer */
+		size = round_page(xchan->maxsegsize);
+		if (vmem_alloc(xchan->vmem, size,
+		    M_BESTFIT | M_NOWAIT, &addr)) {
+			device_printf(xdma->dev,
+			    "%s: Can't allocate memory\n", __func__);
+			xchan_bufs_free_reserved(xchan);
+			return (ENOMEM);
+		}
+		
+		xr->buf.size = size;
+		xr->buf.paddr = addr;
+		xr->buf.vaddr = kva_alloc(size);
+		if (xr->buf.vaddr == 0) {
+			device_printf(xdma->dev,
+			    "%s: Can't allocate KVA\n", __func__);
+			xchan_bufs_free_reserved(xchan);
+			return (ENOMEM);
+		}
+		pmap_kenter_device(xr->buf.vaddr, size, addr);
 	}
 
 	return (0);
 }
 
 static int
-_xchan_bufs_alloc_busdma(xdma_channel_t *xchan)
+xchan_bufs_alloc_busdma(xdma_channel_t *xchan)
 {
 	xdma_controller_t *xdma;
 	struct xdma_request *xr;
@@ -138,9 +189,10 @@ xchan_bufs_alloc(xdma_channel_t *xchan)
 	}
 
 	if (xchan->caps & XCHAN_CAP_BUSDMA)
-		ret = _xchan_bufs_alloc_busdma(xchan);
-	else
-		ret = _xchan_bufs_alloc(xchan);
+		ret = xchan_bufs_alloc_busdma(xchan);
+	else {
+		ret = xchan_bufs_alloc_reserved(xchan);
+	}
 	if (ret != 0) {
 		device_printf(xdma->dev,
 		    "%s: Can't allocate bufs.\n", __func__);
@@ -169,12 +221,8 @@ xchan_bufs_free(xdma_channel_t *xchan)
 			bus_dmamap_destroy(xchan->dma_tag_bufs, b->map);
 		}
 		bus_dma_tag_destroy(xchan->dma_tag_bufs);
-	} else {
-		for (i = 0; i < xchan->xr_num; i++) {
-			xr = &xchan->xr_mem[i];
-			/* TODO: bounce buffer */
-		}
-	}
+	} else
+		xchan_bufs_free_reserved(xchan);
 
 	xchan->flags &= ~XCHAN_BUFS_ALLOCATED;
 
@@ -296,6 +344,11 @@ xchan_seg_done(xdma_channel_t *xchan,
 				bus_dmamap_sync(xchan->dma_tag_bufs, b->map, 
 				    BUS_DMASYNC_POSTREAD);
 			bus_dmamap_unload(xchan->dma_tag_bufs, b->map);
+		} else {
+			if (xr->req_type == XR_TYPE_MBUF &&
+			    xr->direction == XDMA_DEV_TO_MEM)
+				m_copyback(xr->m, 0, st->transferred,
+				    (void *)xr->buf.vaddr);
 		}
 		xr->status.error = st->error;
 		xr->status.transferred = st->transferred;
@@ -437,7 +490,13 @@ _xdma_load_data(xdma_channel_t *xchan, struct xdma_request *xr,
 
 	switch (xr->req_type) {
 	case XR_TYPE_MBUF:
-		seg[0].ds_addr = mtod(m, bus_addr_t);
+		if ((xchan->caps & XCHAN_CAP_NOBUFS) == 0) {
+			if (xr->direction == XDMA_MEM_TO_DEV)
+				m_copydata(m, 0, m->m_pkthdr.len,
+				    (void *)xr->buf.vaddr);
+			seg[0].ds_addr = (bus_addr_t)xr->buf.paddr;
+		} else
+			seg[0].ds_addr = mtod(m, bus_addr_t);
 		seg[0].ds_len = m->m_pkthdr.len;
 		break;
 	case XR_TYPE_BIO:
@@ -494,6 +553,7 @@ xdma_process(xdma_channel_t *xchan,
 	xdma = xchan->xdma;
 
 	n = 0;
+	c = 0;
 
 	ret = XDMA_CHANNEL_CAPACITY(xdma->dma_dev, xchan, &capacity);
 	if (ret != 0) {