5 files changed, 321 insertions, 2 deletions

diff --git a/sys/dev/xdma/xdma.c b/sys/dev/xdma/xdma.c
index 839e62c63a11..d88cb1306536 100644
--- a/sys/dev/xdma/xdma.c
+++ b/sys/dev/xdma/xdma.c
@@ -70,6 +70,39 @@ static struct mtx xdma_mtx;
 
 #define	FDT_REG_CELLS	4
 
+#ifdef FDT
+static int
+xdma_get_iommu_fdt(xdma_controller_t *xdma, xdma_channel_t *xchan)
+{
+	struct xdma_iommu *xio;
+	phandle_t node;
+	pcell_t prop;
+	size_t len;
+
+	node = ofw_bus_get_node(xdma->dma_dev);
+	if (OF_getproplen(node, "xdma,iommu") <= 0)
+		return (0);
+
+	len = OF_getencprop(node, "xdma,iommu", &prop, sizeof(prop));
+	if (len != sizeof(prop)) {
+		device_printf(xdma->dev,
+		    "%s: Can't get iommu device node\n", __func__);
+		return (0);
+	}
+
+	xio = &xchan->xio;
+	xio->dev = OF_device_from_xref(prop);
+	if (xio->dev == NULL) {
+		device_printf(xdma->dev,
+		    "%s: Can't get iommu device\n", __func__);
+		return (0);
+	}
+
+	/* Found */
+	return (1);
+}
+#endif
+
 /*
  * Allocate virtual xDMA channel.
  */
@@ -81,6 +114,13 @@ xdma_channel_alloc(xdma_controller_t *xdma, uint32_t caps)
 
 	xchan = malloc(sizeof(xdma_channel_t), M_XDMA, M_WAITOK | M_ZERO);
 	xchan->xdma = xdma;
+
+#ifdef FDT
+	/* Check if this DMA controller supports IOMMU. */
+	if (xdma_get_iommu_fdt(xdma, xchan))
+		caps |= XCHAN_CAP_IOMMU | XCHAN_CAP_NOSEG;
+#endif
+
 	xchan->caps = caps;
 
 	XDMA_LOCK();
@@ -109,6 +149,9 @@ xdma_channel_alloc(xdma_controller_t *xdma, uint32_t caps)
 	TAILQ_INIT(&xchan->queue_out);
 	TAILQ_INIT(&xchan->processing);
 
+	if (xchan->caps & XCHAN_CAP_IOMMU)
+		xdma_iommu_init(&xchan->xio);
+
 	TAILQ_INSERT_TAIL(&xdma->channels, xchan, xchan_next);
 
 	XDMA_UNLOCK();
@@ -139,6 +182,9 @@ xdma_channel_free(xdma_channel_t *xchan)
 	if (xchan->flags & XCHAN_TYPE_SG)
 		xdma_channel_free_sg(xchan);
 
+	if (xchan->caps & XCHAN_CAP_IOMMU)
+		xdma_iommu_release(&xchan->xio);
+
 	xdma_teardown_all_intr(xchan);
 
 	mtx_destroy(&xchan->mtx_lock);
@@ -306,7 +352,7 @@ xdma_ofw_md_data(xdma_controller_t *xdma, pcell_t *cells, int ncells)
 	return (ret);
 }
 
-static int
+int
 xdma_handle_mem_node(vmem_t *vmem, phandle_t memory)
 {
 	pcell_t reg[FDT_REG_CELLS * FDT_MEM_REGIONS];
diff --git a/sys/dev/xdma/xdma.h b/sys/dev/xdma/xdma.h
index b645bd50adb3..48a03c125b3a 100644
--- a/sys/dev/xdma/xdma.h
+++ b/sys/dev/xdma/xdma.h
@@ -37,6 +37,14 @@
 #include <sys/proc.h>
 #include <sys/vmem.h>
 
+#ifdef FDT
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/openfirm.h>
+#endif
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
 enum xdma_direction {
 	XDMA_MEM_TO_MEM,
 	XDMA_MEM_TO_DEV,
@@ -121,6 +129,12 @@ struct xdma_sglist {
 	bool				last;
 };
 
+struct xdma_iommu {
+	struct pmap p;
+	vmem_t *vmem;		/* VA space */
+	device_t dev;		/* IOMMU device */
+};
+
 struct xdma_channel {
 	xdma_controller_t		*xdma;
 	vmem_t				*vmem;
@@ -138,6 +152,7 @@ struct xdma_channel {
 #define	XCHAN_CAP_BUSDMA		(1 << 0)
 #define	XCHAN_CAP_NOSEG			(1 << 1)
 #define	XCHAN_CAP_BOUNCE		(1 << 2)
+#define	XCHAN_CAP_IOMMU			(1 << 3)
 
 	/* A real hardware driver channel. */
 	void				*chan;
@@ -171,6 +186,9 @@ struct xdma_channel {
 	TAILQ_HEAD(, xdma_request)	queue_in;
 	TAILQ_HEAD(, xdma_request)	queue_out;
 	TAILQ_HEAD(, xdma_request)	processing;
+
+	/* iommu */
+	struct xdma_iommu		xio;
 };
 
 typedef struct xdma_channel xdma_channel_t;
@@ -216,6 +234,9 @@ xdma_controller_t *xdma_ofw_get(device_t dev, const char *prop);
 int xdma_put(xdma_controller_t *xdma);
 vmem_t * xdma_get_memory(device_t dev);
 void xdma_put_memory(vmem_t *vmem);
+#ifdef FDT
+int xdma_handle_mem_node(vmem_t *vmem, phandle_t memory);
+#endif
 
 /* xDMA channel ops */
 xdma_channel_t * xdma_channel_alloc(xdma_controller_t *, uint32_t caps);
@@ -271,4 +292,11 @@ int xchan_bank_free(xdma_channel_t *xchan);
 struct xdma_request * xchan_bank_get(xdma_channel_t *xchan);
 int xchan_bank_put(xdma_channel_t *xchan, struct xdma_request *xr);
 
+/* IOMMU */
+void xdma_iommu_add_entry(xdma_channel_t *xchan, vm_offset_t *va,
+    vm_paddr_t pa, vm_size_t size, vm_prot_t prot);
+void xdma_iommu_remove_entry(xdma_channel_t *xchan, vm_offset_t va);
+int xdma_iommu_init(struct xdma_iommu *xio);
+int xdma_iommu_release(struct xdma_iommu *xio);
+
 #endif /* !_DEV_XDMA_XDMA_H_ */
diff --git a/sys/dev/xdma/xdma_if.m b/sys/dev/xdma/xdma_if.m
index cbfb7fcbb3d1..cbf0c8e8cb05 100644
--- a/sys/dev/xdma/xdma_if.m
+++ b/sys/dev/xdma/xdma_if.m
@@ -1,5 +1,5 @@
 #-
-# Copyright (c) 2016-2018 Ruslan Bukin <br@bsdpad.com>
+# Copyright (c) 2016-2019 Ruslan Bukin <br@bsdpad.com>
 # All rights reserved.
 #
 # This software was developed by SRI International and the University of
@@ -113,3 +113,40 @@ METHOD int channel_control {
 	struct xdma_channel *xchan;
 	int cmd;
 };
+
+# IOMMU interface
+
+#
+# pmap is initialized
+#
+METHOD int iommu_init {
+	device_t dev;
+	struct xdma_iommu *xio;
+};
+
+#
+# pmap is released
+#
+METHOD int iommu_release {
+	device_t dev;
+	struct xdma_iommu *xio;
+};
+
+#
+# Mapping entered
+#
+METHOD int iommu_enter {
+	device_t dev;
+	struct xdma_iommu *xio;
+	vm_offset_t va;
+	vm_offset_t pa;
+};
+
+#
+# Mapping removed
+#
+METHOD int iommu_remove {
+	device_t dev;
+	struct xdma_iommu *xio;
+	vm_offset_t va;
+};
diff --git a/sys/dev/xdma/xdma_iommu.c b/sys/dev/xdma/xdma_iommu.c
new file mode 100644
index 000000000000..e0a70f40d896
--- /dev/null
+++ b/sys/dev/xdma/xdma_iommu.c
@@ -0,0 +1,174 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2019 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_platform.h"
+#include <sys/param.h>
+#include <sys/conf.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+#include <sys/mutex.h>
+#include <sys/rwlock.h>
+
+#include <machine/cache.h>
+#include <machine/bus.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_page.h>
+
+#ifdef FDT
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+#endif
+
+#include <dev/xdma/xdma.h>
+#include "xdma_if.h"
+
+void
+xdma_iommu_remove_entry(xdma_channel_t *xchan, vm_offset_t va)
+{
+	struct xdma_iommu *xio;
+
+	xio = &xchan->xio;
+
+	va &= ~(PAGE_SIZE - 1);
+	pmap_remove(&xio->p, va, va + PAGE_SIZE);
+
+	XDMA_IOMMU_REMOVE(xio->dev, xio, va);
+
+	vmem_free(xio->vmem, va, PAGE_SIZE);
+}
+
+static void
+xdma_iommu_enter(struct xdma_iommu *xio, vm_offset_t va,
+    vm_paddr_t pa, vm_size_t size, vm_prot_t prot)
+{
+	vm_page_t m;
+	pmap_t p;
+
+	p = &xio->p;
+
+	KASSERT((size & PAGE_MASK) == 0,
+	    ("%s: device mapping not page-sized", __func__));
+
+	for (; size > 0; size -= PAGE_SIZE) {
+		m = PHYS_TO_VM_PAGE(pa);
+		pmap_enter(p, va, m, prot, prot | PMAP_ENTER_WIRED, 0);
+
+		XDMA_IOMMU_ENTER(xio->dev, xio, va, pa);
+
+		va += PAGE_SIZE;
+		pa += PAGE_SIZE;
+	}
+}
+
+void
+xdma_iommu_add_entry(xdma_channel_t *xchan, vm_offset_t *va,
+    vm_paddr_t pa, vm_size_t size, vm_prot_t prot)
+{
+	struct xdma_iommu *xio;
+	vm_offset_t addr;
+
+	size = roundup2(size, PAGE_SIZE);
+	xio = &xchan->xio;
+
+	if (vmem_alloc(xio->vmem, size,
+	    M_FIRSTFIT | M_NOWAIT, &addr)) {
+		panic("Could not allocate virtual address.\n");
+	}
+
+	addr |= pa & (PAGE_SIZE - 1);
+
+	if (va)
+		*va = addr;
+
+	xdma_iommu_enter(xio, addr, pa, size, prot);
+}
+
+int
+xdma_iommu_init(struct xdma_iommu *xio)
+{
+#ifdef FDT
+	phandle_t mem_node, node;
+	pcell_t mem_handle;
+#endif
+
+	pmap_pinit(&xio->p);
+
+#ifdef FDT
+	node = ofw_bus_get_node(xio->dev);
+	if (!OF_hasprop(node, "va-region"))
+		return (ENXIO);
+
+	if (OF_getencprop(node, "va-region", (void *)&mem_handle,
+	    sizeof(mem_handle)) <= 0)
+		return (ENXIO);
+#endif
+
+	xio->vmem = vmem_create("xDMA vmem", 0, 0, PAGE_SIZE,
+	    PAGE_SIZE, M_FIRSTFIT | M_WAITOK);
+	if (xio->vmem == NULL)
+		return (ENXIO);
+
+#ifdef FDT
+	mem_node = OF_node_from_xref(mem_handle);
+	if (xdma_handle_mem_node(xio->vmem, mem_node) != 0) {
+		vmem_destroy(xio->vmem);
+		return (ENXIO);
+	}
+#endif
+
+	XDMA_IOMMU_INIT(xio->dev, xio);
+
+	return (0);
+}
+
+int
+xdma_iommu_release(struct xdma_iommu *xio)
+{
+
+	pmap_release(&xio->p);
+
+	vmem_destroy(xio->vmem);
+
+	XDMA_IOMMU_RELEASE(xio->dev, xio);
+
+	return (0);
+}
diff --git a/sys/dev/xdma/xdma_sg.c b/sys/dev/xdma/xdma_sg.c
index a1413ea62b48..d057ff3bb000 100644
--- a/sys/dev/xdma/xdma_sg.c
+++ b/sys/dev/xdma/xdma_sg.c
@@ -327,6 +327,7 @@ xchan_seg_done(xdma_channel_t *xchan,
 	struct xdma_request *xr;
 	xdma_controller_t *xdma;
 	struct xchan_buf *b;
+	bus_addr_t addr;
 
 	xdma = xchan->xdma;
 
@@ -352,6 +353,12 @@ xchan_seg_done(xdma_channel_t *xchan,
 			    xr->direction == XDMA_DEV_TO_MEM)
 				m_copyback(xr->m, 0, st->transferred,
 				    (void *)xr->buf.vaddr);
+		} else if (xchan->caps & XCHAN_CAP_IOMMU) {
+			if (xr->direction == XDMA_MEM_TO_DEV)
+				addr = xr->src_addr;
+			else
+				addr = xr->dst_addr;
+			xdma_iommu_remove_entry(xchan, addr);
 		}
 		xr->status.error = st->error;
 		xr->status.transferred = st->transferred;
@@ -484,11 +491,17 @@ _xdma_load_data(xdma_channel_t *xchan, struct xdma_request *xr,
 	xdma_controller_t *xdma;
 	struct mbuf *m;
 	uint32_t nsegs;
+	vm_offset_t va, addr;
+	bus_addr_t pa;
+	vm_prot_t prot;
 
 	xdma = xchan->xdma;
 
 	m = xr->m;
 
+	KASSERT(xchan->caps & XCHAN_CAP_NOSEG,
+	    ("Handling segmented data is not implemented here."));
+
 	nsegs = 1;
 
 	switch (xr->req_type) {
@@ -498,6 +511,27 @@ _xdma_load_data(xdma_channel_t *xchan, struct xdma_request *xr,
 				m_copydata(m, 0, m->m_pkthdr.len,
 				    (void *)xr->buf.vaddr);
 			seg[0].ds_addr = (bus_addr_t)xr->buf.paddr;
+		} else if (xchan->caps & XCHAN_CAP_IOMMU) {
+			addr = mtod(m, bus_addr_t);
+			pa = vtophys(addr);
+
+			if (xr->direction == XDMA_MEM_TO_DEV)
+				prot = VM_PROT_READ;
+			else
+				prot = VM_PROT_WRITE;
+
+			xdma_iommu_add_entry(xchan, &va,
+			    pa, m->m_pkthdr.len, prot);
+
+			/*
+			 * Save VA so we can unload data later
+			 * after completion of this transfer.
+			 */
+			if (xr->direction == XDMA_MEM_TO_DEV)
+				xr->src_addr = va;
+			else
+				xr->dst_addr = va;
+			seg[0].ds_addr = va;
 		} else
 			seg[0].ds_addr = mtod(m, bus_addr_t);
 		seg[0].ds_len = m->m_pkthdr.len;