diff options
Diffstat (limited to 'sys/dev')
| -rw-r--r-- | sys/dev/virtio/mmio/virtio_mmio.c | 23 | ||||
| -rw-r--r-- | sys/dev/virtio/pci/virtio_pci.c | 1297 | ||||
| -rw-r--r-- | sys/dev/virtio/pci/virtio_pci.h | 173 | ||||
| -rw-r--r-- | sys/dev/virtio/pci/virtio_pci_if.m | 71 | ||||
| -rw-r--r-- | sys/dev/virtio/pci/virtio_pci_legacy.c | 733 | ||||
| -rw-r--r-- | sys/dev/virtio/pci/virtio_pci_legacy_var.h | 78 | ||||
| -rw-r--r-- | sys/dev/virtio/pci/virtio_pci_modern.c | 1448 | ||||
| -rw-r--r-- | sys/dev/virtio/pci/virtio_pci_modern_var.h | 135 | ||||
| -rw-r--r-- | sys/dev/virtio/pci/virtio_pci_var.h | 55 | ||||
| -rw-r--r-- | sys/dev/virtio/virtio.c | 76 | ||||
| -rw-r--r-- | sys/dev/virtio/virtio.h | 8 | ||||
| -rw-r--r-- | sys/dev/virtio/virtio_bus_if.m | 11 | ||||
| -rw-r--r-- | sys/dev/virtio/virtio_endian.h | 106 | ||||
| -rw-r--r-- | sys/dev/virtio/virtqueue.c | 10 | ||||
| -rw-r--r-- | sys/dev/virtio/virtqueue.h | 4 |
15 files changed, 3286 insertions, 942 deletions
diff --git a/sys/dev/virtio/mmio/virtio_mmio.c b/sys/dev/virtio/mmio/virtio_mmio.c index bb5d84754f09..694d2a232fdd 100644 --- a/sys/dev/virtio/mmio/virtio_mmio.c +++ b/sys/dev/virtio/mmio/virtio_mmio.c @@ -84,7 +84,7 @@ static void vtmmio_stop(device_t); static void vtmmio_poll(device_t); static int vtmmio_reinit(device_t, uint64_t); static void vtmmio_reinit_complete(device_t); -static void vtmmio_notify_virtqueue(device_t, uint16_t); +static void vtmmio_notify_virtqueue(device_t, uint16_t, bus_size_t); static uint8_t vtmmio_get_status(device_t); static void vtmmio_set_status(device_t, uint8_t); static void vtmmio_read_dev_config(device_t, bus_size_t, void *, int); @@ -352,6 +352,13 @@ vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) */ *result = 0; break; + case VIRTIO_IVAR_MODERN: + /* + * There are several modern (aka MMIO v2) spec compliance + * issues with this driver, but keep the status quo. + */ + *result = sc->vtmmio_version > 1; + break; default: return (ENOENT); } @@ -388,6 +395,10 @@ vtmmio_negotiate_features(device_t dev, uint64_t child_features) sc = device_get_softc(dev); + if (sc->vtmmio_version > 1) { + child_features |= VIRTIO_F_VERSION_1; + } + vtmmio_write_config_4(sc, VIRTIO_MMIO_HOST_FEATURES_SEL, 1); host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES); host_features <<= 32; @@ -402,7 +413,7 @@ vtmmio_negotiate_features(device_t dev, uint64_t child_features) * host all support. */ features = host_features & child_features; - features = virtqueue_filter_features(features); + features = virtio_filter_transport_features(features); sc->vtmmio_features = features; vtmmio_describe_features(sc, "negotiated", features); @@ -504,7 +515,8 @@ vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs, size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX); error = virtqueue_alloc(dev, idx, size, - VIRTIO_MMIO_VRING_ALIGN, ~(vm_paddr_t)0, info, &vq); + VIRTIO_MMIO_QUEUE_NOTIFY, VIRTIO_MMIO_VRING_ALIGN, + ~(vm_paddr_t)0, info, &vq); if (error) { device_printf(dev, "cannot allocate virtqueue %d: %d\n", @@ -586,13 +598,14 @@ vtmmio_reinit_complete(device_t dev) } static void -vtmmio_notify_virtqueue(device_t dev, uint16_t queue) +vtmmio_notify_virtqueue(device_t dev, uint16_t queue, bus_size_t offset) { struct vtmmio_softc *sc; sc = device_get_softc(dev); + MPASS(offset == VIRTIO_MMIO_QUEUE_NOTIFY); - vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue); + vtmmio_write_config_4(sc, offset, queue); } static uint8_t diff --git a/sys/dev/virtio/pci/virtio_pci.c b/sys/dev/virtio/pci/virtio_pci.c index 05a632f526a8..4d6fa929ef19 100644 --- a/sys/dev/virtio/pci/virtio_pci.c +++ b/sys/dev/virtio/pci/virtio_pci.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * - * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> + * Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -26,8 +26,6 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/* Driver for the VirtIO PCI interface. */ - #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); @@ -37,7 +35,6 @@ __FBSDID("$FreeBSD$"); #include <sys/kernel.h> #include <sys/module.h> #include <sys/malloc.h> -#include <sys/endian.h> #include <machine/bus.h> #include <machine/resource.h> @@ -50,369 +47,236 @@ __FBSDID("$FreeBSD$"); #include <dev/virtio/virtio.h> #include <dev/virtio/virtqueue.h> #include <dev/virtio/pci/virtio_pci.h> +#include <dev/virtio/pci/virtio_pci_var.h> -#include "virtio_bus_if.h" +#include "virtio_pci_if.h" #include "virtio_if.h" -struct vtpci_interrupt { - struct resource *vti_irq; - int vti_rid; - void *vti_handler; -}; - -struct vtpci_virtqueue { - struct virtqueue *vtv_vq; - int vtv_no_intr; -}; - -struct vtpci_softc { - device_t vtpci_dev; - struct resource *vtpci_res; - struct resource *vtpci_msix_res; - uint64_t vtpci_features; - uint32_t vtpci_flags; -#define VTPCI_FLAG_NO_MSI 0x0001 -#define VTPCI_FLAG_NO_MSIX 0x0002 -#define VTPCI_FLAG_LEGACY 0x1000 -#define VTPCI_FLAG_MSI 0x2000 -#define VTPCI_FLAG_MSIX 0x4000 -#define VTPCI_FLAG_SHARED_MSIX 0x8000 -#define VTPCI_FLAG_ITYPE_MASK 0xF000 - - /* This "bus" will only ever have one child. */ - device_t vtpci_child_dev; - struct virtio_feature_desc *vtpci_child_feat_desc; - - int vtpci_nvqs; - struct vtpci_virtqueue *vtpci_vqs; - - /* - * Ideally, each virtqueue that the driver provides a callback for will - * receive its own MSIX vector. If there are not sufficient vectors - * available, then attempt to have all the VQs share one vector. For - * MSIX, the configuration changed notifications must be on their own - * vector. - * - * If MSIX is not available, we will attempt to have the whole device - * share one MSI vector, and then, finally, one legacy interrupt. - */ - struct vtpci_interrupt vtpci_device_interrupt; - struct vtpci_interrupt *vtpci_msix_vq_interrupts; - int vtpci_nmsix_resources; -}; - -static int vtpci_probe(device_t); -static int vtpci_attach(device_t); -static int vtpci_detach(device_t); -static int vtpci_suspend(device_t); -static int vtpci_resume(device_t); -static int vtpci_shutdown(device_t); -static void vtpci_driver_added(device_t, driver_t *); -static void vtpci_child_detached(device_t, device_t); -static int vtpci_read_ivar(device_t, device_t, int, uintptr_t *); -static int vtpci_write_ivar(device_t, device_t, int, uintptr_t); - -static uint64_t vtpci_negotiate_features(device_t, uint64_t); -static int vtpci_with_feature(device_t, uint64_t); -static int vtpci_alloc_virtqueues(device_t, int, int, - struct vq_alloc_info *); -static int vtpci_setup_intr(device_t, enum intr_type); -static void vtpci_stop(device_t); -static int vtpci_reinit(device_t, uint64_t); -static void vtpci_reinit_complete(device_t); -static void vtpci_notify_virtqueue(device_t, uint16_t); -static uint8_t vtpci_get_status(device_t); -static void vtpci_set_status(device_t, uint8_t); -static void vtpci_read_dev_config(device_t, bus_size_t, void *, int); -static void vtpci_write_dev_config(device_t, bus_size_t, void *, int); - -static void vtpci_describe_features(struct vtpci_softc *, const char *, +static void vtpci_describe_features(struct vtpci_common *, const char *, uint64_t); -static void vtpci_probe_and_attach_child(struct vtpci_softc *); - -static int vtpci_alloc_msix(struct vtpci_softc *, int); -static int vtpci_alloc_msi(struct vtpci_softc *); -static int vtpci_alloc_intr_msix_pervq(struct vtpci_softc *); -static int vtpci_alloc_intr_msix_shared(struct vtpci_softc *); -static int vtpci_alloc_intr_msi(struct vtpci_softc *); -static int vtpci_alloc_intr_legacy(struct vtpci_softc *); -static int vtpci_alloc_interrupt(struct vtpci_softc *, int, int, +static int vtpci_alloc_msix(struct vtpci_common *, int); +static int vtpci_alloc_msi(struct vtpci_common *); +static int vtpci_alloc_intr_msix_pervq(struct vtpci_common *); +static int vtpci_alloc_intr_msix_shared(struct vtpci_common *); +static int vtpci_alloc_intr_msi(struct vtpci_common *); +static int vtpci_alloc_intr_intx(struct vtpci_common *); +static int vtpci_alloc_interrupt(struct vtpci_common *, int, int, + struct vtpci_interrupt *); +static void vtpci_free_interrupt(struct vtpci_common *, struct vtpci_interrupt *); -static int vtpci_alloc_intr_resources(struct vtpci_softc *); -static int vtpci_setup_legacy_interrupt(struct vtpci_softc *, +static void vtpci_free_interrupts(struct vtpci_common *); +static void vtpci_free_virtqueues(struct vtpci_common *); +static void vtpci_cleanup_setup_intr_attempt(struct vtpci_common *); +static int vtpci_alloc_intr_resources(struct vtpci_common *); +static int vtpci_setup_intx_interrupt(struct vtpci_common *, enum intr_type); -static int vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *, +static int vtpci_setup_pervq_msix_interrupts(struct vtpci_common *, enum intr_type); -static int vtpci_setup_msix_interrupts(struct vtpci_softc *, +static int vtpci_set_host_msix_vectors(struct vtpci_common *); +static int vtpci_setup_msix_interrupts(struct vtpci_common *, enum intr_type); -static int vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type); - -static int vtpci_register_msix_vector(struct vtpci_softc *, int, - struct vtpci_interrupt *); -static int vtpci_set_host_msix_vectors(struct vtpci_softc *); -static int vtpci_reinit_virtqueue(struct vtpci_softc *, int); - -static void vtpci_free_interrupt(struct vtpci_softc *, - struct vtpci_interrupt *); -static void vtpci_free_interrupts(struct vtpci_softc *); -static void vtpci_free_virtqueues(struct vtpci_softc *); -static void vtpci_release_child_resources(struct vtpci_softc *); -static void vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *); -static void vtpci_reset(struct vtpci_softc *); - -static void vtpci_select_virtqueue(struct vtpci_softc *, int); - -static void vtpci_legacy_intr(void *); +static int vtpci_setup_intrs(struct vtpci_common *, enum intr_type); +static int vtpci_reinit_virtqueue(struct vtpci_common *, int); +static void vtpci_intx_intr(void *); static int vtpci_vq_shared_intr_filter(void *); static void vtpci_vq_shared_intr(void *); static int vtpci_vq_intr_filter(void *); static void vtpci_vq_intr(void *); static void vtpci_config_intr(void *); -#define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt - -#define VIRTIO_PCI_CONFIG(_sc) \ - VIRTIO_PCI_CONFIG_OFF((((_sc)->vtpci_flags & VTPCI_FLAG_MSIX)) != 0) - -/* - * I/O port read/write wrappers. - */ -#define vtpci_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o)) -#define vtpci_write_config_1(sc, o, v) bus_write_1((sc)->vtpci_res, (o), (v)) +#define vtpci_setup_msi_interrupt vtpci_setup_intx_interrupt /* - * Virtio-pci specifies that PCI Configuration area is guest endian. However, - * since PCI devices are inherently little-endian, on BE systems the bus layer - * transparently converts it to BE. For virtio-legacy, this conversion is - * undesired, so an extra byte swap is required to fix it. + * This module contains two drivers: + * - virtio_pci_legacy for pre-V1 support + * - virtio_pci_modern for V1 support */ -#define vtpci_read_config_2(sc, o) le16toh(bus_read_2((sc)->vtpci_res, (o))) -#define vtpci_read_config_4(sc, o) le32toh(bus_read_4((sc)->vtpci_res, (o))) -#define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (htole16(v))) -#define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (htole32(v))) - -/* PCI Header LE. On BE systems the bus layer takes care of byte swapping */ -#define vtpci_read_header_2(sc, o) (bus_read_2((sc)->vtpci_res, (o))) -#define vtpci_read_header_4(sc, o) (bus_read_4((sc)->vtpci_res, (o))) -#define vtpci_write_header_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v)) -#define vtpci_write_header_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v)) - -/* Tunables. */ -static int vtpci_disable_msix = 0; -TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix); - -static device_method_t vtpci_methods[] = { - /* Device interface. */ - DEVMETHOD(device_probe, vtpci_probe), - DEVMETHOD(device_attach, vtpci_attach), - DEVMETHOD(device_detach, vtpci_detach), - DEVMETHOD(device_suspend, vtpci_suspend), - DEVMETHOD(device_resume, vtpci_resume), - DEVMETHOD(device_shutdown, vtpci_shutdown), - - /* Bus interface. */ - DEVMETHOD(bus_driver_added, vtpci_driver_added), - DEVMETHOD(bus_child_detached, vtpci_child_detached), - DEVMETHOD(bus_child_pnpinfo_str, virtio_child_pnpinfo_str), - DEVMETHOD(bus_read_ivar, vtpci_read_ivar), - DEVMETHOD(bus_write_ivar, vtpci_write_ivar), - - /* VirtIO bus interface. */ - DEVMETHOD(virtio_bus_negotiate_features, vtpci_negotiate_features), - DEVMETHOD(virtio_bus_with_feature, vtpci_with_feature), - DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_alloc_virtqueues), - DEVMETHOD(virtio_bus_setup_intr, vtpci_setup_intr), - DEVMETHOD(virtio_bus_stop, vtpci_stop), - DEVMETHOD(virtio_bus_reinit, vtpci_reinit), - DEVMETHOD(virtio_bus_reinit_complete, vtpci_reinit_complete), - DEVMETHOD(virtio_bus_notify_vq, vtpci_notify_virtqueue), - DEVMETHOD(virtio_bus_read_device_config, vtpci_read_dev_config), - DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config), - - DEVMETHOD_END -}; - -static driver_t vtpci_driver = { - "virtio_pci", - vtpci_methods, - sizeof(struct vtpci_softc) -}; - -devclass_t vtpci_devclass; - -DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0); MODULE_VERSION(virtio_pci, 1); MODULE_DEPEND(virtio_pci, pci, 1, 1, 1); MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1); -static int -vtpci_probe(device_t dev) -{ - char desc[36]; - const char *name; +int vtpci_disable_msix = 0; +TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix); - if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) - return (ENXIO); +static uint8_t +vtpci_read_isr(struct vtpci_common *cn) +{ + return (VIRTIO_PCI_READ_ISR(cn->vtpci_dev)); +} - if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || - pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX) - return (ENXIO); +static uint16_t +vtpci_get_vq_size(struct vtpci_common *cn, int idx) +{ + return (VIRTIO_PCI_GET_VQ_SIZE(cn->vtpci_dev, idx)); +} - if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION) - return (ENXIO); +static bus_size_t +vtpci_get_vq_notify_off(struct vtpci_common *cn, int idx) +{ + return (VIRTIO_PCI_GET_VQ_NOTIFY_OFF(cn->vtpci_dev, idx)); +} - name = virtio_device_name(pci_get_subdevice(dev)); - if (name == NULL) - name = "Unknown"; +static void +vtpci_set_vq(struct vtpci_common *cn, struct virtqueue *vq) +{ + VIRTIO_PCI_SET_VQ(cn->vtpci_dev, vq); +} - snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name); - device_set_desc_copy(dev, desc); +static void +vtpci_disable_vq(struct vtpci_common *cn, int idx) +{ + VIRTIO_PCI_DISABLE_VQ(cn->vtpci_dev, idx); +} - return (BUS_PROBE_DEFAULT); +static int +vtpci_register_cfg_msix(struct vtpci_common *cn, struct vtpci_interrupt *intr) +{ + return (VIRTIO_PCI_REGISTER_CFG_MSIX(cn->vtpci_dev, intr)); } static int -vtpci_attach(device_t dev) +vtpci_register_vq_msix(struct vtpci_common *cn, int idx, + struct vtpci_interrupt *intr) { - struct vtpci_softc *sc; - device_t child; - int rid; + return (VIRTIO_PCI_REGISTER_VQ_MSIX(cn->vtpci_dev, idx, intr)); +} - sc = device_get_softc(dev); - sc->vtpci_dev = dev; +void +vtpci_init(struct vtpci_common *cn, device_t dev, bool modern) +{ - pci_enable_busmaster(dev); + cn->vtpci_dev = dev; - rid = PCIR_BAR(0); - sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid, - RF_ACTIVE); - if (sc->vtpci_res == NULL) { - device_printf(dev, "cannot map I/O space\n"); - return (ENXIO); - } + pci_enable_busmaster(dev); + if (modern) + cn->vtpci_flags |= VTPCI_FLAG_MODERN; if (pci_find_cap(dev, PCIY_MSI, NULL) != 0) - sc->vtpci_flags |= VTPCI_FLAG_NO_MSI; - - if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) { - rid = PCIR_BAR(1); - sc->vtpci_msix_res = bus_alloc_resource_any(dev, - SYS_RES_MEMORY, &rid, RF_ACTIVE); - } - - if (sc->vtpci_msix_res == NULL) - sc->vtpci_flags |= VTPCI_FLAG_NO_MSIX; + cn->vtpci_flags |= VTPCI_FLAG_NO_MSI; + if (pci_find_cap(dev, PCIY_MSIX, NULL) != 0) + cn->vtpci_flags |= VTPCI_FLAG_NO_MSIX; +} - vtpci_reset(sc); +int +vtpci_add_child(struct vtpci_common *cn) +{ + device_t dev, child; - /* Tell the host we've noticed this device. */ - vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); + dev = cn->vtpci_dev; - if ((child = device_add_child(dev, NULL, -1)) == NULL) { + child = device_add_child(dev, NULL, -1); + if (child == NULL) { device_printf(dev, "cannot create child device\n"); - vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); - vtpci_detach(dev); return (ENOMEM); } - sc->vtpci_child_dev = child; - vtpci_probe_and_attach_child(sc); + cn->vtpci_child_dev = child; return (0); } -static int -vtpci_detach(device_t dev) +int +vtpci_delete_child(struct vtpci_common *cn) { - struct vtpci_softc *sc; - device_t child; + device_t dev, child; int error; - sc = device_get_softc(dev); + dev = cn->vtpci_dev; - if ((child = sc->vtpci_child_dev) != NULL) { + child = cn->vtpci_child_dev; + if (child != NULL) { error = device_delete_child(dev, child); if (error) return (error); - sc->vtpci_child_dev = NULL; - } - - vtpci_reset(sc); - - if (sc->vtpci_msix_res != NULL) { - bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1), - sc->vtpci_msix_res); - sc->vtpci_msix_res = NULL; - } - - if (sc->vtpci_res != NULL) { - bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0), - sc->vtpci_res); - sc->vtpci_res = NULL; + cn->vtpci_child_dev = NULL; } return (0); } -static int -vtpci_suspend(device_t dev) +void +vtpci_child_detached(struct vtpci_common *cn) { - return (bus_generic_suspend(dev)); -} - -static int -vtpci_resume(device_t dev) -{ + vtpci_release_child_resources(cn); - return (bus_generic_resume(dev)); + cn->vtpci_child_feat_desc = NULL; + cn->vtpci_features = 0; } -static int -vtpci_shutdown(device_t dev) +int +vtpci_reinit(struct vtpci_common *cn) { + int idx, error; - (void) bus_generic_shutdown(dev); - /* Forcibly stop the host device. */ - vtpci_stop(dev); + for (idx = 0; idx < cn->vtpci_nvqs; idx++) { + error = vtpci_reinit_virtqueue(cn, idx); + if (error) + return (error); + } + + if (vtpci_is_msix_enabled(cn)) { + error = vtpci_set_host_msix_vectors(cn); + if (error) + return (error); + } return (0); } static void -vtpci_driver_added(device_t dev, driver_t *driver) +vtpci_describe_features(struct vtpci_common *cn, const char *msg, + uint64_t features) { - struct vtpci_softc *sc; + device_t dev, child; + + dev = cn->vtpci_dev; + child = cn->vtpci_child_dev; - sc = device_get_softc(dev); + if (device_is_attached(child) || bootverbose == 0) + return; - vtpci_probe_and_attach_child(sc); + virtio_describe(dev, msg, features, cn->vtpci_child_feat_desc); } -static void -vtpci_child_detached(device_t dev, device_t child) +uint64_t +vtpci_negotiate_features(struct vtpci_common *cn, + uint64_t child_features, uint64_t host_features) { - struct vtpci_softc *sc; + uint64_t features; - sc = device_get_softc(dev); + vtpci_describe_features(cn, "host", host_features); - vtpci_reset(sc); - vtpci_release_child_resources(sc); + /* + * Limit negotiated features to what the driver, virtqueue, and + * host all support. + */ + features = host_features & child_features; + features = virtio_filter_transport_features(features); + vtpci_describe_features(cn, "negotiated", features); + + cn->vtpci_features = features; + + return (features); } -static int -vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) +int +vtpci_with_feature(struct vtpci_common *cn, uint64_t feature) { - struct vtpci_softc *sc; + return ((cn->vtpci_features & feature) != 0); +} - sc = device_get_softc(dev); +int +vtpci_read_ivar(struct vtpci_common *cn, int index, uintptr_t *result) +{ + device_t dev; + int error; - if (sc->vtpci_child_dev != child) - return (ENOENT); + dev = cn->vtpci_dev; + error = 0; switch (index) { - case VIRTIO_IVAR_DEVTYPE: case VIRTIO_IVAR_SUBDEVICE: *result = pci_get_subdevice(dev); break; @@ -425,100 +289,74 @@ vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) case VIRTIO_IVAR_SUBVENDOR: *result = pci_get_subvendor(dev); break; + case VIRTIO_IVAR_MODERN: + *result = vtpci_is_modern(cn); + break; default: - return (ENOENT); + error = ENOENT; } - return (0); + return (error); } -static int -vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value) +int +vtpci_write_ivar(struct vtpci_common *cn, int index, uintptr_t value) { - struct vtpci_softc *sc; - - sc = device_get_softc(dev); + int error; - if (sc->vtpci_child_dev != child) - return (ENOENT); + error = 0; switch (index) { case VIRTIO_IVAR_FEATURE_DESC: - sc->vtpci_child_feat_desc = (void *) value; + cn->vtpci_child_feat_desc = (void *) value; break; default: - return (ENOENT); + error = ENOENT; } - return (0); + return (error); } -static uint64_t -vtpci_negotiate_features(device_t dev, uint64_t child_features) +int +vtpci_alloc_virtqueues(struct vtpci_common *cn, int flags, int nvqs, + struct vq_alloc_info *vq_info) { - struct vtpci_softc *sc; - uint64_t host_features, features; - - sc = device_get_softc(dev); + device_t dev; + int idx, align, error; - host_features = vtpci_read_header_4(sc, VIRTIO_PCI_HOST_FEATURES); - vtpci_describe_features(sc, "host", host_features); + dev = cn->vtpci_dev; /* - * Limit negotiated features to what the driver, virtqueue, and - * host all support. + * This is VIRTIO_PCI_VRING_ALIGN from legacy VirtIO. In modern VirtIO, + * the tables do not have to be allocated contiguously, but we do so + * anyways. */ - features = host_features & child_features; - features = virtqueue_filter_features(features); - sc->vtpci_features = features; - - vtpci_describe_features(sc, "negotiated", features); - vtpci_write_header_4(sc, VIRTIO_PCI_GUEST_FEATURES, features); - - return (features); -} - -static int -vtpci_with_feature(device_t dev, uint64_t feature) -{ - struct vtpci_softc *sc; + align = 4096; - sc = device_get_softc(dev); - - return ((sc->vtpci_features & feature) != 0); -} - -static int -vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs, - struct vq_alloc_info *vq_info) -{ - struct vtpci_softc *sc; - struct virtqueue *vq; - struct vtpci_virtqueue *vqx; - struct vq_alloc_info *info; - int idx, error; - uint16_t size; - - sc = device_get_softc(dev); - - if (sc->vtpci_nvqs != 0) + if (cn->vtpci_nvqs != 0) return (EALREADY); if (nvqs <= 0) return (EINVAL); - sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue), + cn->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue), M_DEVBUF, M_NOWAIT | M_ZERO); - if (sc->vtpci_vqs == NULL) + if (cn->vtpci_vqs == NULL) return (ENOMEM); for (idx = 0; idx < nvqs; idx++) { - vqx = &sc->vtpci_vqs[idx]; + struct vtpci_virtqueue *vqx; + struct vq_alloc_info *info; + struct virtqueue *vq; + bus_size_t notify_offset; + uint16_t size; + + vqx = &cn->vtpci_vqs[idx]; info = &vq_info[idx]; - vtpci_select_virtqueue(sc, idx); - size = vtpci_read_header_2(sc, VIRTIO_PCI_QUEUE_NUM); + size = vtpci_get_vq_size(cn, idx); + notify_offset = vtpci_get_vq_notify_off(cn, idx); - error = virtqueue_alloc(dev, idx, size, VIRTIO_PCI_VRING_ALIGN, + error = virtqueue_alloc(dev, idx, size, notify_offset, align, ~(vm_paddr_t)0, info, &vq); if (error) { device_printf(dev, @@ -526,270 +364,27 @@ vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs, break; } - vtpci_write_header_4(sc, VIRTIO_PCI_QUEUE_PFN, - virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); + vtpci_set_vq(cn, vq); vqx->vtv_vq = *info->vqai_vq = vq; vqx->vtv_no_intr = info->vqai_intr == NULL; - sc->vtpci_nvqs++; + cn->vtpci_nvqs++; } if (error) - vtpci_free_virtqueues(sc); + vtpci_free_virtqueues(cn); return (error); } static int -vtpci_setup_intr(device_t dev, enum intr_type type) -{ - struct vtpci_softc *sc; - int attempt, error; - - sc = device_get_softc(dev); - - for (attempt = 0; attempt < 5; attempt++) { - /* - * Start with the most desirable interrupt configuration and - * fallback towards less desirable ones. - */ - switch (attempt) { - case 0: - error = vtpci_alloc_intr_msix_pervq(sc); - break; - case 1: - error = vtpci_alloc_intr_msix_shared(sc); - break; - case 2: - error = vtpci_alloc_intr_msi(sc); - break; - case 3: - error = vtpci_alloc_intr_legacy(sc); - break; - default: - device_printf(dev, - "exhausted all interrupt allocation attempts\n"); - return (ENXIO); - } - - if (error == 0 && vtpci_setup_interrupts(sc, type) == 0) - break; - - vtpci_cleanup_setup_intr_attempt(sc); - } - - if (bootverbose) { - if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) - device_printf(dev, "using legacy interrupt\n"); - else if (sc->vtpci_flags & VTPCI_FLAG_MSI) - device_printf(dev, "using MSI interrupt\n"); - else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) - device_printf(dev, "using shared MSIX interrupts\n"); - else - device_printf(dev, "using per VQ MSIX interrupts\n"); - } - - return (0); -} - -static void -vtpci_stop(device_t dev) -{ - - vtpci_reset(device_get_softc(dev)); -} - -static int -vtpci_reinit(device_t dev, uint64_t features) -{ - struct vtpci_softc *sc; - int idx, error; - - sc = device_get_softc(dev); - - /* - * Redrive the device initialization. This is a bit of an abuse of - * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to - * play nice. - * - * We do not allow the host device to change from what was originally - * negotiated beyond what the guest driver changed. MSIX state should - * not change, number of virtqueues and their size remain the same, etc. - * This will need to be rethought when we want to support migration. - */ - - if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET) - vtpci_stop(dev); - - /* - * Quickly drive the status through ACK and DRIVER. The device - * does not become usable again until vtpci_reinit_complete(). - */ - vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); - vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); - - vtpci_negotiate_features(dev, features); - - for (idx = 0; idx < sc->vtpci_nvqs; idx++) { - error = vtpci_reinit_virtqueue(sc, idx); - if (error) - return (error); - } - - if (sc->vtpci_flags & VTPCI_FLAG_MSIX) { - error = vtpci_set_host_msix_vectors(sc); - if (error) - return (error); - } - - return (0); -} - -static void -vtpci_reinit_complete(device_t dev) -{ - - vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); -} - -static void -vtpci_notify_virtqueue(device_t dev, uint16_t queue) -{ - struct vtpci_softc *sc; - - sc = device_get_softc(dev); - - vtpci_write_header_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue); -} - -static uint8_t -vtpci_get_status(device_t dev) -{ - struct vtpci_softc *sc; - - sc = device_get_softc(dev); - - return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS)); -} - -static void -vtpci_set_status(device_t dev, uint8_t status) -{ - struct vtpci_softc *sc; - - sc = device_get_softc(dev); - - if (status != VIRTIO_CONFIG_STATUS_RESET) - status |= vtpci_get_status(dev); - - vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status); -} - -static void -vtpci_read_dev_config(device_t dev, bus_size_t offset, - void *dst, int length) -{ - struct vtpci_softc *sc; - bus_size_t off; - uint8_t *d; - int size; - - sc = device_get_softc(dev); - off = VIRTIO_PCI_CONFIG(sc) + offset; - - for (d = dst; length > 0; d += size, off += size, length -= size) { - if (length >= 4) { - size = 4; - *(uint32_t *)d = vtpci_read_config_4(sc, off); - } else if (length >= 2) { - size = 2; - *(uint16_t *)d = vtpci_read_config_2(sc, off); - } else { - size = 1; - *d = vtpci_read_config_1(sc, off); - } - } -} - -static void -vtpci_write_dev_config(device_t dev, bus_size_t offset, - void *src, int length) -{ - struct vtpci_softc *sc; - bus_size_t off; - uint8_t *s; - int size; - - sc = device_get_softc(dev); - off = VIRTIO_PCI_CONFIG(sc) + offset; - - for (s = src; length > 0; s += size, off += size, length -= size) { - if (length >= 4) { - size = 4; - vtpci_write_config_4(sc, off, *(uint32_t *)s); - } else if (length >= 2) { - size = 2; - vtpci_write_config_2(sc, off, *(uint16_t *)s); - } else { - size = 1; - vtpci_write_config_1(sc, off, *s); - } - } -} - -static void -vtpci_describe_features(struct vtpci_softc *sc, const char *msg, - uint64_t features) -{ - device_t dev, child; - - dev = sc->vtpci_dev; - child = sc->vtpci_child_dev; - - if (device_is_attached(child) || bootverbose == 0) - return; - - virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc); -} - -static void -vtpci_probe_and_attach_child(struct vtpci_softc *sc) -{ - device_t dev, child; - - dev = sc->vtpci_dev; - child = sc->vtpci_child_dev; - - if (child == NULL) - return; - - if (device_get_state(child) != DS_NOTPRESENT) - return; - - if (device_probe(child) != 0) - return; - - vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); - if (device_attach(child) != 0) { - vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); - vtpci_reset(sc); - vtpci_release_child_resources(sc); - /* Reset status for future attempt. */ - vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); - } else { - vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); - VIRTIO_ATTACH_COMPLETED(child); - } -} - -static int -vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors) +vtpci_alloc_msix(struct vtpci_common *cn, int nvectors) { device_t dev; int nmsix, cnt, required; - dev = sc->vtpci_dev; + dev = cn->vtpci_dev; /* Allocate an additional vector for the config changes. */ required = nvectors + 1; @@ -800,7 +395,7 @@ vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors) cnt = required; if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { - sc->vtpci_nmsix_resources = required; + cn->vtpci_nmsix_resources = required; return (0); } @@ -810,12 +405,12 @@ vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors) } static int -vtpci_alloc_msi(struct vtpci_softc *sc) +vtpci_alloc_msi(struct vtpci_common *cn) { device_t dev; int nmsi, cnt, required; - dev = sc->vtpci_dev; + dev = cn->vtpci_dev; required = 1; nmsi = pci_msi_count(dev); @@ -832,80 +427,78 @@ vtpci_alloc_msi(struct vtpci_softc *sc) } static int -vtpci_alloc_intr_msix_pervq(struct vtpci_softc *sc) +vtpci_alloc_intr_msix_pervq(struct vtpci_common *cn) { int i, nvectors, error; - if (vtpci_disable_msix != 0 || - sc->vtpci_flags & VTPCI_FLAG_NO_MSIX) + if (vtpci_disable_msix != 0 || cn->vtpci_flags & VTPCI_FLAG_NO_MSIX) return (ENOTSUP); - for (nvectors = 0, i = 0; i < sc->vtpci_nvqs; i++) { - if (sc->vtpci_vqs[i].vtv_no_intr == 0) + for (nvectors = 0, i = 0; i < cn->vtpci_nvqs; i++) { + if (cn->vtpci_vqs[i].vtv_no_intr == 0) nvectors++; } - error = vtpci_alloc_msix(sc, nvectors); + error = vtpci_alloc_msix(cn, nvectors); if (error) return (error); - sc->vtpci_flags |= VTPCI_FLAG_MSIX; + cn->vtpci_flags |= VTPCI_FLAG_MSIX; return (0); } static int -vtpci_alloc_intr_msix_shared(struct vtpci_softc *sc) +vtpci_alloc_intr_msix_shared(struct vtpci_common *cn) { int error; - if (vtpci_disable_msix != 0 || - sc->vtpci_flags & VTPCI_FLAG_NO_MSIX) + if (vtpci_disable_msix != 0 || cn->vtpci_flags & VTPCI_FLAG_NO_MSIX) return (ENOTSUP); - error = vtpci_alloc_msix(sc, 1); + error = vtpci_alloc_msix(cn, 1); if (error) return (error); - sc->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX; + cn->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX; return (0); } static int -vtpci_alloc_intr_msi(struct vtpci_softc *sc) +vtpci_alloc_intr_msi(struct vtpci_common *cn) { int error; /* Only BHyVe supports MSI. */ - if (sc->vtpci_flags & VTPCI_FLAG_NO_MSI) + if (cn->vtpci_flags & VTPCI_FLAG_NO_MSI) return (ENOTSUP); - error = vtpci_alloc_msi(sc); + error = vtpci_alloc_msi(cn); if (error) return (error); - sc->vtpci_flags |= VTPCI_FLAG_MSI; + cn->vtpci_flags |= VTPCI_FLAG_MSI; return (0); } static int -vtpci_alloc_intr_legacy(struct vtpci_softc *sc) +vtpci_alloc_intr_intx(struct vtpci_common *cn) { - sc->vtpci_flags |= VTPCI_FLAG_LEGACY; + cn->vtpci_flags |= VTPCI_FLAG_INTX; return (0); } static int -vtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags, +vtpci_alloc_interrupt(struct vtpci_common *cn, int rid, int flags, struct vtpci_interrupt *intr) { struct resource *irq; - irq = bus_alloc_resource_any(sc->vtpci_dev, SYS_RES_IRQ, &rid, flags); + irq = bus_alloc_resource_any(cn->vtpci_dev, SYS_RES_IRQ, &rid, flags); if (irq == NULL) return (ENXIO); @@ -915,40 +508,136 @@ vtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags, return (0); } +static void +vtpci_free_interrupt(struct vtpci_common *cn, struct vtpci_interrupt *intr) +{ + device_t dev; + + dev = cn->vtpci_dev; + + if (intr->vti_handler != NULL) { + bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler); + intr->vti_handler = NULL; + } + + if (intr->vti_irq != NULL) { + bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid, + intr->vti_irq); + intr->vti_irq = NULL; + intr->vti_rid = -1; + } +} + +static void +vtpci_free_interrupts(struct vtpci_common *cn) +{ + struct vtpci_interrupt *intr; + int i, nvq_intrs; + + vtpci_free_interrupt(cn, &cn->vtpci_device_interrupt); + + if (cn->vtpci_nmsix_resources != 0) { + nvq_intrs = cn->vtpci_nmsix_resources - 1; + cn->vtpci_nmsix_resources = 0; + + if ((intr = cn->vtpci_msix_vq_interrupts) != NULL) { + for (i = 0; i < nvq_intrs; i++, intr++) + vtpci_free_interrupt(cn, intr); + + free(cn->vtpci_msix_vq_interrupts, M_DEVBUF); + cn->vtpci_msix_vq_interrupts = NULL; + } + } + + if (cn->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX)) + pci_release_msi(cn->vtpci_dev); + + cn->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK; +} + +static void +vtpci_free_virtqueues(struct vtpci_common *cn) +{ + struct vtpci_virtqueue *vqx; + int idx; + + for (idx = 0; idx < cn->vtpci_nvqs; idx++) { + vtpci_disable_vq(cn, idx); + + vqx = &cn->vtpci_vqs[idx]; + virtqueue_free(vqx->vtv_vq); + vqx->vtv_vq = NULL; + } + + free(cn->vtpci_vqs, M_DEVBUF); + cn->vtpci_vqs = NULL; + cn->vtpci_nvqs = 0; +} + +void +vtpci_release_child_resources(struct vtpci_common *cn) +{ + + vtpci_free_interrupts(cn); + vtpci_free_virtqueues(cn); +} + +static void +vtpci_cleanup_setup_intr_attempt(struct vtpci_common *cn) +{ + int idx; + + if (cn->vtpci_flags & VTPCI_FLAG_MSIX) { + vtpci_register_cfg_msix(cn, NULL); + + for (idx = 0; idx < cn->vtpci_nvqs; idx++) + vtpci_register_vq_msix(cn, idx, NULL); + } + + vtpci_free_interrupts(cn); +} + static int -vtpci_alloc_intr_resources(struct vtpci_softc *sc) +vtpci_alloc_intr_resources(struct vtpci_common *cn) { struct vtpci_interrupt *intr; int i, rid, flags, nvq_intrs, error; - rid = 0; flags = RF_ACTIVE; - if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) + if (cn->vtpci_flags & VTPCI_FLAG_INTX) { + rid = 0; flags |= RF_SHAREABLE; - else + } else rid = 1; /* - * For legacy and MSI interrupts, this single resource handles all - * interrupts. For MSIX, this resource is used for the configuration - * changed interrupt. + * When using INTX or MSI interrupts, this resource handles all + * interrupts. When using MSIX, this resource handles just the + * configuration changed interrupt. */ - intr = &sc->vtpci_device_interrupt; - error = vtpci_alloc_interrupt(sc, rid, flags, intr); - if (error || sc->vtpci_flags & (VTPCI_FLAG_LEGACY | VTPCI_FLAG_MSI)) + intr = &cn->vtpci_device_interrupt; + + error = vtpci_alloc_interrupt(cn, rid, flags, intr); + if (error || cn->vtpci_flags & (VTPCI_FLAG_INTX | VTPCI_FLAG_MSI)) return (error); - /* Subtract one for the configuration changed interrupt. */ - nvq_intrs = sc->vtpci_nmsix_resources - 1; + /* + * Now allocate the interrupts for the virtqueues. This may be one + * for all the virtqueues, or one for each virtqueue. Subtract one + * below for because of the configuration changed interrupt. + */ + nvq_intrs = cn->vtpci_nmsix_resources - 1; - intr = sc->vtpci_msix_vq_interrupts = malloc(nvq_intrs * + cn->vtpci_msix_vq_interrupts = malloc(nvq_intrs * sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO); - if (sc->vtpci_msix_vq_interrupts == NULL) + if (cn->vtpci_msix_vq_interrupts == NULL) return (ENOMEM); + intr = cn->vtpci_msix_vq_interrupts; + for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) { - error = vtpci_alloc_interrupt(sc, rid, flags, intr); + error = vtpci_alloc_interrupt(cn, rid, flags, intr); if (error) return (error); } @@ -957,34 +646,35 @@ vtpci_alloc_intr_resources(struct vtpci_softc *sc) } static int -vtpci_setup_legacy_interrupt(struct vtpci_softc *sc, enum intr_type type) +vtpci_setup_intx_interrupt(struct vtpci_common *cn, enum intr_type type) { struct vtpci_interrupt *intr; int error; - intr = &sc->vtpci_device_interrupt; - error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, NULL, - vtpci_legacy_intr, sc, &intr->vti_handler); + intr = &cn->vtpci_device_interrupt; + + error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, NULL, + vtpci_intx_intr, cn, &intr->vti_handler); return (error); } static int -vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type) +vtpci_setup_pervq_msix_interrupts(struct vtpci_common *cn, enum intr_type type) { struct vtpci_virtqueue *vqx; struct vtpci_interrupt *intr; int i, error; - intr = sc->vtpci_msix_vq_interrupts; + intr = cn->vtpci_msix_vq_interrupts; - for (i = 0; i < sc->vtpci_nvqs; i++) { - vqx = &sc->vtpci_vqs[i]; + for (i = 0; i < cn->vtpci_nvqs; i++) { + vqx = &cn->vtpci_vqs[i]; if (vqx->vtv_no_intr) continue; - error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, + error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq, &intr->vti_handler); if (error) @@ -997,106 +687,24 @@ vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type) } static int -vtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type) -{ - device_t dev; - struct vtpci_interrupt *intr; - int error; - - dev = sc->vtpci_dev; - intr = &sc->vtpci_device_interrupt; - - error = bus_setup_intr(dev, intr->vti_irq, type, NULL, - vtpci_config_intr, sc, &intr->vti_handler); - if (error) - return (error); - - if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) { - intr = sc->vtpci_msix_vq_interrupts; - error = bus_setup_intr(dev, intr->vti_irq, type, - vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, sc, - &intr->vti_handler); - } else - error = vtpci_setup_pervq_msix_interrupts(sc, type); - - return (error ? error : vtpci_set_host_msix_vectors(sc)); -} - -static int -vtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type) -{ - int error; - - type |= INTR_MPSAFE; - KASSERT(sc->vtpci_flags & VTPCI_FLAG_ITYPE_MASK, - ("%s: no interrupt type selected %#x", __func__, sc->vtpci_flags)); - - error = vtpci_alloc_intr_resources(sc); - if (error) - return (error); - - if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) - error = vtpci_setup_legacy_interrupt(sc, type); - else if (sc->vtpci_flags & VTPCI_FLAG_MSI) - error = vtpci_setup_msi_interrupt(sc, type); - else - error = vtpci_setup_msix_interrupts(sc, type); - - return (error); -} - -static int -vtpci_register_msix_vector(struct vtpci_softc *sc, int offset, - struct vtpci_interrupt *intr) -{ - device_t dev; - uint16_t vector; - - dev = sc->vtpci_dev; - - if (intr != NULL) { - /* Map from guest rid to host vector. */ - vector = intr->vti_rid - 1; - } else - vector = VIRTIO_MSI_NO_VECTOR; - - vtpci_write_header_2(sc, offset, vector); - - /* Read vector to determine if the host had sufficient resources. */ - if (vtpci_read_header_2(sc, offset) != vector) { - device_printf(dev, - "insufficient host resources for MSIX interrupts\n"); - return (ENODEV); - } - - return (0); -} - -static int -vtpci_set_host_msix_vectors(struct vtpci_softc *sc) +vtpci_set_host_msix_vectors(struct vtpci_common *cn) { struct vtpci_interrupt *intr, *tintr; - int idx, offset, error; - - intr = &sc->vtpci_device_interrupt; - offset = VIRTIO_MSI_CONFIG_VECTOR; + int idx, error; - error = vtpci_register_msix_vector(sc, offset, intr); + intr = &cn->vtpci_device_interrupt; + error = vtpci_register_cfg_msix(cn, intr); if (error) return (error); - intr = sc->vtpci_msix_vq_interrupts; - offset = VIRTIO_MSI_QUEUE_VECTOR; - - for (idx = 0; idx < sc->vtpci_nvqs; idx++) { - vtpci_select_virtqueue(sc, idx); - - if (sc->vtpci_vqs[idx].vtv_no_intr) + intr = cn->vtpci_msix_vq_interrupts; + for (idx = 0; idx < cn->vtpci_nvqs; idx++) { + if (cn->vtpci_vqs[idx].vtv_no_intr) tintr = NULL; else tintr = intr; - error = vtpci_register_msix_vector(sc, offset, tintr); + error = vtpci_register_vq_msix(cn, idx, tintr); if (error) break; @@ -1104,8 +712,8 @@ vtpci_set_host_msix_vectors(struct vtpci_softc *sc) * For shared MSIX, all the virtqueues share the first * interrupt. */ - if (!sc->vtpci_vqs[idx].vtv_no_intr && - (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0) + if (!cn->vtpci_vqs[idx].vtv_no_intr && + (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0) intr++; } @@ -1113,164 +721,141 @@ vtpci_set_host_msix_vectors(struct vtpci_softc *sc) } static int -vtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx) +vtpci_setup_msix_interrupts(struct vtpci_common *cn, enum intr_type type) { - struct vtpci_virtqueue *vqx; - struct virtqueue *vq; + struct vtpci_interrupt *intr; int error; - uint16_t size; - - vqx = &sc->vtpci_vqs[idx]; - vq = vqx->vtv_vq; - - KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); - vtpci_select_virtqueue(sc, idx); - size = vtpci_read_header_2(sc, VIRTIO_PCI_QUEUE_NUM); + intr = &cn->vtpci_device_interrupt; - error = virtqueue_reinit(vq, size); + error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, NULL, + vtpci_config_intr, cn, &intr->vti_handler); if (error) return (error); - vtpci_write_header_4(sc, VIRTIO_PCI_QUEUE_PFN, - virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); - - return (0); -} + if (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) { + intr = &cn->vtpci_msix_vq_interrupts[0]; -static void -vtpci_free_interrupt(struct vtpci_softc *sc, struct vtpci_interrupt *intr) -{ - device_t dev; - - dev = sc->vtpci_dev; - - if (intr->vti_handler != NULL) { - bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler); - intr->vti_handler = NULL; - } + error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, + vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, cn, + &intr->vti_handler); + } else + error = vtpci_setup_pervq_msix_interrupts(cn, type); - if (intr->vti_irq != NULL) { - bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid, - intr->vti_irq); - intr->vti_irq = NULL; - intr->vti_rid = -1; - } + return (error ? error : vtpci_set_host_msix_vectors(cn)); } -static void -vtpci_free_interrupts(struct vtpci_softc *sc) +static int +vtpci_setup_intrs(struct vtpci_common *cn, enum intr_type type) { - struct vtpci_interrupt *intr; - int i, nvq_intrs; - - vtpci_free_interrupt(sc, &sc->vtpci_device_interrupt); - - if (sc->vtpci_nmsix_resources != 0) { - nvq_intrs = sc->vtpci_nmsix_resources - 1; - sc->vtpci_nmsix_resources = 0; + int error; - intr = sc->vtpci_msix_vq_interrupts; - if (intr != NULL) { - for (i = 0; i < nvq_intrs; i++, intr++) - vtpci_free_interrupt(sc, intr); + type |= INTR_MPSAFE; + KASSERT(cn->vtpci_flags & VTPCI_FLAG_ITYPE_MASK, + ("%s: no interrupt type selected %#x", __func__, cn->vtpci_flags)); - free(sc->vtpci_msix_vq_interrupts, M_DEVBUF); - sc->vtpci_msix_vq_interrupts = NULL; - } - } + error = vtpci_alloc_intr_resources(cn); + if (error) + return (error); - if (sc->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX)) - pci_release_msi(sc->vtpci_dev); + if (cn->vtpci_flags & VTPCI_FLAG_INTX) + error = vtpci_setup_intx_interrupt(cn, type); + else if (cn->vtpci_flags & VTPCI_FLAG_MSI) + error = vtpci_setup_msi_interrupt(cn, type); + else + error = vtpci_setup_msix_interrupts(cn, type); - sc->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK; + return (error); } -static void -vtpci_free_virtqueues(struct vtpci_softc *sc) +int +vtpci_setup_interrupts(struct vtpci_common *cn, enum intr_type type) { - struct vtpci_virtqueue *vqx; - int idx; - - for (idx = 0; idx < sc->vtpci_nvqs; idx++) { - vqx = &sc->vtpci_vqs[idx]; - - vtpci_select_virtqueue(sc, idx); - vtpci_write_header_4(sc, VIRTIO_PCI_QUEUE_PFN, 0); - - virtqueue_free(vqx->vtv_vq); - vqx->vtv_vq = NULL; - } - - free(sc->vtpci_vqs, M_DEVBUF); - sc->vtpci_vqs = NULL; - sc->vtpci_nvqs = 0; -} + device_t dev; + int attempt, error; -static void -vtpci_release_child_resources(struct vtpci_softc *sc) -{ + dev = cn->vtpci_dev; - vtpci_free_interrupts(sc); - vtpci_free_virtqueues(sc); -} + for (attempt = 0; attempt < 5; attempt++) { + /* + * Start with the most desirable interrupt configuration and + * fallback towards less desirable ones. + */ + switch (attempt) { + case 0: + error = vtpci_alloc_intr_msix_pervq(cn); + break; + case 1: + error = vtpci_alloc_intr_msix_shared(cn); + break; + case 2: + error = vtpci_alloc_intr_msi(cn); + break; + case 3: + error = vtpci_alloc_intr_intx(cn); + break; + default: + device_printf(dev, + "exhausted all interrupt allocation attempts\n"); + return (ENXIO); + } -static void -vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc) -{ - int idx; + if (error == 0 && vtpci_setup_intrs(cn, type) == 0) + break; - if (sc->vtpci_flags & VTPCI_FLAG_MSIX) { - vtpci_write_header_2(sc, VIRTIO_MSI_CONFIG_VECTOR, - VIRTIO_MSI_NO_VECTOR); + vtpci_cleanup_setup_intr_attempt(cn); + } - for (idx = 0; idx < sc->vtpci_nvqs; idx++) { - vtpci_select_virtqueue(sc, idx); - vtpci_write_header_2(sc, VIRTIO_MSI_QUEUE_VECTOR, - VIRTIO_MSI_NO_VECTOR); - } + if (bootverbose) { + if (cn->vtpci_flags & VTPCI_FLAG_INTX) + device_printf(dev, "using legacy interrupt\n"); + else if (cn->vtpci_flags & VTPCI_FLAG_MSI) + device_printf(dev, "using MSI interrupt\n"); + else if (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) + device_printf(dev, "using shared MSIX interrupts\n"); + else + device_printf(dev, "using per VQ MSIX interrupts\n"); } - vtpci_free_interrupts(sc); + return (0); } -static void -vtpci_reset(struct vtpci_softc *sc) +static int +vtpci_reinit_virtqueue(struct vtpci_common *cn, int idx) { + struct vtpci_virtqueue *vqx; + struct virtqueue *vq; + int error; - /* - * Setting the status to RESET sets the host device to - * the original, uninitialized state. - */ - vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET); -} + vqx = &cn->vtpci_vqs[idx]; + vq = vqx->vtv_vq; -static void -vtpci_select_virtqueue(struct vtpci_softc *sc, int idx) -{ + KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); - vtpci_write_header_2(sc, VIRTIO_PCI_QUEUE_SEL, idx); + error = virtqueue_reinit(vq, vtpci_get_vq_size(cn, idx)); + if (error == 0) + vtpci_set_vq(cn, vq); + + return (error); } static void -vtpci_legacy_intr(void *xsc) +vtpci_intx_intr(void *xcn) { - struct vtpci_softc *sc; + struct vtpci_common *cn; struct vtpci_virtqueue *vqx; int i; uint8_t isr; - sc = xsc; - vqx = &sc->vtpci_vqs[0]; - - /* Reading the ISR also clears it. */ - isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR); + cn = xcn; + isr = vtpci_read_isr(cn); if (isr & VIRTIO_PCI_ISR_CONFIG) - vtpci_config_intr(sc); + vtpci_config_intr(cn); if (isr & VIRTIO_PCI_ISR_INTR) { - for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { + vqx = &cn->vtpci_vqs[0]; + for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) { if (vqx->vtv_no_intr == 0) virtqueue_intr(vqx->vtv_vq); } @@ -1278,17 +863,17 @@ vtpci_legacy_intr(void *xsc) } static int -vtpci_vq_shared_intr_filter(void *xsc) +vtpci_vq_shared_intr_filter(void *xcn) { - struct vtpci_softc *sc; + struct vtpci_common *cn; struct vtpci_virtqueue *vqx; int i, rc; + cn = xcn; + vqx = &cn->vtpci_vqs[0]; rc = 0; - sc = xsc; - vqx = &sc->vtpci_vqs[0]; - for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { + for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) { if (vqx->vtv_no_intr == 0) rc |= virtqueue_intr_filter(vqx->vtv_vq); } @@ -1297,16 +882,16 @@ vtpci_vq_shared_intr_filter(void *xsc) } static void -vtpci_vq_shared_intr(void *xsc) +vtpci_vq_shared_intr(void *xcn) { - struct vtpci_softc *sc; + struct vtpci_common *cn; struct vtpci_virtqueue *vqx; int i; - sc = xsc; - vqx = &sc->vtpci_vqs[0]; + cn = xcn; + vqx = &cn->vtpci_vqs[0]; - for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { + for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) { if (vqx->vtv_no_intr == 0) virtqueue_intr(vqx->vtv_vq); } @@ -1334,13 +919,13 @@ vtpci_vq_intr(void *xvq) } static void -vtpci_config_intr(void *xsc) +vtpci_config_intr(void *xcn) { - struct vtpci_softc *sc; + struct vtpci_common *cn; device_t child; - sc = xsc; - child = sc->vtpci_child_dev; + cn = xcn; + child = cn->vtpci_child_dev; if (child != NULL) VIRTIO_CONFIG_CHANGE(child); diff --git a/sys/dev/virtio/pci/virtio_pci.h b/sys/dev/virtio/pci/virtio_pci.h index 8212e36bb733..d65ed853c33e 100644 --- a/sys/dev/virtio/pci/virtio_pci.h +++ b/sys/dev/virtio/pci/virtio_pci.h @@ -1,36 +1,29 @@ /*- - * SPDX-License-Identifier: BSD-3-Clause + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * - * Copyright IBM Corp. 2007 - * - * Authors: - * Anthony Liguori <aliguori@us.ibm.com> - * - * This header is BSD licensed so anyone can use the definitions to implement - * compatible drivers/servers. + * Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org> + * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. + * notice unmodified, this list of conditions, and the following + * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. - * 3. Neither the name of IBM nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ @@ -38,51 +31,101 @@ #ifndef _VIRTIO_PCI_H #define _VIRTIO_PCI_H -/* VirtIO PCI vendor/device ID. */ -#define VIRTIO_PCI_VENDORID 0x1AF4 -#define VIRTIO_PCI_DEVICEID_MIN 0x1000 -#define VIRTIO_PCI_DEVICEID_MAX 0x103F +struct vtpci_interrupt { + struct resource *vti_irq; + int vti_rid; + void *vti_handler; +}; -/* VirtIO ABI version, this must match exactly. */ -#define VIRTIO_PCI_ABI_VERSION 0 +struct vtpci_virtqueue { + struct virtqueue *vtv_vq; + int vtv_no_intr; + int vtv_notify_offset; +}; -/* - * VirtIO Header, located in BAR 0. - */ -#define VIRTIO_PCI_HOST_FEATURES 0 /* host's supported features (32bit, RO)*/ -#define VIRTIO_PCI_GUEST_FEATURES 4 /* guest's supported features (32, RW) */ -#define VIRTIO_PCI_QUEUE_PFN 8 /* physical address of VQ (32, RW) */ -#define VIRTIO_PCI_QUEUE_NUM 12 /* number of ring entries (16, RO) */ -#define VIRTIO_PCI_QUEUE_SEL 14 /* current VQ selection (16, RW) */ -#define VIRTIO_PCI_QUEUE_NOTIFY 16 /* notify host regarding VQ (16, RW) */ -#define VIRTIO_PCI_STATUS 18 /* device status register (8, RW) */ -#define VIRTIO_PCI_ISR 19 /* interrupt status register, reading - * also clears the register (8, RO) */ -/* Only if MSIX is enabled: */ -#define VIRTIO_MSI_CONFIG_VECTOR 20 /* configuration change vector (16, RW) */ -#define VIRTIO_MSI_QUEUE_VECTOR 22 /* vector for selected VQ notifications - (16, RW) */ - -/* The bit of the ISR which indicates a device has an interrupt. */ -#define VIRTIO_PCI_ISR_INTR 0x1 -/* The bit of the ISR which indicates a device configuration change. */ -#define VIRTIO_PCI_ISR_CONFIG 0x2 -/* Vector value used to disable MSI for queue. */ -#define VIRTIO_MSI_NO_VECTOR 0xFFFF - -/* - * The remaining space is defined by each driver as the per-driver - * configuration space. - */ -#define VIRTIO_PCI_CONFIG_OFF(msix_enabled) ((msix_enabled) ? 24 : 20) +struct vtpci_common { + device_t vtpci_dev; + uint64_t vtpci_features; + struct vtpci_virtqueue *vtpci_vqs; + int vtpci_nvqs; -/* - * How many bits to shift physical queue address written to QUEUE_PFN. - * 12 is historical, and due to x86 page size. - */ -#define VIRTIO_PCI_QUEUE_ADDR_SHIFT 12 + uint32_t vtpci_flags; +#define VTPCI_FLAG_NO_MSI 0x0001 +#define VTPCI_FLAG_NO_MSIX 0x0002 +#define VTPCI_FLAG_MODERN 0x0004 +#define VTPCI_FLAG_INTX 0x1000 +#define VTPCI_FLAG_MSI 0x2000 +#define VTPCI_FLAG_MSIX 0x4000 +#define VTPCI_FLAG_SHARED_MSIX 0x8000 +#define VTPCI_FLAG_ITYPE_MASK 0xF000 + + /* The VirtIO PCI "bus" will only ever have one child. */ + device_t vtpci_child_dev; + struct virtio_feature_desc *vtpci_child_feat_desc; + + /* + * Ideally, each virtqueue that the driver provides a callback for will + * receive its own MSIX vector. If there are not sufficient vectors + * available, then attempt to have all the VQs share one vector. For + * MSIX, the configuration changed notifications must be on their own + * vector. + * + * If MSIX is not available, attempt to have the whole device share + * one MSI vector, and then, finally, one intx interrupt. + */ + struct vtpci_interrupt vtpci_device_interrupt; + struct vtpci_interrupt *vtpci_msix_vq_interrupts; + int vtpci_nmsix_resources; +}; + +extern int vtpci_disable_msix; + +static inline device_t +vtpci_child_device(struct vtpci_common *cn) +{ + return (cn->vtpci_child_dev); +} + +static inline bool +vtpci_is_msix_available(struct vtpci_common *cn) +{ + return ((cn->vtpci_flags & VTPCI_FLAG_NO_MSIX) == 0); +} + +static inline bool +vtpci_is_msix_enabled(struct vtpci_common *cn) +{ + return ((cn->vtpci_flags & VTPCI_FLAG_MSIX) != 0); +} + +static inline bool +vtpci_is_modern(struct vtpci_common *cn) +{ + return ((cn->vtpci_flags & VTPCI_FLAG_MODERN) != 0); +} + +static inline int +vtpci_virtqueue_count(struct vtpci_common *cn) +{ + return (cn->vtpci_nvqs); +} + +void vtpci_init(struct vtpci_common *cn, device_t dev, bool modern); +int vtpci_add_child(struct vtpci_common *cn); +int vtpci_delete_child(struct vtpci_common *cn); +void vtpci_child_detached(struct vtpci_common *cn); +int vtpci_reinit(struct vtpci_common *cn); + +uint64_t vtpci_negotiate_features(struct vtpci_common *cn, + uint64_t child_features, uint64_t host_features); +int vtpci_with_feature(struct vtpci_common *cn, uint64_t feature); + +int vtpci_read_ivar(struct vtpci_common *cn, int index, uintptr_t *result); +int vtpci_write_ivar(struct vtpci_common *cn, int index, uintptr_t value); -/* The alignment to use between consumer and producer parts of vring. */ -#define VIRTIO_PCI_VRING_ALIGN 4096 +int vtpci_alloc_virtqueues(struct vtpci_common *cn, int flags, int nvqs, + struct vq_alloc_info *vq_info); +int vtpci_setup_interrupts(struct vtpci_common *cn, enum intr_type type); +void vtpci_release_child_resources(struct vtpci_common *cn); #endif /* _VIRTIO_PCI_H */ diff --git a/sys/dev/virtio/pci/virtio_pci_if.m b/sys/dev/virtio/pci/virtio_pci_if.m new file mode 100644 index 000000000000..077ed6e1e51e --- /dev/null +++ b/sys/dev/virtio/pci/virtio_pci_if.m @@ -0,0 +1,71 @@ +#- +# Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org> +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ + +#include <sys/bus.h> +#include <machine/bus.h> + +INTERFACE virtio_pci; + +HEADER { +struct virtqueue; +struct vtpci_interrupt; +}; + +METHOD uint8_t read_isr { + device_t dev; +}; + +METHOD uint16_t get_vq_size { + device_t dev; + int idx; +}; + +METHOD bus_size_t get_vq_notify_off { + device_t dev; + int idx; +}; + +METHOD void set_vq { + device_t dev; + struct virtqueue *vq; +}; + +METHOD void disable_vq { + device_t dev; + int idx; +}; + +METHOD int register_cfg_msix { + device_t dev; + struct vtpci_interrupt *intr; +}; + +METHOD int register_vq_msix { + device_t dev; + int idx; + struct vtpci_interrupt *intr; +}; diff --git a/sys/dev/virtio/pci/virtio_pci_legacy.c b/sys/dev/virtio/pci/virtio_pci_legacy.c new file mode 100644 index 000000000000..22e369097e2e --- /dev/null +++ b/sys/dev/virtio/pci/virtio_pci_legacy.c @@ -0,0 +1,733 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* Driver for the legacy VirtIO PCI interface. */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/bus.h> +#include <sys/lock.h> +#include <sys/kernel.h> +#include <sys/module.h> +#include <sys/endian.h> + +#include <machine/bus.h> +#include <machine/resource.h> +#include <sys/bus.h> +#include <sys/rman.h> + +#include <dev/pci/pcivar.h> +#include <dev/pci/pcireg.h> + +#include <dev/virtio/virtio.h> +#include <dev/virtio/virtqueue.h> +#include <dev/virtio/pci/virtio_pci.h> +#include <dev/virtio/pci/virtio_pci_legacy_var.h> + +#include "virtio_bus_if.h" +#include "virtio_pci_if.h" +#include "virtio_if.h" + +struct vtpci_legacy_softc { + device_t vtpci_dev; + struct vtpci_common vtpci_common; + struct resource *vtpci_res; + struct resource *vtpci_msix_res; +}; + +static int vtpci_legacy_probe(device_t); +static int vtpci_legacy_attach(device_t); +static int vtpci_legacy_detach(device_t); +static int vtpci_legacy_suspend(device_t); +static int vtpci_legacy_resume(device_t); +static int vtpci_legacy_shutdown(device_t); + +static void vtpci_legacy_driver_added(device_t, driver_t *); +static void vtpci_legacy_child_detached(device_t, device_t); +static int vtpci_legacy_read_ivar(device_t, device_t, int, uintptr_t *); +static int vtpci_legacy_write_ivar(device_t, device_t, int, uintptr_t); + +static uint8_t vtpci_legacy_read_isr(device_t); +static uint16_t vtpci_legacy_get_vq_size(device_t, int); +static bus_size_t vtpci_legacy_get_vq_notify_off(device_t, int); +static void vtpci_legacy_set_vq(device_t, struct virtqueue *); +static void vtpci_legacy_disable_vq(device_t, int); +static int vtpci_legacy_register_cfg_msix(device_t, + struct vtpci_interrupt *); +static int vtpci_legacy_register_vq_msix(device_t, int idx, + struct vtpci_interrupt *); + +static uint64_t vtpci_legacy_negotiate_features(device_t, uint64_t); +static int vtpci_legacy_with_feature(device_t, uint64_t); +static int vtpci_legacy_alloc_virtqueues(device_t, int, int, + struct vq_alloc_info *); +static int vtpci_legacy_setup_interrupts(device_t, enum intr_type); +static void vtpci_legacy_stop(device_t); +static int vtpci_legacy_reinit(device_t, uint64_t); +static void vtpci_legacy_reinit_complete(device_t); +static void vtpci_legacy_notify_vq(device_t, uint16_t, bus_size_t); +static void vtpci_legacy_read_dev_config(device_t, bus_size_t, void *, int); +static void vtpci_legacy_write_dev_config(device_t, bus_size_t, void *, int); + +static int vtpci_legacy_alloc_resources(struct vtpci_legacy_softc *); +static void vtpci_legacy_free_resources(struct vtpci_legacy_softc *); + +static void vtpci_legacy_probe_and_attach_child(struct vtpci_legacy_softc *); + +static uint8_t vtpci_legacy_get_status(struct vtpci_legacy_softc *); +static void vtpci_legacy_set_status(struct vtpci_legacy_softc *, uint8_t); +static void vtpci_legacy_select_virtqueue(struct vtpci_legacy_softc *, int); +static void vtpci_legacy_reset(struct vtpci_legacy_softc *); + +#define VIRTIO_PCI_LEGACY_CONFIG(_sc) \ + VIRTIO_PCI_CONFIG_OFF(vtpci_is_msix_enabled(&(_sc)->vtpci_common)) + +#define vtpci_legacy_read_config_1(sc, o) \ + bus_read_1((sc)->vtpci_res, (o)) +#define vtpci_legacy_write_config_1(sc, o, v) \ + bus_write_1((sc)->vtpci_res, (o), (v)) +/* + * VirtIO specifies that PCI Configuration area is guest endian. However, + * since PCI devices are inherently little-endian, on big-endian systems + * the bus layer transparently converts it to BE. For virtio-legacy, this + * conversion is undesired, so an extra byte swap is required to fix it. + */ +#define vtpci_legacy_read_config_2(sc, o) \ + le16toh(bus_read_2((sc)->vtpci_res, (o))) +#define vtpci_legacy_read_config_4(sc, o) \ + le32toh(bus_read_4((sc)->vtpci_res, (o))) +#define vtpci_legacy_write_config_2(sc, o, v) \ + bus_write_2((sc)->vtpci_res, (o), (htole16(v))) +#define vtpci_legacy_write_config_4(sc, o, v) \ + bus_write_4((sc)->vtpci_res, (o), (htole32(v))) +/* PCI Header LE. On BE systems the bus layer takes care of byte swapping. */ +#define vtpci_legacy_read_header_2(sc, o) \ + bus_read_2((sc)->vtpci_res, (o)) +#define vtpci_legacy_read_header_4(sc, o) \ + bus_read_4((sc)->vtpci_res, (o)) +#define vtpci_legacy_write_header_2(sc, o, v) \ + bus_write_2((sc)->vtpci_res, (o), (v)) +#define vtpci_legacy_write_header_4(sc, o, v) \ + bus_write_4((sc)->vtpci_res, (o), (v)) + +static device_method_t vtpci_legacy_methods[] = { + /* Device interface. */ + DEVMETHOD(device_probe, vtpci_legacy_probe), + DEVMETHOD(device_attach, vtpci_legacy_attach), + DEVMETHOD(device_detach, vtpci_legacy_detach), + DEVMETHOD(device_suspend, vtpci_legacy_suspend), + DEVMETHOD(device_resume, vtpci_legacy_resume), + DEVMETHOD(device_shutdown, vtpci_legacy_shutdown), + + /* Bus interface. */ + DEVMETHOD(bus_driver_added, vtpci_legacy_driver_added), + DEVMETHOD(bus_child_detached, vtpci_legacy_child_detached), + DEVMETHOD(bus_child_pnpinfo_str, virtio_child_pnpinfo_str), + DEVMETHOD(bus_read_ivar, vtpci_legacy_read_ivar), + DEVMETHOD(bus_write_ivar, vtpci_legacy_write_ivar), + + /* VirtIO PCI interface. */ + DEVMETHOD(virtio_pci_read_isr, vtpci_legacy_read_isr), + DEVMETHOD(virtio_pci_get_vq_size, vtpci_legacy_get_vq_size), + DEVMETHOD(virtio_pci_get_vq_notify_off, vtpci_legacy_get_vq_notify_off), + DEVMETHOD(virtio_pci_set_vq, vtpci_legacy_set_vq), + DEVMETHOD(virtio_pci_disable_vq, vtpci_legacy_disable_vq), + DEVMETHOD(virtio_pci_register_cfg_msix, vtpci_legacy_register_cfg_msix), + DEVMETHOD(virtio_pci_register_vq_msix, vtpci_legacy_register_vq_msix), + + /* VirtIO bus interface. */ + DEVMETHOD(virtio_bus_negotiate_features, vtpci_legacy_negotiate_features), + DEVMETHOD(virtio_bus_with_feature, vtpci_legacy_with_feature), + DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_legacy_alloc_virtqueues), + DEVMETHOD(virtio_bus_setup_intr, vtpci_legacy_setup_interrupts), + DEVMETHOD(virtio_bus_stop, vtpci_legacy_stop), + DEVMETHOD(virtio_bus_reinit, vtpci_legacy_reinit), + DEVMETHOD(virtio_bus_reinit_complete, vtpci_legacy_reinit_complete), + DEVMETHOD(virtio_bus_notify_vq, vtpci_legacy_notify_vq), + DEVMETHOD(virtio_bus_read_device_config, vtpci_legacy_read_dev_config), + DEVMETHOD(virtio_bus_write_device_config, vtpci_legacy_write_dev_config), + + DEVMETHOD_END +}; + +static driver_t vtpci_legacy_driver = { + .name = "virtio_pci", + .methods = vtpci_legacy_methods, + .size = sizeof(struct vtpci_legacy_softc) +}; + +devclass_t vtpci_legacy_devclass; + +DRIVER_MODULE(virtio_pci_legacy, pci, vtpci_legacy_driver, + vtpci_legacy_devclass, 0, 0); + +static int +vtpci_legacy_probe(device_t dev) +{ + char desc[64]; + const char *name; + + if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) + return (ENXIO); + + if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || + pci_get_device(dev) > VIRTIO_PCI_DEVICEID_LEGACY_MAX) + return (ENXIO); + + if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION) + return (ENXIO); + + name = virtio_device_name(pci_get_subdevice(dev)); + if (name == NULL) + name = "Unknown"; + + snprintf(desc, sizeof(desc), "VirtIO PCI (legacy) %s adapter", name); + device_set_desc_copy(dev, desc); + + /* Prefer transitional modern VirtIO PCI. */ + return (BUS_PROBE_LOW_PRIORITY); +} + +static int +vtpci_legacy_attach(device_t dev) +{ + struct vtpci_legacy_softc *sc; + int error; + + sc = device_get_softc(dev); + sc->vtpci_dev = dev; + vtpci_init(&sc->vtpci_common, dev, false); + + error = vtpci_legacy_alloc_resources(sc); + if (error) { + device_printf(dev, "cannot map I/O space\n"); + return (error); + } + + vtpci_legacy_reset(sc); + + /* Tell the host we've noticed this device. */ + vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); + + error = vtpci_add_child(&sc->vtpci_common); + if (error) + goto fail; + + vtpci_legacy_probe_and_attach_child(sc); + + return (0); + +fail: + vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); + vtpci_legacy_detach(dev); + + return (error); +} + +static int +vtpci_legacy_detach(device_t dev) +{ + struct vtpci_legacy_softc *sc; + int error; + + sc = device_get_softc(dev); + + error = vtpci_delete_child(&sc->vtpci_common); + if (error) + return (error); + + vtpci_legacy_reset(sc); + vtpci_legacy_free_resources(sc); + + return (0); +} + +static int +vtpci_legacy_suspend(device_t dev) +{ + return (bus_generic_suspend(dev)); +} + +static int +vtpci_legacy_resume(device_t dev) +{ + return (bus_generic_resume(dev)); +} + +static int +vtpci_legacy_shutdown(device_t dev) +{ + (void) bus_generic_shutdown(dev); + /* Forcibly stop the host device. */ + vtpci_legacy_stop(dev); + + return (0); +} + +static void +vtpci_legacy_driver_added(device_t dev, driver_t *driver) +{ + vtpci_legacy_probe_and_attach_child(device_get_softc(dev)); +} + +static void +vtpci_legacy_child_detached(device_t dev, device_t child) +{ + struct vtpci_legacy_softc *sc; + + sc = device_get_softc(dev); + + vtpci_legacy_reset(sc); + vtpci_child_detached(&sc->vtpci_common); + + /* After the reset, retell the host we've noticed this device. */ + vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); +} + +static int +vtpci_legacy_read_ivar(device_t dev, device_t child, int index, + uintptr_t *result) +{ + struct vtpci_legacy_softc *sc; + struct vtpci_common *cn; + + sc = device_get_softc(dev); + cn = &sc->vtpci_common; + + if (vtpci_child_device(cn) != child) + return (ENOENT); + + switch (index) { + case VIRTIO_IVAR_DEVTYPE: + *result = pci_get_subdevice(dev); + break; + default: + return (vtpci_read_ivar(cn, index, result)); + } + + return (0); +} + +static int +vtpci_legacy_write_ivar(device_t dev, device_t child, int index, uintptr_t value) +{ + struct vtpci_legacy_softc *sc; + struct vtpci_common *cn; + + sc = device_get_softc(dev); + cn = &sc->vtpci_common; + + if (vtpci_child_device(cn) != child) + return (ENOENT); + + switch (index) { + default: + return (vtpci_write_ivar(cn, index, value)); + } + + return (0); +} + +static uint64_t +vtpci_legacy_negotiate_features(device_t dev, uint64_t child_features) +{ + struct vtpci_legacy_softc *sc; + uint64_t host_features, features; + + sc = device_get_softc(dev); + host_features = vtpci_legacy_read_header_4(sc, VIRTIO_PCI_HOST_FEATURES); + + features = vtpci_negotiate_features(&sc->vtpci_common, + child_features, host_features); + vtpci_legacy_write_header_4(sc, VIRTIO_PCI_GUEST_FEATURES, features); + + return (features); +} + +static int +vtpci_legacy_with_feature(device_t dev, uint64_t feature) +{ + struct vtpci_legacy_softc *sc; + + sc = device_get_softc(dev); + + return (vtpci_with_feature(&sc->vtpci_common, feature)); +} + +static int +vtpci_legacy_alloc_virtqueues(device_t dev, int flags, int nvqs, + struct vq_alloc_info *vq_info) +{ + struct vtpci_legacy_softc *sc; + struct vtpci_common *cn; + + sc = device_get_softc(dev); + cn = &sc->vtpci_common; + + return (vtpci_alloc_virtqueues(cn, flags, nvqs, vq_info)); +} + +static int +vtpci_legacy_setup_interrupts(device_t dev, enum intr_type type) +{ + struct vtpci_legacy_softc *sc; + + sc = device_get_softc(dev); + + return (vtpci_setup_interrupts(&sc->vtpci_common, type)); +} + +static void +vtpci_legacy_stop(device_t dev) +{ + vtpci_legacy_reset(device_get_softc(dev)); +} + +static int +vtpci_legacy_reinit(device_t dev, uint64_t features) +{ + struct vtpci_legacy_softc *sc; + struct vtpci_common *cn; + int error; + + sc = device_get_softc(dev); + cn = &sc->vtpci_common; + + /* + * Redrive the device initialization. This is a bit of an abuse of + * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to + * play nice. + * + * We do not allow the host device to change from what was originally + * negotiated beyond what the guest driver changed. MSIX state should + * not change, number of virtqueues and their size remain the same, etc. + * This will need to be rethought when we want to support migration. + */ + + if (vtpci_legacy_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET) + vtpci_legacy_stop(dev); + + /* + * Quickly drive the status through ACK and DRIVER. The device does + * not become usable again until DRIVER_OK in reinit complete. + */ + vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); + vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); + + vtpci_legacy_negotiate_features(dev, features); + + error = vtpci_reinit(cn); + if (error) + return (error); + + return (0); +} + +static void +vtpci_legacy_reinit_complete(device_t dev) +{ + struct vtpci_legacy_softc *sc; + + sc = device_get_softc(dev); + + vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); +} + +static void +vtpci_legacy_notify_vq(device_t dev, uint16_t queue, bus_size_t offset) +{ + struct vtpci_legacy_softc *sc; + + sc = device_get_softc(dev); + MPASS(offset == VIRTIO_PCI_QUEUE_NOTIFY); + + vtpci_legacy_write_header_2(sc, offset, queue); +} + +static uint8_t +vtpci_legacy_get_status(struct vtpci_legacy_softc *sc) +{ + return (vtpci_legacy_read_config_1(sc, VIRTIO_PCI_STATUS)); +} + +static void +vtpci_legacy_set_status(struct vtpci_legacy_softc *sc, uint8_t status) +{ + if (status != VIRTIO_CONFIG_STATUS_RESET) + status |= vtpci_legacy_get_status(sc); + + vtpci_legacy_write_config_1(sc, VIRTIO_PCI_STATUS, status); +} + +static void +vtpci_legacy_read_dev_config(device_t dev, bus_size_t offset, + void *dst, int length) +{ + struct vtpci_legacy_softc *sc; + bus_size_t off; + uint8_t *d; + int size; + + sc = device_get_softc(dev); + off = VIRTIO_PCI_LEGACY_CONFIG(sc) + offset; + + for (d = dst; length > 0; d += size, off += size, length -= size) { + if (length >= 4) { + size = 4; + *(uint32_t *)d = vtpci_legacy_read_config_4(sc, off); + } else if (length >= 2) { + size = 2; + *(uint16_t *)d = vtpci_legacy_read_config_2(sc, off); + } else { + size = 1; + *d = vtpci_legacy_read_config_1(sc, off); + } + } +} + +static void +vtpci_legacy_write_dev_config(device_t dev, bus_size_t offset, + void *src, int length) +{ + struct vtpci_legacy_softc *sc; + bus_size_t off; + uint8_t *s; + int size; + + sc = device_get_softc(dev); + off = VIRTIO_PCI_LEGACY_CONFIG(sc) + offset; + + for (s = src; length > 0; s += size, off += size, length -= size) { + if (length >= 4) { + size = 4; + vtpci_legacy_write_config_4(sc, off, *(uint32_t *)s); + } else if (length >= 2) { + size = 2; + vtpci_legacy_write_config_2(sc, off, *(uint16_t *)s); + } else { + size = 1; + vtpci_legacy_write_config_1(sc, off, *s); + } + } +} + +static int +vtpci_legacy_alloc_resources(struct vtpci_legacy_softc *sc) +{ + device_t dev; + int rid; + + dev = sc->vtpci_dev; + + rid = PCIR_BAR(0); + if ((sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, + &rid, RF_ACTIVE)) == NULL) + return (ENXIO); + + if (vtpci_is_msix_available(&sc->vtpci_common)) { + rid = PCIR_BAR(1); + if ((sc->vtpci_msix_res = bus_alloc_resource_any(dev, + SYS_RES_MEMORY, &rid, RF_ACTIVE)) == NULL) + return (ENXIO); + } + + return (0); +} + +static void +vtpci_legacy_free_resources(struct vtpci_legacy_softc *sc) +{ + device_t dev; + + dev = sc->vtpci_dev; + + if (sc->vtpci_msix_res != NULL) { + bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1), + sc->vtpci_msix_res); + sc->vtpci_msix_res = NULL; + } + + if (sc->vtpci_res != NULL) { + bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0), + sc->vtpci_res); + sc->vtpci_res = NULL; + } +} + +static void +vtpci_legacy_probe_and_attach_child(struct vtpci_legacy_softc *sc) +{ + device_t dev, child; + + dev = sc->vtpci_dev; + child = vtpci_child_device(&sc->vtpci_common); + + if (child == NULL || device_get_state(child) != DS_NOTPRESENT) + return; + + if (device_probe(child) != 0) + return; + + vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); + + if (device_attach(child) != 0) { + vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); + /* Reset status for future attempt. */ + vtpci_legacy_child_detached(dev, child); + } else { + vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); + VIRTIO_ATTACH_COMPLETED(child); + } +} + +static int +vtpci_legacy_register_msix(struct vtpci_legacy_softc *sc, int offset, + struct vtpci_interrupt *intr) +{ + device_t dev; + uint16_t vector; + + dev = sc->vtpci_dev; + + if (intr != NULL) { + /* Map from guest rid to host vector. */ + vector = intr->vti_rid - 1; + } else + vector = VIRTIO_MSI_NO_VECTOR; + + vtpci_legacy_write_header_2(sc, offset, vector); + return (vtpci_legacy_read_header_2(sc, offset) == vector ? 0 : ENODEV); +} + +static int +vtpci_legacy_register_cfg_msix(device_t dev, struct vtpci_interrupt *intr) +{ + struct vtpci_legacy_softc *sc; + int error; + + sc = device_get_softc(dev); + + error = vtpci_legacy_register_msix(sc, VIRTIO_MSI_CONFIG_VECTOR, intr); + if (error) { + device_printf(dev, + "unable to register config MSIX interrupt\n"); + return (error); + } + + return (0); +} + +static int +vtpci_legacy_register_vq_msix(device_t dev, int idx, + struct vtpci_interrupt *intr) +{ + struct vtpci_legacy_softc *sc; + int error; + + sc = device_get_softc(dev); + + vtpci_legacy_select_virtqueue(sc, idx); + error = vtpci_legacy_register_msix(sc, VIRTIO_MSI_QUEUE_VECTOR, intr); + if (error) { + device_printf(dev, + "unable to register virtqueue MSIX interrupt\n"); + return (error); + } + + return (0); +} + +static void +vtpci_legacy_reset(struct vtpci_legacy_softc *sc) +{ + /* + * Setting the status to RESET sets the host device to the + * original, uninitialized state. + */ + vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_RESET); + (void) vtpci_legacy_get_status(sc); +} + +static void +vtpci_legacy_select_virtqueue(struct vtpci_legacy_softc *sc, int idx) +{ + vtpci_legacy_write_header_2(sc, VIRTIO_PCI_QUEUE_SEL, idx); +} + +static uint8_t +vtpci_legacy_read_isr(device_t dev) +{ + struct vtpci_legacy_softc *sc; + + sc = device_get_softc(dev); + + return (vtpci_legacy_read_config_1(sc, VIRTIO_PCI_ISR)); +} + +static uint16_t +vtpci_legacy_get_vq_size(device_t dev, int idx) +{ + struct vtpci_legacy_softc *sc; + + sc = device_get_softc(dev); + + vtpci_legacy_select_virtqueue(sc, idx); + return (vtpci_legacy_read_header_2(sc, VIRTIO_PCI_QUEUE_NUM)); +} + +static bus_size_t +vtpci_legacy_get_vq_notify_off(device_t dev, int idx) +{ + return (VIRTIO_PCI_QUEUE_NOTIFY); +} + +static void +vtpci_legacy_set_vq(device_t dev, struct virtqueue *vq) +{ + struct vtpci_legacy_softc *sc; + + sc = device_get_softc(dev); + + vtpci_legacy_select_virtqueue(sc, virtqueue_index(vq)); + vtpci_legacy_write_header_4(sc, VIRTIO_PCI_QUEUE_PFN, + virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); +} + +static void +vtpci_legacy_disable_vq(device_t dev, int idx) +{ + struct vtpci_legacy_softc *sc; + + sc = device_get_softc(dev); + + vtpci_legacy_select_virtqueue(sc, idx); + vtpci_legacy_write_header_4(sc, VIRTIO_PCI_QUEUE_PFN, 0); +} diff --git a/sys/dev/virtio/pci/virtio_pci_legacy_var.h b/sys/dev/virtio/pci/virtio_pci_legacy_var.h new file mode 100644 index 000000000000..615570efb13a --- /dev/null +++ b/sys/dev/virtio/pci/virtio_pci_legacy_var.h @@ -0,0 +1,78 @@ +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright IBM Corp. 2007 + * + * Authors: + * Anthony Liguori <aliguori@us.ibm.com> + * + * This header is BSD licensed so anyone can use the definitions to implement + * compatible drivers/servers. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of IBM nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _VIRTIO_PCI_LEGACY_VAR_H +#define _VIRTIO_PCI_LEGACY_VAR_H + +#include <dev/virtio/pci/virtio_pci_var.h> + +/* VirtIO ABI version, this must match exactly. */ +#define VIRTIO_PCI_ABI_VERSION 0 + +/* + * VirtIO Header, located in BAR 0. + */ +#define VIRTIO_PCI_HOST_FEATURES 0 /* host's supported features (32bit, RO)*/ +#define VIRTIO_PCI_GUEST_FEATURES 4 /* guest's supported features (32, RW) */ +#define VIRTIO_PCI_QUEUE_PFN 8 /* physical address of VQ (32, RW) */ +#define VIRTIO_PCI_QUEUE_NUM 12 /* number of ring entries (16, RO) */ +#define VIRTIO_PCI_QUEUE_SEL 14 /* current VQ selection (16, RW) */ +#define VIRTIO_PCI_QUEUE_NOTIFY 16 /* notify host regarding VQ (16, RW) */ +#define VIRTIO_PCI_STATUS 18 /* device status register (8, RW) */ +#define VIRTIO_PCI_ISR 19 /* interrupt status register, reading + * also clears the register (8, RO) */ +/* Only if MSIX is enabled: */ +#define VIRTIO_MSI_CONFIG_VECTOR 20 /* configuration change vector (16, RW) */ +#define VIRTIO_MSI_QUEUE_VECTOR 22 /* vector for selected VQ notifications + (16, RW) */ + +/* + * The remaining space is defined by each driver as the per-driver + * configuration space. + */ +#define VIRTIO_PCI_CONFIG_OFF(msix_enabled) ((msix_enabled) ? 24 : 20) + +/* + * How many bits to shift physical queue address written to QUEUE_PFN. + * 12 is historical, and due to x86 page size. + */ +#define VIRTIO_PCI_QUEUE_ADDR_SHIFT 12 + +/* The alignment to use between consumer and producer parts of vring. */ +#define VIRTIO_PCI_VRING_ALIGN 4096 + +#endif /* _VIRTIO_PCI_LEGACY_VAR_H */ diff --git a/sys/dev/virtio/pci/virtio_pci_modern.c b/sys/dev/virtio/pci/virtio_pci_modern.c new file mode 100644 index 000000000000..09ac0a1232e7 --- /dev/null +++ b/sys/dev/virtio/pci/virtio_pci_modern.c @@ -0,0 +1,1448 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* Driver for the modern VirtIO PCI interface. */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/bus.h> +#include <sys/lock.h> +#include <sys/kernel.h> +#include <sys/module.h> + +#include <machine/bus.h> +#include <machine/cpu.h> +#include <machine/resource.h> +#include <sys/bus.h> +#include <sys/rman.h> + +#include <dev/pci/pcivar.h> +#include <dev/pci/pcireg.h> + +#include <dev/virtio/virtio.h> +#include <dev/virtio/virtqueue.h> +#include <dev/virtio/pci/virtio_pci.h> +#include <dev/virtio/pci/virtio_pci_modern_var.h> + +#include "virtio_bus_if.h" +#include "virtio_pci_if.h" +#include "virtio_if.h" + +struct vtpci_modern_resource_map { + struct resource_map vtrm_map; + int vtrm_cap_offset; + int vtrm_bar; + int vtrm_offset; + int vtrm_length; + int vtrm_type; /* SYS_RES_{MEMORY, IOPORT} */ +}; + +struct vtpci_modern_bar_resource { + struct resource *vtbr_res; + int vtbr_type; +}; + +struct vtpci_modern_softc { + device_t vtpci_dev; + struct vtpci_common vtpci_common; + uint32_t vtpci_notify_offset_multiplier; + uint16_t vtpci_devid; + int vtpci_msix_bar; + struct resource *vtpci_msix_res; + + struct vtpci_modern_resource_map vtpci_common_res_map; + struct vtpci_modern_resource_map vtpci_notify_res_map; + struct vtpci_modern_resource_map vtpci_isr_res_map; + struct vtpci_modern_resource_map vtpci_device_res_map; + +#define VTPCI_MODERN_MAX_BARS 6 + struct vtpci_modern_bar_resource vtpci_bar_res[VTPCI_MODERN_MAX_BARS]; +}; + +static int vtpci_modern_probe(device_t); +static int vtpci_modern_attach(device_t); +static int vtpci_modern_detach(device_t); +static int vtpci_modern_suspend(device_t); +static int vtpci_modern_resume(device_t); +static int vtpci_modern_shutdown(device_t); + +static void vtpci_modern_driver_added(device_t, driver_t *); +static void vtpci_modern_child_detached(device_t, device_t); +static int vtpci_modern_read_ivar(device_t, device_t, int, uintptr_t *); +static int vtpci_modern_write_ivar(device_t, device_t, int, uintptr_t); + +static uint8_t vtpci_modern_read_isr(device_t); +static uint16_t vtpci_modern_get_vq_size(device_t, int); +static bus_size_t vtpci_modern_get_vq_notify_off(device_t, int); +static void vtpci_modern_set_vq(device_t, struct virtqueue *); +static void vtpci_modern_disable_vq(device_t, int); +static int vtpci_modern_register_msix(struct vtpci_modern_softc *, int, + struct vtpci_interrupt *); +static int vtpci_modern_register_cfg_msix(device_t, + struct vtpci_interrupt *); +static int vtpci_modern_register_vq_msix(device_t, int idx, + struct vtpci_interrupt *); + +static uint64_t vtpci_modern_negotiate_features(device_t, uint64_t); +static int vtpci_modern_finalize_features(device_t); +static int vtpci_modern_with_feature(device_t, uint64_t); +static int vtpci_modern_alloc_virtqueues(device_t, int, int, + struct vq_alloc_info *); +static int vtpci_modern_setup_interrupts(device_t, enum intr_type); +static void vtpci_modern_stop(device_t); +static int vtpci_modern_reinit(device_t, uint64_t); +static void vtpci_modern_reinit_complete(device_t); +static void vtpci_modern_notify_vq(device_t, uint16_t, bus_size_t); +static int vtpci_modern_config_generation(device_t); +static void vtpci_modern_read_dev_config(device_t, bus_size_t, void *, int); +static void vtpci_modern_write_dev_config(device_t, bus_size_t, void *, int); + +static int vtpci_modern_probe_configs(device_t); +static int vtpci_modern_find_cap(device_t, uint8_t, int *); +static int vtpci_modern_map_configs(struct vtpci_modern_softc *); +static void vtpci_modern_unmap_configs(struct vtpci_modern_softc *); +static int vtpci_modern_find_cap_resource(struct vtpci_modern_softc *, + uint8_t, int, int, struct vtpci_modern_resource_map *); +static int vtpci_modern_bar_type(struct vtpci_modern_softc *, int); +static struct resource *vtpci_modern_get_bar_resource( + struct vtpci_modern_softc *, int, int); +static struct resource *vtpci_modern_alloc_bar_resource( + struct vtpci_modern_softc *, int, int); +static void vtpci_modern_free_bar_resources(struct vtpci_modern_softc *); +static int vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *, + struct vtpci_modern_resource_map *); +static void vtpci_modern_free_resource_map(struct vtpci_modern_softc *, + struct vtpci_modern_resource_map *); +static void vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *); +static void vtpci_modern_free_msix_resource(struct vtpci_modern_softc *); + +static void vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *); + +static uint64_t vtpci_modern_read_features(struct vtpci_modern_softc *); +static void vtpci_modern_write_features(struct vtpci_modern_softc *, + uint64_t); +static void vtpci_modern_select_virtqueue(struct vtpci_modern_softc *, int); +static uint8_t vtpci_modern_get_status(struct vtpci_modern_softc *); +static void vtpci_modern_set_status(struct vtpci_modern_softc *, uint8_t); +static void vtpci_modern_reset(struct vtpci_modern_softc *); +static void vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *); + +static uint8_t vtpci_modern_read_common_1(struct vtpci_modern_softc *, + bus_size_t); +static uint16_t vtpci_modern_read_common_2(struct vtpci_modern_softc *, + bus_size_t); +static uint32_t vtpci_modern_read_common_4(struct vtpci_modern_softc *, + bus_size_t); +static void vtpci_modern_write_common_1(struct vtpci_modern_softc *, + bus_size_t, uint8_t); +static void vtpci_modern_write_common_2(struct vtpci_modern_softc *, + bus_size_t, uint16_t); +static void vtpci_modern_write_common_4(struct vtpci_modern_softc *, + bus_size_t, uint32_t); +static void vtpci_modern_write_common_8(struct vtpci_modern_softc *, + bus_size_t, uint64_t); +static void vtpci_modern_write_notify_2(struct vtpci_modern_softc *, + bus_size_t, uint16_t); +static uint8_t vtpci_modern_read_isr_1(struct vtpci_modern_softc *, + bus_size_t); +static uint8_t vtpci_modern_read_device_1(struct vtpci_modern_softc *, + bus_size_t); +static uint16_t vtpci_modern_read_device_2(struct vtpci_modern_softc *, + bus_size_t); +static uint32_t vtpci_modern_read_device_4(struct vtpci_modern_softc *, + bus_size_t); +static uint64_t vtpci_modern_read_device_8(struct vtpci_modern_softc *, + bus_size_t); +static void vtpci_modern_write_device_1(struct vtpci_modern_softc *, + bus_size_t, uint8_t); +static void vtpci_modern_write_device_2(struct vtpci_modern_softc *, + bus_size_t, uint16_t); +static void vtpci_modern_write_device_4(struct vtpci_modern_softc *, + bus_size_t, uint32_t); +static void vtpci_modern_write_device_8(struct vtpci_modern_softc *, + bus_size_t, uint64_t); + +/* Tunables. */ +static int vtpci_modern_transitional = 0; +TUNABLE_INT("hw.virtio.pci.transitional", &vtpci_modern_transitional); + +static device_method_t vtpci_modern_methods[] = { + /* Device interface. */ + DEVMETHOD(device_probe, vtpci_modern_probe), + DEVMETHOD(device_attach, vtpci_modern_attach), + DEVMETHOD(device_detach, vtpci_modern_detach), + DEVMETHOD(device_suspend, vtpci_modern_suspend), + DEVMETHOD(device_resume, vtpci_modern_resume), + DEVMETHOD(device_shutdown, vtpci_modern_shutdown), + + /* Bus interface. */ + DEVMETHOD(bus_driver_added, vtpci_modern_driver_added), + DEVMETHOD(bus_child_detached, vtpci_modern_child_detached), + DEVMETHOD(bus_child_pnpinfo_str, virtio_child_pnpinfo_str), + DEVMETHOD(bus_read_ivar, vtpci_modern_read_ivar), + DEVMETHOD(bus_write_ivar, vtpci_modern_write_ivar), + + /* VirtIO PCI interface. */ + DEVMETHOD(virtio_pci_read_isr, vtpci_modern_read_isr), + DEVMETHOD(virtio_pci_get_vq_size, vtpci_modern_get_vq_size), + DEVMETHOD(virtio_pci_get_vq_notify_off, vtpci_modern_get_vq_notify_off), + DEVMETHOD(virtio_pci_set_vq, vtpci_modern_set_vq), + DEVMETHOD(virtio_pci_disable_vq, vtpci_modern_disable_vq), + DEVMETHOD(virtio_pci_register_cfg_msix, vtpci_modern_register_cfg_msix), + DEVMETHOD(virtio_pci_register_vq_msix, vtpci_modern_register_vq_msix), + + /* VirtIO bus interface. */ + DEVMETHOD(virtio_bus_negotiate_features, vtpci_modern_negotiate_features), + DEVMETHOD(virtio_bus_finalize_features, vtpci_modern_finalize_features), + DEVMETHOD(virtio_bus_with_feature, vtpci_modern_with_feature), + DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_modern_alloc_virtqueues), + DEVMETHOD(virtio_bus_setup_intr, vtpci_modern_setup_interrupts), + DEVMETHOD(virtio_bus_stop, vtpci_modern_stop), + DEVMETHOD(virtio_bus_reinit, vtpci_modern_reinit), + DEVMETHOD(virtio_bus_reinit_complete, vtpci_modern_reinit_complete), + DEVMETHOD(virtio_bus_notify_vq, vtpci_modern_notify_vq), + DEVMETHOD(virtio_bus_config_generation, vtpci_modern_config_generation), + DEVMETHOD(virtio_bus_read_device_config, vtpci_modern_read_dev_config), + DEVMETHOD(virtio_bus_write_device_config, vtpci_modern_write_dev_config), + + DEVMETHOD_END +}; + +static driver_t vtpci_modern_driver = { + .name = "virtio_pci", + .methods = vtpci_modern_methods, + .size = sizeof(struct vtpci_modern_softc) +}; + +devclass_t vtpci_modern_devclass; + +DRIVER_MODULE(virtio_pci_modern, pci, vtpci_modern_driver, + vtpci_modern_devclass, 0, 0); + +static int +vtpci_modern_probe(device_t dev) +{ + char desc[64]; + const char *name; + uint16_t devid; + + if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) + return (ENXIO); + + if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || + pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MODERN_MAX) + return (ENXIO); + + if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) { + if (!vtpci_modern_transitional) + return (ENXIO); + devid = pci_get_subdevice(dev); + } else + devid = pci_get_device(dev) - VIRTIO_PCI_DEVICEID_MODERN_MIN; + + if (vtpci_modern_probe_configs(dev) != 0) + return (ENXIO); + + name = virtio_device_name(devid); + if (name == NULL) + name = "Unknown"; + + snprintf(desc, sizeof(desc), "VirtIO PCI (modern) %s adapter", name); + device_set_desc_copy(dev, desc); + + return (BUS_PROBE_DEFAULT); +} + +static int +vtpci_modern_attach(device_t dev) +{ + struct vtpci_modern_softc *sc; + int error; + + sc = device_get_softc(dev); + sc->vtpci_dev = dev; + vtpci_init(&sc->vtpci_common, dev, true); + + if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) + sc->vtpci_devid = pci_get_subdevice(dev); + else + sc->vtpci_devid = pci_get_device(dev) - + VIRTIO_PCI_DEVICEID_MODERN_MIN; + + error = vtpci_modern_map_configs(sc); + if (error) { + device_printf(dev, "cannot map configs\n"); + vtpci_modern_unmap_configs(sc); + return (error); + } + + vtpci_modern_reset(sc); + + /* Tell the host we've noticed this device. */ + vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); + + error = vtpci_add_child(&sc->vtpci_common); + if (error) + goto fail; + + vtpci_modern_probe_and_attach_child(sc); + + return (0); + +fail: + vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); + vtpci_modern_detach(dev); + + return (error); +} + +static int +vtpci_modern_detach(device_t dev) +{ + struct vtpci_modern_softc *sc; + int error; + + sc = device_get_softc(dev); + + error = vtpci_delete_child(&sc->vtpci_common); + if (error) + return (error); + + vtpci_modern_reset(sc); + vtpci_modern_unmap_configs(sc); + + return (0); +} + +static int +vtpci_modern_suspend(device_t dev) +{ + return (bus_generic_suspend(dev)); +} + +static int +vtpci_modern_resume(device_t dev) +{ + return (bus_generic_resume(dev)); +} + +static int +vtpci_modern_shutdown(device_t dev) +{ + (void) bus_generic_shutdown(dev); + /* Forcibly stop the host device. */ + vtpci_modern_stop(dev); + + return (0); +} + +static void +vtpci_modern_driver_added(device_t dev, driver_t *driver) +{ + vtpci_modern_probe_and_attach_child(device_get_softc(dev)); +} + +static void +vtpci_modern_child_detached(device_t dev, device_t child) +{ + struct vtpci_modern_softc *sc; + + sc = device_get_softc(dev); + + vtpci_modern_reset(sc); + vtpci_child_detached(&sc->vtpci_common); + + /* After the reset, retell the host we've noticed this device. */ + vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); +} + +static int +vtpci_modern_read_ivar(device_t dev, device_t child, int index, + uintptr_t *result) +{ + struct vtpci_modern_softc *sc; + struct vtpci_common *cn; + + sc = device_get_softc(dev); + cn = &sc->vtpci_common; + + if (vtpci_child_device(cn) != child) + return (ENOENT); + + switch (index) { + case VIRTIO_IVAR_DEVTYPE: + *result = sc->vtpci_devid; + break; + default: + return (vtpci_read_ivar(cn, index, result)); + } + + return (0); +} + +static int +vtpci_modern_write_ivar(device_t dev, device_t child, int index, + uintptr_t value) +{ + struct vtpci_modern_softc *sc; + struct vtpci_common *cn; + + sc = device_get_softc(dev); + cn = &sc->vtpci_common; + + if (vtpci_child_device(cn) != child) + return (ENOENT); + + switch (index) { + default: + return (vtpci_write_ivar(cn, index, value)); + } + + return (0); +} + +static uint64_t +vtpci_modern_negotiate_features(device_t dev, uint64_t child_features) +{ + struct vtpci_modern_softc *sc; + uint64_t host_features, features; + + sc = device_get_softc(dev); + host_features = vtpci_modern_read_features(sc); + + /* + * Since the driver was added as a child of the modern PCI bus, + * always add the V1 flag. + */ + child_features |= VIRTIO_F_VERSION_1; + + features = vtpci_negotiate_features(&sc->vtpci_common, + child_features, host_features); + vtpci_modern_write_features(sc, features); + + return (features); +} + +static int +vtpci_modern_finalize_features(device_t dev) +{ + struct vtpci_modern_softc *sc; + uint8_t status; + + sc = device_get_softc(dev); + + /* + * Must re-read the status after setting it to verify the negotiated + * features were accepted by the device. + * + * BMV: TODO Drivers need to handle possible failure of this method! + */ + vtpci_modern_set_status(sc, VIRTIO_CONFIG_S_FEATURES_OK); + + status = vtpci_modern_get_status(sc); + if ((status & VIRTIO_CONFIG_S_FEATURES_OK) == 0) { + device_printf(dev, "desired features were not accepted\n"); + return (ENOTSUP); + } + + return (0); +} + +static int +vtpci_modern_with_feature(device_t dev, uint64_t feature) +{ + struct vtpci_modern_softc *sc; + + sc = device_get_softc(dev); + + return (vtpci_with_feature(&sc->vtpci_common, feature)); +} + +static uint64_t +vtpci_modern_read_features(struct vtpci_modern_softc *sc) +{ + uint32_t features0, features1; + + vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 0); + features0 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF); + vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 1); + features1 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF); + + return (((uint64_t) features1 << 32) | features0); +} + +static void +vtpci_modern_write_features(struct vtpci_modern_softc *sc, uint64_t features) +{ + uint32_t features0, features1; + + features0 = features; + features1 = features >> 32; + + vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 0); + vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features0); + vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 1); + vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features1); +} + +static int +vtpci_modern_alloc_virtqueues(device_t dev, int flags, int nvqs, + struct vq_alloc_info *vq_info) +{ + struct vtpci_modern_softc *sc; + struct vtpci_common *cn; + uint16_t max_nvqs; + + sc = device_get_softc(dev); + cn = &sc->vtpci_common; + + max_nvqs = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_NUMQ); + if (nvqs > max_nvqs) { + device_printf(sc->vtpci_dev, "requested virtqueue count %d " + "exceeds max %d\n", nvqs, max_nvqs); + return (E2BIG); + } + + return (vtpci_alloc_virtqueues(cn, flags, nvqs, vq_info)); +} + +static int +vtpci_modern_setup_interrupts(device_t dev, enum intr_type type) +{ + struct vtpci_modern_softc *sc; + int error; + + sc = device_get_softc(dev); + + error = vtpci_setup_interrupts(&sc->vtpci_common, type); + if (error == 0) + vtpci_modern_enable_virtqueues(sc); + + return (error); +} + +static void +vtpci_modern_stop(device_t dev) +{ + vtpci_modern_reset(device_get_softc(dev)); +} + +static int +vtpci_modern_reinit(device_t dev, uint64_t features) +{ + struct vtpci_modern_softc *sc; + struct vtpci_common *cn; + int error; + + sc = device_get_softc(dev); + cn = &sc->vtpci_common; + + /* + * Redrive the device initialization. This is a bit of an abuse of + * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to + * play nice. + * + * We do not allow the host device to change from what was originally + * negotiated beyond what the guest driver changed. MSIX state should + * not change, number of virtqueues and their size remain the same, etc. + * This will need to be rethought when we want to support migration. + */ + + if (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET) + vtpci_modern_stop(dev); + + /* + * Quickly drive the status through ACK and DRIVER. The device does + * not become usable again until DRIVER_OK in reinit complete. + */ + vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); + vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); + + /* + * TODO: Check that features are not added as to what was + * originally negotiated. + */ + vtpci_modern_negotiate_features(dev, features); + error = vtpci_modern_finalize_features(dev); + if (error) { + device_printf(dev, "cannot finalize features during reinit\n"); + return (error); + } + + error = vtpci_reinit(cn); + if (error) + return (error); + + return (0); +} + +static void +vtpci_modern_reinit_complete(device_t dev) +{ + struct vtpci_modern_softc *sc; + + sc = device_get_softc(dev); + + vtpci_modern_enable_virtqueues(sc); + vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); +} + +static void +vtpci_modern_notify_vq(device_t dev, uint16_t queue, bus_size_t offset) +{ + struct vtpci_modern_softc *sc; + + sc = device_get_softc(dev); + + vtpci_modern_write_notify_2(sc, offset, queue); +} + +static uint8_t +vtpci_modern_get_status(struct vtpci_modern_softc *sc) +{ + return (vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_STATUS)); +} + +static void +vtpci_modern_set_status(struct vtpci_modern_softc *sc, uint8_t status) +{ + if (status != VIRTIO_CONFIG_STATUS_RESET) + status |= vtpci_modern_get_status(sc); + + vtpci_modern_write_common_1(sc, VIRTIO_PCI_COMMON_STATUS, status); +} + +static int +vtpci_modern_config_generation(device_t dev) +{ + struct vtpci_modern_softc *sc; + uint8_t gen; + + sc = device_get_softc(dev); + gen = vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_CFGGENERATION); + + return (gen); +} + +static void +vtpci_modern_read_dev_config(device_t dev, bus_size_t offset, void *dst, + int length) +{ + struct vtpci_modern_softc *sc; + + sc = device_get_softc(dev); + + if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) { + panic("%s: attempt to read dev config but not present", + __func__); + } + + switch (length) { + case 1: + *(uint8_t *) dst = vtpci_modern_read_device_1(sc, offset); + break; + case 2: + *(uint16_t *) dst = virtio_htog16(true, + vtpci_modern_read_device_2(sc, offset)); + break; + case 4: + *(uint32_t *) dst = virtio_htog32(true, + vtpci_modern_read_device_4(sc, offset)); + break; + case 8: + *(uint64_t *) dst = virtio_htog64(true, + vtpci_modern_read_device_8(sc, offset)); + break; + default: + panic("%s: device %s invalid device read length %d offset %d", + __func__, device_get_nameunit(dev), length, (int) offset); + } +} + +static void +vtpci_modern_write_dev_config(device_t dev, bus_size_t offset, void *src, + int length) +{ + struct vtpci_modern_softc *sc; + + sc = device_get_softc(dev); + + if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) { + panic("%s: attempt to write dev config but not present", + __func__); + } + + switch (length) { + case 1: + vtpci_modern_write_device_1(sc, offset, *(uint8_t *) src); + break; + case 2: { + uint16_t val = virtio_gtoh16(true, *(uint16_t *) src); + vtpci_modern_write_device_2(sc, offset, val); + break; + } + case 4: { + uint32_t val = virtio_gtoh32(true, *(uint32_t *) src); + vtpci_modern_write_device_4(sc, offset, val); + break; + } + case 8: { + uint64_t val = virtio_gtoh64(true, *(uint64_t *) src); + vtpci_modern_write_device_8(sc, offset, val); + break; + } + default: + panic("%s: device %s invalid device write length %d offset %d", + __func__, device_get_nameunit(dev), length, (int) offset); + } +} + +static int +vtpci_modern_probe_configs(device_t dev) +{ + int error; + + /* + * These config capabilities must be present. The DEVICE_CFG + * capability is only present if the device requires it. + */ + + error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_COMMON_CFG, NULL); + if (error) { + device_printf(dev, "cannot find COMMON_CFG capability\n"); + return (error); + } + + error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_NOTIFY_CFG, NULL); + if (error) { + device_printf(dev, "cannot find NOTIFY_CFG capability\n"); + return (error); + } + + error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_ISR_CFG, NULL); + if (error) { + device_printf(dev, "cannot find ISR_CFG capability\n"); + return (error); + } + + return (0); +} + +static int +vtpci_modern_find_cap(device_t dev, uint8_t cfg_type, int *cap_offset) +{ + uint32_t type, bar; + int capreg, error; + + for (error = pci_find_cap(dev, PCIY_VENDOR, &capreg); + error == 0; + error = pci_find_next_cap(dev, PCIY_VENDOR, capreg, &capreg)) { + + type = pci_read_config(dev, capreg + + offsetof(struct virtio_pci_cap, cfg_type), 1); + bar = pci_read_config(dev, capreg + + offsetof(struct virtio_pci_cap, bar), 1); + + /* Must ignore reserved BARs. */ + if (bar >= VTPCI_MODERN_MAX_BARS) + continue; + + if (type == cfg_type) { + if (cap_offset != NULL) + *cap_offset = capreg; + break; + } + } + + return (error); +} + +static int +vtpci_modern_map_common_config(struct vtpci_modern_softc *sc) +{ + device_t dev; + int error; + + dev = sc->vtpci_dev; + + error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_COMMON_CFG, + sizeof(struct virtio_pci_common_cfg), 4, &sc->vtpci_common_res_map); + if (error) { + device_printf(dev, "cannot find cap COMMON_CFG resource\n"); + return (error); + } + + error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_common_res_map); + if (error) { + device_printf(dev, "cannot alloc resource for COMMON_CFG\n"); + return (error); + } + + return (0); +} + +static int +vtpci_modern_map_notify_config(struct vtpci_modern_softc *sc) +{ + device_t dev; + int cap_offset, error; + + dev = sc->vtpci_dev; + + error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_NOTIFY_CFG, + -1, 2, &sc->vtpci_notify_res_map); + if (error) { + device_printf(dev, "cannot find cap NOTIFY_CFG resource\n"); + return (error); + } + + cap_offset = sc->vtpci_notify_res_map.vtrm_cap_offset; + + sc->vtpci_notify_offset_multiplier = pci_read_config(dev, cap_offset + + offsetof(struct virtio_pci_notify_cap, notify_off_multiplier), 4); + + error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_notify_res_map); + if (error) { + device_printf(dev, "cannot alloc resource for NOTIFY_CFG\n"); + return (error); + } + + return (0); +} + +static int +vtpci_modern_map_isr_config(struct vtpci_modern_softc *sc) +{ + device_t dev; + int error; + + dev = sc->vtpci_dev; + + error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_ISR_CFG, + sizeof(uint8_t), 1, &sc->vtpci_isr_res_map); + if (error) { + device_printf(dev, "cannot find cap ISR_CFG resource\n"); + return (error); + } + + error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_isr_res_map); + if (error) { + device_printf(dev, "cannot alloc resource for ISR_CFG\n"); + return (error); + } + + return (0); +} + +static int +vtpci_modern_map_device_config(struct vtpci_modern_softc *sc) +{ + device_t dev; + int error; + + dev = sc->vtpci_dev; + + error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_DEVICE_CFG, + -1, 4, &sc->vtpci_device_res_map); + if (error == ENOENT) { + /* Device configuration is optional depending on device. */ + return (0); + } else if (error) { + device_printf(dev, "cannot find cap DEVICE_CFG resource\n"); + return (error); + } + + error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_device_res_map); + if (error) { + device_printf(dev, "cannot alloc resource for DEVICE_CFG\n"); + return (error); + } + + return (0); +} + +static int +vtpci_modern_map_configs(struct vtpci_modern_softc *sc) +{ + int error; + + error = vtpci_modern_map_common_config(sc); + if (error) + return (error); + + error = vtpci_modern_map_notify_config(sc); + if (error) + return (error); + + error = vtpci_modern_map_isr_config(sc); + if (error) + return (error); + + error = vtpci_modern_map_device_config(sc); + if (error) + return (error); + + vtpci_modern_alloc_msix_resource(sc); + + return (0); +} + +static void +vtpci_modern_unmap_configs(struct vtpci_modern_softc *sc) +{ + + vtpci_modern_free_resource_map(sc, &sc->vtpci_common_res_map); + vtpci_modern_free_resource_map(sc, &sc->vtpci_notify_res_map); + vtpci_modern_free_resource_map(sc, &sc->vtpci_isr_res_map); + vtpci_modern_free_resource_map(sc, &sc->vtpci_device_res_map); + + vtpci_modern_free_bar_resources(sc); + vtpci_modern_free_msix_resource(sc); + + sc->vtpci_notify_offset_multiplier = 0; +} + +static int +vtpci_modern_find_cap_resource(struct vtpci_modern_softc *sc, uint8_t cfg_type, + int min_size, int alignment, struct vtpci_modern_resource_map *res) +{ + device_t dev; + int cap_offset, offset, length, error; + uint8_t bar, cap_length; + + dev = sc->vtpci_dev; + + error = vtpci_modern_find_cap(dev, cfg_type, &cap_offset); + if (error) + return (error); + + cap_length = pci_read_config(dev, + cap_offset + offsetof(struct virtio_pci_cap, cap_len), 1); + + if (cap_length < sizeof(struct virtio_pci_cap)) { + device_printf(dev, "cap %u length %d less than expected\n", + cfg_type, cap_length); + return (ENXIO); + } + + bar = pci_read_config(dev, + cap_offset + offsetof(struct virtio_pci_cap, bar), 1); + offset = pci_read_config(dev, + cap_offset + offsetof(struct virtio_pci_cap, offset), 4); + length = pci_read_config(dev, + cap_offset + offsetof(struct virtio_pci_cap, length), 4); + + if (min_size != -1 && length < min_size) { + device_printf(dev, "cap %u struct length %d less than min %d\n", + cfg_type, length, min_size); + return (ENXIO); + } + + if (offset % alignment) { + device_printf(dev, "cap %u struct offset %d not aligned to %d\n", + cfg_type, offset, alignment); + return (ENXIO); + } + + /* BMV: TODO Can we determine the size of the BAR here? */ + + res->vtrm_cap_offset = cap_offset; + res->vtrm_bar = bar; + res->vtrm_offset = offset; + res->vtrm_length = length; + res->vtrm_type = vtpci_modern_bar_type(sc, bar); + + return (0); +} + +static int +vtpci_modern_bar_type(struct vtpci_modern_softc *sc, int bar) +{ + uint32_t val; + + /* + * The BAR described by a config capability may be either an IOPORT or + * MEM, but we must know the type when calling bus_alloc_resource(). + */ + val = pci_read_config(sc->vtpci_dev, PCIR_BAR(bar), 4); + if (PCI_BAR_IO(val)) + return (SYS_RES_IOPORT); + else + return (SYS_RES_MEMORY); +} + +static struct resource * +vtpci_modern_get_bar_resource(struct vtpci_modern_softc *sc, int bar, int type) +{ + struct resource *res; + + MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS); + res = sc->vtpci_bar_res[bar].vtbr_res; + MPASS(res == NULL || sc->vtpci_bar_res[bar].vtbr_type == type); + + return (res); +} + +static struct resource * +vtpci_modern_alloc_bar_resource(struct vtpci_modern_softc *sc, int bar, + int type) +{ + struct resource *res; + int rid; + + MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS); + MPASS(type == SYS_RES_MEMORY || type == SYS_RES_IOPORT); + + res = sc->vtpci_bar_res[bar].vtbr_res; + if (res != NULL) { + MPASS(sc->vtpci_bar_res[bar].vtbr_type == type); + return (res); + } + + rid = PCIR_BAR(bar); + res = bus_alloc_resource_any(sc->vtpci_dev, type, &rid, + RF_ACTIVE | RF_UNMAPPED); + if (res != NULL) { + sc->vtpci_bar_res[bar].vtbr_res = res; + sc->vtpci_bar_res[bar].vtbr_type = type; + } + + return (res); +} + +static void +vtpci_modern_free_bar_resources(struct vtpci_modern_softc *sc) +{ + device_t dev; + struct resource *res; + int bar, rid, type; + + dev = sc->vtpci_dev; + + for (bar = 0; bar < VTPCI_MODERN_MAX_BARS; bar++) { + res = sc->vtpci_bar_res[bar].vtbr_res; + type = sc->vtpci_bar_res[bar].vtbr_type; + + if (res != NULL) { + rid = PCIR_BAR(bar); + bus_release_resource(dev, type, rid, res); + sc->vtpci_bar_res[bar].vtbr_res = NULL; + sc->vtpci_bar_res[bar].vtbr_type = 0; + } + } +} + +static int +vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *sc, + struct vtpci_modern_resource_map *map) +{ + struct resource_map_request req; + struct resource *res; + int type; + + type = map->vtrm_type; + + res = vtpci_modern_alloc_bar_resource(sc, map->vtrm_bar, type); + if (res == NULL) + return (ENXIO); + + resource_init_map_request(&req); + req.offset = map->vtrm_offset; + req.length = map->vtrm_length; + + return (bus_map_resource(sc->vtpci_dev, type, res, &req, + &map->vtrm_map)); +} + +static void +vtpci_modern_free_resource_map(struct vtpci_modern_softc *sc, + struct vtpci_modern_resource_map *map) +{ + struct resource *res; + int type; + + type = map->vtrm_type; + res = vtpci_modern_get_bar_resource(sc, map->vtrm_bar, type); + + if (res != NULL && map->vtrm_map.r_size != 0) { + bus_unmap_resource(sc->vtpci_dev, type, res, &map->vtrm_map); + bzero(map, sizeof(struct vtpci_modern_resource_map)); + } +} + +static void +vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *sc) +{ + device_t dev; + int bar; + + dev = sc->vtpci_dev; + + if (!vtpci_is_msix_available(&sc->vtpci_common) || + (bar = pci_msix_table_bar(dev)) == -1) + return; + + /* TODO: Can this BAR be in the 0-5 range? */ + sc->vtpci_msix_bar = bar; + if ((sc->vtpci_msix_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &bar, RF_ACTIVE)) == NULL) + device_printf(dev, "Unable to map MSIX table\n"); +} + +static void +vtpci_modern_free_msix_resource(struct vtpci_modern_softc *sc) +{ + device_t dev; + + dev = sc->vtpci_dev; + + if (sc->vtpci_msix_res != NULL) { + bus_release_resource(dev, SYS_RES_MEMORY, sc->vtpci_msix_bar, + sc->vtpci_msix_res); + sc->vtpci_msix_bar = 0; + sc->vtpci_msix_res = NULL; + } +} + +static void +vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *sc) +{ + device_t dev, child; + + dev = sc->vtpci_dev; + child = vtpci_child_device(&sc->vtpci_common); + + if (child == NULL || device_get_state(child) != DS_NOTPRESENT) + return; + + if (device_probe(child) != 0) + return; + + vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); + + if (device_attach(child) != 0) { + vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); + /* Reset state for later attempt. */ + vtpci_modern_child_detached(dev, child); + } else { + vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); + VIRTIO_ATTACH_COMPLETED(child); + } +} + +static int +vtpci_modern_register_msix(struct vtpci_modern_softc *sc, int offset, + struct vtpci_interrupt *intr) +{ + uint16_t vector; + + if (intr != NULL) { + /* Map from guest rid to host vector. */ + vector = intr->vti_rid - 1; + } else + vector = VIRTIO_MSI_NO_VECTOR; + + vtpci_modern_write_common_2(sc, offset, vector); + return (vtpci_modern_read_common_2(sc, offset) == vector ? 0 : ENODEV); +} + +static int +vtpci_modern_register_cfg_msix(device_t dev, struct vtpci_interrupt *intr) +{ + struct vtpci_modern_softc *sc; + int error; + + sc = device_get_softc(dev); + + error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_MSIX, intr); + if (error) { + device_printf(dev, + "unable to register config MSIX interrupt\n"); + return (error); + } + + return (0); +} + +static int +vtpci_modern_register_vq_msix(device_t dev, int idx, + struct vtpci_interrupt *intr) +{ + struct vtpci_modern_softc *sc; + int error; + + sc = device_get_softc(dev); + + vtpci_modern_select_virtqueue(sc, idx); + error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_Q_MSIX, intr); + if (error) { + device_printf(dev, + "unable to register virtqueue MSIX interrupt\n"); + return (error); + } + + return (0); +} + +static void +vtpci_modern_reset(struct vtpci_modern_softc *sc) +{ + /* + * Setting the status to RESET sets the host device to the + * original, uninitialized state. Must poll the status until + * the reset is complete. + */ + vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_RESET); + + while (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET) + cpu_spinwait(); +} + +static void +vtpci_modern_select_virtqueue(struct vtpci_modern_softc *sc, int idx) +{ + vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_SELECT, idx); +} + +static uint8_t +vtpci_modern_read_isr(device_t dev) +{ + return (vtpci_modern_read_isr_1(device_get_softc(dev), 0)); +} + +static uint16_t +vtpci_modern_get_vq_size(device_t dev, int idx) +{ + struct vtpci_modern_softc *sc; + + sc = device_get_softc(dev); + + vtpci_modern_select_virtqueue(sc, idx); + return (vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_SIZE)); +} + +static bus_size_t +vtpci_modern_get_vq_notify_off(device_t dev, int idx) +{ + struct vtpci_modern_softc *sc; + uint16_t q_notify_off; + + sc = device_get_softc(dev); + + vtpci_modern_select_virtqueue(sc, idx); + q_notify_off = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_NOFF); + + return (q_notify_off * sc->vtpci_notify_offset_multiplier); +} + +static void +vtpci_modern_set_vq(device_t dev, struct virtqueue *vq) +{ + struct vtpci_modern_softc *sc; + + sc = device_get_softc(dev); + + vtpci_modern_select_virtqueue(sc, virtqueue_index(vq)); + + /* BMV: Currently we never adjust the device's proposed VQ size. */ + vtpci_modern_write_common_2(sc, + VIRTIO_PCI_COMMON_Q_SIZE, virtqueue_size(vq)); + + vtpci_modern_write_common_8(sc, + VIRTIO_PCI_COMMON_Q_DESCLO, virtqueue_desc_paddr(vq)); + vtpci_modern_write_common_8(sc, + VIRTIO_PCI_COMMON_Q_AVAILLO, virtqueue_avail_paddr(vq)); + vtpci_modern_write_common_8(sc, + VIRTIO_PCI_COMMON_Q_USEDLO, virtqueue_used_paddr(vq)); +} + +static void +vtpci_modern_disable_vq(device_t dev, int idx) +{ + struct vtpci_modern_softc *sc; + + sc = device_get_softc(dev); + + vtpci_modern_select_virtqueue(sc, idx); + vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_DESCLO, 0ULL); + vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_AVAILLO, 0ULL); + vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_USEDLO, 0ULL); +} + +static void +vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *sc) +{ + int idx; + + for (idx = 0; idx < sc->vtpci_common.vtpci_nvqs; idx++) { + vtpci_modern_select_virtqueue(sc, idx); + vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_ENABLE, 1); + } +} + +static uint8_t +vtpci_modern_read_common_1(struct vtpci_modern_softc *sc, bus_size_t off) +{ + return (bus_read_1(&sc->vtpci_common_res_map.vtrm_map, off)); +} + +static uint16_t +vtpci_modern_read_common_2(struct vtpci_modern_softc *sc, bus_size_t off) +{ + return (bus_read_2(&sc->vtpci_common_res_map.vtrm_map, off)); +} + +static uint32_t +vtpci_modern_read_common_4(struct vtpci_modern_softc *sc, bus_size_t off) +{ + return (bus_read_4(&sc->vtpci_common_res_map.vtrm_map, off)); +} + +static void +vtpci_modern_write_common_1(struct vtpci_modern_softc *sc, bus_size_t off, + uint8_t val) +{ + bus_write_1(&sc->vtpci_common_res_map.vtrm_map, off, val); +} + +static void +vtpci_modern_write_common_2(struct vtpci_modern_softc *sc, bus_size_t off, + uint16_t val) +{ + bus_write_2(&sc->vtpci_common_res_map.vtrm_map, off, val); +} + +static void +vtpci_modern_write_common_4(struct vtpci_modern_softc *sc, bus_size_t off, + uint32_t val) +{ + bus_write_4(&sc->vtpci_common_res_map.vtrm_map, off, val); +} + +static void +vtpci_modern_write_common_8(struct vtpci_modern_softc *sc, bus_size_t off, + uint64_t val) +{ + uint32_t val0, val1; + + val0 = (uint32_t) val; + val1 = val >> 32; + + vtpci_modern_write_common_4(sc, off, val0); + vtpci_modern_write_common_4(sc, off + 4, val1); +} + +static void +vtpci_modern_write_notify_2(struct vtpci_modern_softc *sc, bus_size_t off, + uint16_t val) +{ + bus_write_2(&sc->vtpci_notify_res_map.vtrm_map, off, val); +} + +static uint8_t +vtpci_modern_read_isr_1(struct vtpci_modern_softc *sc, bus_size_t off) +{ + return (bus_read_1(&sc->vtpci_isr_res_map.vtrm_map, off)); +} + +static uint8_t +vtpci_modern_read_device_1(struct vtpci_modern_softc *sc, bus_size_t off) +{ + return (bus_read_1(&sc->vtpci_device_res_map.vtrm_map, off)); +} + +static uint16_t +vtpci_modern_read_device_2(struct vtpci_modern_softc *sc, bus_size_t off) +{ + return (bus_read_2(&sc->vtpci_device_res_map.vtrm_map, off)); +} + +static uint32_t +vtpci_modern_read_device_4(struct vtpci_modern_softc *sc, bus_size_t off) +{ + return (bus_read_4(&sc->vtpci_device_res_map.vtrm_map, off)); +} + +static uint64_t +vtpci_modern_read_device_8(struct vtpci_modern_softc *sc, bus_size_t off) +{ + device_t dev; + int gen; + uint32_t val0, val1; + + dev = sc->vtpci_dev; + + /* + * Treat the 64-bit field as two 32-bit fields. Use the generation + * to ensure a consistent read. + */ + do { + gen = vtpci_modern_config_generation(dev); + val0 = vtpci_modern_read_device_4(sc, off); + val1 = vtpci_modern_read_device_4(sc, off + 4); + } while (gen != vtpci_modern_config_generation(dev)); + + return (((uint64_t) val1 << 32) | val0); +} + +static void +vtpci_modern_write_device_1(struct vtpci_modern_softc *sc, bus_size_t off, + uint8_t val) +{ + bus_write_1(&sc->vtpci_device_res_map.vtrm_map, off, val); +} + +static void +vtpci_modern_write_device_2(struct vtpci_modern_softc *sc, bus_size_t off, + uint16_t val) +{ + bus_write_2(&sc->vtpci_device_res_map.vtrm_map, off, val); +} + +static void +vtpci_modern_write_device_4(struct vtpci_modern_softc *sc, bus_size_t off, + uint32_t val) +{ + bus_write_4(&sc->vtpci_device_res_map.vtrm_map, off, val); +} + +static void +vtpci_modern_write_device_8(struct vtpci_modern_softc *sc, bus_size_t off, + uint64_t val) +{ + uint32_t val0, val1; + + val0 = (uint32_t) val; + val1 = val >> 32; + + vtpci_modern_write_device_4(sc, off, val0); + vtpci_modern_write_device_4(sc, off + 4, val1); +} diff --git a/sys/dev/virtio/pci/virtio_pci_modern_var.h b/sys/dev/virtio/pci/virtio_pci_modern_var.h new file mode 100644 index 000000000000..86d24a3cf54b --- /dev/null +++ b/sys/dev/virtio/pci/virtio_pci_modern_var.h @@ -0,0 +1,135 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright IBM Corp. 2007 + * + * Authors: + * Anthony Liguori <aliguori@us.ibm.com> + * + * This header is BSD licensed so anyone can use the definitions to implement + * compatible drivers/servers. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of IBM nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _VIRTIO_PCI_MODERN_VAR_H +#define _VIRTIO_PCI_MODERN_VAR_H + +#include <dev/virtio/pci/virtio_pci_var.h> + +/* IDs for different capabilities. Must all exist. */ +/* Common configuration */ +#define VIRTIO_PCI_CAP_COMMON_CFG 1 +/* Notifications */ +#define VIRTIO_PCI_CAP_NOTIFY_CFG 2 +/* ISR access */ +#define VIRTIO_PCI_CAP_ISR_CFG 3 +/* Device specific configuration */ +#define VIRTIO_PCI_CAP_DEVICE_CFG 4 +/* PCI configuration access */ +#define VIRTIO_PCI_CAP_PCI_CFG 5 + +/* This is the PCI capability header: */ +struct virtio_pci_cap { + uint8_t cap_vndr; /* Generic PCI field: PCI_CAP_ID_VNDR */ + uint8_t cap_next; /* Generic PCI field: next ptr. */ + uint8_t cap_len; /* Generic PCI field: capability length */ + uint8_t cfg_type; /* Identifies the structure. */ + uint8_t bar; /* Where to find it. */ + uint8_t padding[3]; /* Pad to full dword. */ + uint32_t offset; /* Offset within bar. */ + uint32_t length; /* Length of the structure, in bytes. */ +}; + +struct virtio_pci_notify_cap { + struct virtio_pci_cap cap; + uint32_t notify_off_multiplier; /* Multiplier for queue_notify_off. */ +}; + +/* Fields in VIRTIO_PCI_CAP_COMMON_CFG: */ +struct virtio_pci_common_cfg { + /* About the whole device. */ + uint32_t device_feature_select; /* read-write */ + uint32_t device_feature; /* read-only */ + uint32_t guest_feature_select; /* read-write */ + uint32_t guest_feature; /* read-write */ + uint16_t msix_config; /* read-write */ + uint16_t num_queues; /* read-only */ + uint8_t device_status; /* read-write */ + uint8_t config_generation; /* read-only */ + + /* About a specific virtqueue. */ + uint16_t queue_select; /* read-write */ + uint16_t queue_size; /* read-write, power of 2. */ + uint16_t queue_msix_vector; /* read-write */ + uint16_t queue_enable; /* read-write */ + uint16_t queue_notify_off; /* read-only */ + uint32_t queue_desc_lo; /* read-write */ + uint32_t queue_desc_hi; /* read-write */ + uint32_t queue_avail_lo; /* read-write */ + uint32_t queue_avail_hi; /* read-write */ + uint32_t queue_used_lo; /* read-write */ + uint32_t queue_used_hi; /* read-write */ +}; + +/* Fields in VIRTIO_PCI_CAP_PCI_CFG: */ +struct virtio_pci_cfg_cap { + struct virtio_pci_cap cap; + uint8_t pci_cfg_data[4]; /* Data for BAR access. */ +}; + +/* Macro versions of offsets for the Old Timers! */ +#define VIRTIO_PCI_CAP_VNDR 0 +#define VIRTIO_PCI_CAP_NEXT 1 +#define VIRTIO_PCI_CAP_LEN 2 +#define VIRTIO_PCI_CAP_CFG_TYPE 3 +#define VIRTIO_PCI_CAP_BAR 4 +#define VIRTIO_PCI_CAP_OFFSET 8 +#define VIRTIO_PCI_CAP_LENGTH 12 + +#define VIRTIO_PCI_NOTIFY_CAP_MULT 16 + +#define VIRTIO_PCI_COMMON_DFSELECT 0 +#define VIRTIO_PCI_COMMON_DF 4 +#define VIRTIO_PCI_COMMON_GFSELECT 8 +#define VIRTIO_PCI_COMMON_GF 12 +#define VIRTIO_PCI_COMMON_MSIX 16 +#define VIRTIO_PCI_COMMON_NUMQ 18 +#define VIRTIO_PCI_COMMON_STATUS 20 +#define VIRTIO_PCI_COMMON_CFGGENERATION 21 +#define VIRTIO_PCI_COMMON_Q_SELECT 22 +#define VIRTIO_PCI_COMMON_Q_SIZE 24 +#define VIRTIO_PCI_COMMON_Q_MSIX 26 +#define VIRTIO_PCI_COMMON_Q_ENABLE 28 +#define VIRTIO_PCI_COMMON_Q_NOFF 30 +#define VIRTIO_PCI_COMMON_Q_DESCLO 32 +#define VIRTIO_PCI_COMMON_Q_DESCHI 36 +#define VIRTIO_PCI_COMMON_Q_AVAILLO 40 +#define VIRTIO_PCI_COMMON_Q_AVAILHI 44 +#define VIRTIO_PCI_COMMON_Q_USEDLO 48 +#define VIRTIO_PCI_COMMON_Q_USEDHI 52 + +#endif /* _VIRTIO_PCI_MODERN_VAR_H */ diff --git a/sys/dev/virtio/pci/virtio_pci_var.h b/sys/dev/virtio/pci/virtio_pci_var.h new file mode 100644 index 000000000000..bd0345a4c62a --- /dev/null +++ b/sys/dev/virtio/pci/virtio_pci_var.h @@ -0,0 +1,55 @@ +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright IBM Corp. 2007 + * + * Authors: + * Anthony Liguori <aliguori@us.ibm.com> + * + * This header is BSD licensed so anyone can use the definitions to implement + * compatible drivers/servers. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of IBM nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _VIRTIO_PCI_VAR_H +#define _VIRTIO_PCI_VAR_H + +/* VirtIO PCI vendor/device ID. */ +#define VIRTIO_PCI_VENDORID 0x1AF4 +#define VIRTIO_PCI_DEVICEID_MIN 0x1000 +#define VIRTIO_PCI_DEVICEID_LEGACY_MAX 0x103F +#define VIRTIO_PCI_DEVICEID_MODERN_MIN 0x1040 +#define VIRTIO_PCI_DEVICEID_MODERN_MAX 0x107F + +/* The bit of the ISR which indicates a device has an interrupt. */ +#define VIRTIO_PCI_ISR_INTR 0x1 +/* The bit of the ISR which indicates a device configuration change. */ +#define VIRTIO_PCI_ISR_CONFIG 0x2 +/* Vector value used to disable MSI for queue. */ +#define VIRTIO_MSI_NO_VECTOR 0xFFFF + +#endif /* _VIRTIO_PCI_VAR_H */ diff --git a/sys/dev/virtio/virtio.c b/sys/dev/virtio/virtio.c index b6537e9305ea..18eace65a12b 100644 --- a/sys/dev/virtio/virtio.c +++ b/sys/dev/virtio/virtio.c @@ -60,24 +60,28 @@ static struct virtio_ident { { VIRTIO_ID_ENTROPY, "Entropy" }, { VIRTIO_ID_BALLOON, "Balloon" }, { VIRTIO_ID_IOMEMORY, "IOMemory" }, - { VIRTIO_ID_RPMSG, "Remote Processor Messaging" }, + { VIRTIO_ID_RPMSG, "Remote Processor Messaging" }, { VIRTIO_ID_SCSI, "SCSI" }, { VIRTIO_ID_9P, "9P Transport" }, { VIRTIO_ID_RPROC_SERIAL, "Remote Processor Serial" }, { VIRTIO_ID_CAIF, "CAIF" }, { VIRTIO_ID_GPU, "GPU" }, - { VIRTIO_ID_INPUT, "Input" }, - { VIRTIO_ID_VSOCK, "VSOCK Transport" }, - { VIRTIO_ID_CRYPTO, "Crypto" }, + { VIRTIO_ID_INPUT, "Input" }, + { VIRTIO_ID_VSOCK, "VSOCK Transport" }, + { VIRTIO_ID_CRYPTO, "Crypto" }, + { 0, NULL } }; /* Device independent features. */ static struct virtio_feature_desc virtio_common_feature_desc[] = { - { VIRTIO_F_NOTIFY_ON_EMPTY, "NotifyOnEmpty" }, - { VIRTIO_RING_F_INDIRECT_DESC, "RingIndirect" }, - { VIRTIO_RING_F_EVENT_IDX, "EventIdx" }, - { VIRTIO_F_BAD_FEATURE, "BadFeature" }, + { VIRTIO_F_NOTIFY_ON_EMPTY, "NotifyOnEmpty" }, /* Legacy */ + { VIRTIO_F_ANY_LAYOUT, "AnyLayout" }, /* Legacy */ + { VIRTIO_RING_F_INDIRECT_DESC, "RingIndirectDesc" }, + { VIRTIO_RING_F_EVENT_IDX, "RingEventIdx" }, + { VIRTIO_F_BAD_FEATURE, "BadFeature" }, /* Legacy */ + { VIRTIO_F_VERSION_1, "Version1" }, + { VIRTIO_F_IOMMU_PLATFORM, "IOMMUPlatform" }, { 0, NULL } }; @@ -125,12 +129,13 @@ virtio_describe(device_t dev, const char *msg, const char *name; int n; - if ((buf = malloc(512, M_TEMP, M_NOWAIT)) == NULL) { - device_printf(dev, "%s features: %#jx\n", msg, (uintmax_t) features); + if ((buf = malloc(1024, M_TEMP, M_NOWAIT)) == NULL) { + device_printf(dev, "%s features: %#jx\n", + msg, (uintmax_t) features); return; } - sbuf_new(&sb, buf, 512, SBUF_FIXEDLEN); + sbuf_new(&sb, buf, 1024, SBUF_FIXEDLEN); sbuf_printf(&sb, "%s features: %#jx", msg, (uintmax_t) features); for (n = 0, val = 1ULL << 63; val != 0; val >>= 1) { @@ -163,6 +168,48 @@ virtio_describe(device_t dev, const char *msg, free(buf, M_TEMP); } +uint64_t +virtio_filter_transport_features(uint64_t features) +{ + uint64_t transport, mask; + + transport = (1ULL << + (VIRTIO_TRANSPORT_F_END - VIRTIO_TRANSPORT_F_START)) - 1; + transport <<= VIRTIO_TRANSPORT_F_START; + + mask = -1ULL & ~transport; + mask |= VIRTIO_RING_F_INDIRECT_DESC; + mask |= VIRTIO_RING_F_EVENT_IDX; + mask |= VIRTIO_F_VERSION_1; + + return (features & mask); +} + +int +virtio_bus_is_modern(device_t dev) +{ + uintptr_t modern; + + virtio_read_ivar(dev, VIRTIO_IVAR_MODERN, &modern); + return (modern != 0); +} + +void +virtio_read_device_config_array(device_t dev, bus_size_t offset, void *dst, + int size, int count) +{ + int i, gen; + + do { + gen = virtio_config_generation(dev); + + for (i = 0; i < count; i++) { + virtio_read_device_config(dev, offset + i * size, + (uint8_t *) dst + i * size, size); + } + } while (gen != virtio_config_generation(dev)); +} + /* * VirtIO bus method wrappers. */ @@ -191,6 +238,13 @@ virtio_negotiate_features(device_t dev, uint64_t child_features) } int +virtio_finalize_features(device_t dev) +{ + + return (VIRTIO_BUS_FINALIZE_FEATURES(device_get_parent(dev))); +} + +int virtio_alloc_virtqueues(device_t dev, int flags, int nvqs, struct vq_alloc_info *info) { diff --git a/sys/dev/virtio/virtio.h b/sys/dev/virtio/virtio.h index 6628b3765bcd..85160eb18d0d 100644 --- a/sys/dev/virtio/virtio.h +++ b/sys/dev/virtio/virtio.h @@ -31,6 +31,7 @@ #ifndef _VIRTIO_H_ #define _VIRTIO_H_ +#include <dev/virtio/virtio_endian.h> #include <dev/virtio/virtio_ids.h> #include <dev/virtio/virtio_config.h> @@ -57,6 +58,7 @@ struct vq_alloc_info; #define VIRTIO_IVAR_DEVICE 4 #define VIRTIO_IVAR_SUBVENDOR 5 #define VIRTIO_IVAR_SUBDEVICE 6 +#define VIRTIO_IVAR_MODERN 7 struct virtio_feature_desc { uint64_t vfd_val; @@ -81,6 +83,10 @@ struct virtio_pnp_match { const char *virtio_device_name(uint16_t devid); void virtio_describe(device_t dev, const char *msg, uint64_t features, struct virtio_feature_desc *feature_desc); +uint64_t virtio_filter_transport_features(uint64_t features); +int virtio_bus_is_modern(device_t dev); +void virtio_read_device_config_array(device_t dev, bus_size_t offset, + void *dst, int size, int count); /* * VirtIO Bus Methods. @@ -88,6 +94,7 @@ void virtio_describe(device_t dev, const char *msg, void virtio_read_ivar(device_t dev, int ivar, uintptr_t *val); void virtio_write_ivar(device_t dev, int ivar, uintptr_t val); uint64_t virtio_negotiate_features(device_t dev, uint64_t child_features); +int virtio_finalize_features(device_t dev); int virtio_alloc_virtqueues(device_t dev, int flags, int nvqs, struct vq_alloc_info *info); int virtio_setup_intr(device_t dev, enum intr_type type); @@ -147,6 +154,7 @@ VIRTIO_READ_IVAR(vendor, VIRTIO_IVAR_VENDOR); VIRTIO_READ_IVAR(device, VIRTIO_IVAR_DEVICE); VIRTIO_READ_IVAR(subvendor, VIRTIO_IVAR_SUBVENDOR); VIRTIO_READ_IVAR(subdevice, VIRTIO_IVAR_SUBDEVICE); +VIRTIO_READ_IVAR(modern, VIRTIO_IVAR_MODERN); #undef VIRTIO_READ_IVAR diff --git a/sys/dev/virtio/virtio_bus_if.m b/sys/dev/virtio/virtio_bus_if.m index ec280e2d80be..2c3424204842 100644 --- a/sys/dev/virtio/virtio_bus_if.m +++ b/sys/dev/virtio/virtio_bus_if.m @@ -36,6 +36,12 @@ struct vq_alloc_info; CODE { static int + virtio_bus_default_finalize_features(device_t dev) + { + return (0); + } + + static int virtio_bus_default_config_generation(device_t dev) { return (0); @@ -47,6 +53,10 @@ METHOD uint64_t negotiate_features { uint64_t child_features; }; +METHOD int finalize_features { + device_t dev; +} DEFAULT virtio_bus_default_finalize_features; + METHOD int with_feature { device_t dev; uint64_t feature; @@ -80,6 +90,7 @@ METHOD void reinit_complete { METHOD void notify_vq { device_t dev; uint16_t queue; + bus_size_t offset; }; METHOD int config_generation { diff --git a/sys/dev/virtio/virtio_endian.h b/sys/dev/virtio/virtio_endian.h new file mode 100644 index 000000000000..d0de299c7227 --- /dev/null +++ b/sys/dev/virtio/virtio_endian.h @@ -0,0 +1,106 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _VIRTIO_ENDIAN_H_ +#define _VIRTIO_ENDIAN_H_ + +#include <sys/endian.h> + +/* + * VirtIO V1 (modern) uses little endian, while legacy VirtIO uses the guest's + * native endian. These functions convert to and from the Guest's (driver's) + * and the Host's (device's) endianness when needed. + */ + +static inline bool +virtio_swap_endian(bool modern) +{ +#if _BYTE_ORDER == _LITTLE_ENDIAN + return (false); +#else + return (modern); +#endif +} + +static inline uint16_t +virtio_htog16(bool modern, uint16_t val) +{ + if (virtio_swap_endian(modern)) + return (le16toh(val)); + else + return (val); +} + +static inline uint16_t +virtio_gtoh16(bool modern, uint16_t val) +{ + if (virtio_swap_endian(modern)) + return (htole16(val)); + else + return (val); +} + +static inline uint32_t +virtio_htog32(bool modern, uint32_t val) +{ + if (virtio_swap_endian(modern)) + return (le32toh(val)); + else + return (val); +} + +static inline uint32_t +virtio_gtoh32(bool modern, uint32_t val) +{ + if (virtio_swap_endian(modern)) + return (htole32(val)); + else + return (val); +} + +static inline uint64_t +virtio_htog64(bool modern, uint64_t val) +{ + if (virtio_swap_endian(modern)) + return (le64toh(val)); + else + return (val); +} + +static inline uint64_t +virtio_gtoh64(bool modern, uint64_t val) +{ + if (virtio_swap_endian(modern)) + return (htole64(val)); + else + return (val); +} + +#endif /* _VIRTIO_ENDIAN_H_ */ diff --git a/sys/dev/virtio/virtqueue.c b/sys/dev/virtio/virtqueue.c index e23d4d25c47f..da6e7bf89fd5 100644 --- a/sys/dev/virtio/virtqueue.c +++ b/sys/dev/virtio/virtqueue.c @@ -64,6 +64,7 @@ struct virtqueue { #define VIRTQUEUE_FLAG_INDIRECT 0x0001 #define VIRTQUEUE_FLAG_EVENT_IDX 0x0002 + bus_size_t vq_notify_offset; int vq_alignment; int vq_ring_size; void *vq_ring_mem; @@ -147,8 +148,9 @@ virtqueue_filter_features(uint64_t features) } int -virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align, - vm_paddr_t highaddr, struct vq_alloc_info *info, struct virtqueue **vqp) +virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, + bus_size_t notify_offset, int align, vm_paddr_t highaddr, + struct vq_alloc_info *info, struct virtqueue **vqp) { struct virtqueue *vq; int error; @@ -184,6 +186,7 @@ virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align, vq->vq_dev = dev; strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name)); vq->vq_queue_index = queue; + vq->vq_notify_offset = notify_offset; vq->vq_alignment = align; vq->vq_nentries = size; vq->vq_free_cnt = size; @@ -820,7 +823,8 @@ static void vq_ring_notify_host(struct virtqueue *vq) { - VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index); + VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index, + vq->vq_notify_offset); } static void diff --git a/sys/dev/virtio/virtqueue.h b/sys/dev/virtio/virtqueue.h index 7b2db673f7ae..6ac5899cf0c3 100644 --- a/sys/dev/virtio/virtqueue.h +++ b/sys/dev/virtio/virtqueue.h @@ -70,8 +70,8 @@ struct vq_alloc_info { uint64_t virtqueue_filter_features(uint64_t features); int virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, - int align, vm_paddr_t highaddr, struct vq_alloc_info *info, - struct virtqueue **vqp); + bus_size_t notify_offset, int align, vm_paddr_t highaddr, + struct vq_alloc_info *info, struct virtqueue **vqp); void *virtqueue_drain(struct virtqueue *vq, int *last); void virtqueue_free(struct virtqueue *vq); int virtqueue_reinit(struct virtqueue *vq, uint16_t size); |