diff options
Diffstat (limited to 'sys/arm/mv/mv_pci.c')
| -rw-r--r-- | sys/arm/mv/mv_pci.c | 1382 |
1 files changed, 1382 insertions, 0 deletions
diff --git a/sys/arm/mv/mv_pci.c b/sys/arm/mv/mv_pci.c new file mode 100644 index 000000000000..5983076d6be8 --- /dev/null +++ b/sys/arm/mv/mv_pci.c @@ -0,0 +1,1382 @@ +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 2008 MARVELL INTERNATIONAL LTD. + * Copyright (c) 2010 The FreeBSD Foundation + * Copyright (c) 2010-2015 Semihalf + * All rights reserved. + * + * Developed by Semihalf. + * + * Portions of this software were developed by Semihalf + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of MARVELL nor the names of contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Marvell integrated PCI/PCI-Express controller driver. + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/kernel.h> +#include <sys/lock.h> +#include <sys/malloc.h> +#include <sys/module.h> +#include <sys/mutex.h> +#include <sys/queue.h> +#include <sys/bus.h> +#include <sys/rman.h> +#include <sys/endian.h> +#include <sys/devmap.h> + +#include <machine/fdt.h> +#include <machine/intr.h> + +#include <vm/vm.h> +#include <vm/pmap.h> + +#include <dev/fdt/fdt_common.h> +#include <dev/ofw/ofw_bus.h> +#include <dev/ofw/ofw_bus_subr.h> +#include <dev/ofw/ofw_pci.h> +#include <dev/pci/pcivar.h> +#include <dev/pci/pcireg.h> +#include <dev/pci/pcib_private.h> + +#include "ofw_bus_if.h" +#include "pcib_if.h" + +#include <machine/resource.h> +#include <machine/bus.h> + +#include <arm/mv/mvreg.h> +#include <arm/mv/mvvar.h> +#include <arm/mv/mvwin.h> + +#ifdef DEBUG +#define debugf(fmt, args...) do { printf(fmt,##args); } while (0) +#else +#define debugf(fmt, args...) +#endif + +/* + * Code and data related to fdt-based PCI configuration. + * + * This stuff used to be in dev/fdt/fdt_pci.c and fdt_common.h, but it was + * always Marvell-specific so that was deleted and the code now lives here. + */ + +struct mv_pci_range { + u_long base_pci; + u_long base_parent; + u_long len; +}; + +#define FDT_RANGES_CELLS ((3 + 3 + 2) * 2) +#define PCI_SPACE_LEN 0x00400000 + +static void +mv_pci_range_dump(struct mv_pci_range *range) +{ +#ifdef DEBUG + printf("\n"); + printf(" base_pci = 0x%08lx\n", range->base_pci); + printf(" base_par = 0x%08lx\n", range->base_parent); + printf(" len = 0x%08lx\n", range->len); +#endif +} + +static int +mv_pci_ranges_decode(phandle_t node, struct mv_pci_range *io_space, + struct mv_pci_range *mem_space) +{ + pcell_t ranges[FDT_RANGES_CELLS]; + struct mv_pci_range *pci_space; + pcell_t addr_cells, size_cells, par_addr_cells; + pcell_t *rangesptr; + pcell_t cell0, cell2; + int tuple_size, tuples, i, rv, offset_cells, len; + int portid, is_io_space; + + /* + * Retrieve 'ranges' property. + */ + if ((fdt_addrsize_cells(node, &addr_cells, &size_cells)) != 0) + return (EINVAL); + if (addr_cells != 3 || size_cells != 2) + return (ERANGE); + + par_addr_cells = fdt_parent_addr_cells(node); + if (par_addr_cells > 3) + return (ERANGE); + + len = OF_getproplen(node, "ranges"); + if (len > sizeof(ranges)) + return (ENOMEM); + + if (OF_getprop(node, "ranges", ranges, sizeof(ranges)) <= 0) + return (EINVAL); + + tuple_size = sizeof(pcell_t) * (addr_cells + par_addr_cells + + size_cells); + tuples = len / tuple_size; + + /* + * Initialize the ranges so that we don't have to worry about + * having them all defined in the FDT. In particular, it is + * perfectly fine not to want I/O space on PCI buses. + */ + bzero(io_space, sizeof(*io_space)); + bzero(mem_space, sizeof(*mem_space)); + + rangesptr = &ranges[0]; + offset_cells = 0; + for (i = 0; i < tuples; i++) { + cell0 = fdt_data_get((void *)rangesptr, 1); + rangesptr++; + /* cell1 */ + rangesptr++; + cell2 = fdt_data_get((void *)rangesptr, 1); + rangesptr++; + portid = fdt_data_get((void *)(rangesptr+1), 1); + + if (cell0 & 0x02000000) { + pci_space = mem_space; + is_io_space = 0; + } else if (cell0 & 0x01000000) { + pci_space = io_space; + is_io_space = 1; + } else { + rv = ERANGE; + goto out; + } + + if (par_addr_cells == 3) { + /* + * This is a PCI subnode 'ranges'. Skip cell0 and + * cell1 of this entry and only use cell2. + */ + offset_cells = 2; + rangesptr += offset_cells; + } + + if ((par_addr_cells - offset_cells) > 2) { + rv = ERANGE; + goto out; + } + pci_space->base_parent = fdt_data_get((void *)rangesptr, + par_addr_cells - offset_cells); + rangesptr += par_addr_cells - offset_cells; + + if (size_cells > 2) { + rv = ERANGE; + goto out; + } + pci_space->len = fdt_data_get((void *)rangesptr, size_cells); + rangesptr += size_cells; + + pci_space->base_pci = cell2; + + if (pci_space->len == 0) { + pci_space->len = PCI_SPACE_LEN; + pci_space->base_parent = fdt_immr_va + + PCI_SPACE_LEN * ( 2 * portid + is_io_space); + } + } + rv = 0; +out: + return (rv); +} + +static int +mv_pci_ranges(phandle_t node, struct mv_pci_range *io_space, + struct mv_pci_range *mem_space) +{ + int err; + + debugf("Processing PCI node: %x\n", node); + if ((err = mv_pci_ranges_decode(node, io_space, mem_space)) != 0) { + debugf("could not decode parent PCI node 'ranges'\n"); + return (err); + } + + debugf("Post fixup dump:\n"); + mv_pci_range_dump(io_space); + mv_pci_range_dump(mem_space); + return (0); +} + +int +mv_pci_devmap(phandle_t node, struct devmap_entry *devmap, vm_offset_t io_va, + vm_offset_t mem_va) +{ + struct mv_pci_range io_space, mem_space; + int error; + + if ((error = mv_pci_ranges_decode(node, &io_space, &mem_space)) != 0) + return (error); + + devmap->pd_va = (io_va ? io_va : io_space.base_parent); + devmap->pd_pa = io_space.base_parent; + devmap->pd_size = io_space.len; + devmap++; + + devmap->pd_va = (mem_va ? mem_va : mem_space.base_parent); + devmap->pd_pa = mem_space.base_parent; + devmap->pd_size = mem_space.len; + return (0); +} + +/* + * Code and data related to the Marvell pcib driver. + */ + +#define PCI_CFG_ENA (1U << 31) +#define PCI_CFG_BUS(bus) (((bus) & 0xff) << 16) +#define PCI_CFG_DEV(dev) (((dev) & 0x1f) << 11) +#define PCI_CFG_FUN(fun) (((fun) & 0x7) << 8) +#define PCI_CFG_PCIE_REG(reg) ((reg) & 0xfc) + +#define PCI_REG_CFG_ADDR 0x0C78 +#define PCI_REG_CFG_DATA 0x0C7C + +#define PCIE_REG_CFG_ADDR 0x18F8 +#define PCIE_REG_CFG_DATA 0x18FC +#define PCIE_REG_CONTROL 0x1A00 +#define PCIE_CTRL_LINK1X 0x00000001 +#define PCIE_REG_STATUS 0x1A04 +#define PCIE_REG_IRQ_MASK 0x1910 + +#define PCIE_CONTROL_ROOT_CMPLX (1 << 1) +#define PCIE_CONTROL_HOT_RESET (1 << 24) + +#define PCIE_LINK_TIMEOUT 1000000 + +#define PCIE_STATUS_LINK_DOWN 1 +#define PCIE_STATUS_DEV_OFFS 16 + +/* Minimum PCI Memory and I/O allocations taken from PCI spec (in bytes) */ +#define PCI_MIN_IO_ALLOC 4 +#define PCI_MIN_MEM_ALLOC 16 + +#define BITS_PER_UINT32 (NBBY * sizeof(uint32_t)) + +struct mv_pcib_softc { + device_t sc_dev; + + struct rman sc_mem_rman; + bus_addr_t sc_mem_base; + bus_addr_t sc_mem_size; + uint32_t sc_mem_map[MV_PCI_MEM_SLICE_SIZE / + (PCI_MIN_MEM_ALLOC * BITS_PER_UINT32)]; + int sc_win_target; + int sc_mem_win_attr; + + struct rman sc_io_rman; + bus_addr_t sc_io_base; + bus_addr_t sc_io_size; + uint32_t sc_io_map[MV_PCI_IO_SLICE_SIZE / + (PCI_MIN_IO_ALLOC * BITS_PER_UINT32)]; + int sc_io_win_attr; + + struct resource *sc_res; + bus_space_handle_t sc_bsh; + bus_space_tag_t sc_bst; + int sc_rid; + + struct mtx sc_msi_mtx; + uint32_t sc_msi_bitmap; + + int sc_busnr; /* Host bridge bus number */ + int sc_devnr; /* Host bridge device number */ + int sc_type; + int sc_mode; /* Endpoint / Root Complex */ + + int sc_msi_supported; + int sc_skip_enable_procedure; + int sc_enable_find_root_slot; + struct ofw_bus_iinfo sc_pci_iinfo; + + int ap_segment; /* PCI domain */ +}; + +/* Local forward prototypes */ +static int mv_pcib_decode_win(phandle_t, struct mv_pcib_softc *); +static void mv_pcib_hw_cfginit(void); +static uint32_t mv_pcib_hw_cfgread(struct mv_pcib_softc *, u_int, u_int, + u_int, u_int, int); +static void mv_pcib_hw_cfgwrite(struct mv_pcib_softc *, u_int, u_int, + u_int, u_int, uint32_t, int); +static int mv_pcib_init(struct mv_pcib_softc *, int, int); +static int mv_pcib_init_all_bars(struct mv_pcib_softc *, int, int, int, int); +static void mv_pcib_init_bridge(struct mv_pcib_softc *, int, int, int); +static inline void pcib_write_irq_mask(struct mv_pcib_softc *, uint32_t); +static void mv_pcib_enable(struct mv_pcib_softc *, uint32_t); +static int mv_pcib_mem_init(struct mv_pcib_softc *); + +/* Forward prototypes */ +static int mv_pcib_probe(device_t); +static int mv_pcib_attach(device_t); + +static struct rman *mv_pcib_get_rman(device_t, int, u_int); +static struct resource *mv_pcib_alloc_resource(device_t, device_t, int, int *, + rman_res_t, rman_res_t, rman_res_t, u_int); +static int mv_pcib_adjust_resource(device_t, device_t, struct resource *, + rman_res_t, rman_res_t); +static int mv_pcib_release_resource(device_t, device_t, struct resource *); +static int mv_pcib_activate_resource(device_t, device_t, struct resource *); +static int mv_pcib_deactivate_resource(device_t, device_t, struct resource *); +static int mv_pcib_map_resource(device_t, device_t, struct resource *, + struct resource_map_request *, struct resource_map *); +static int mv_pcib_unmap_resource(device_t, device_t, struct resource *, + struct resource_map *); +static int mv_pcib_read_ivar(device_t, device_t, int, uintptr_t *); +static int mv_pcib_write_ivar(device_t, device_t, int, uintptr_t); + +static int mv_pcib_maxslots(device_t); +static uint32_t mv_pcib_read_config(device_t, u_int, u_int, u_int, u_int, int); +static void mv_pcib_write_config(device_t, u_int, u_int, u_int, u_int, + uint32_t, int); +static int mv_pcib_route_interrupt(device_t, device_t, int); + +static int mv_pcib_alloc_msi(device_t, device_t, int, int, int *); +static int mv_pcib_map_msi(device_t, device_t, int, uint64_t *, uint32_t *); +static int mv_pcib_release_msi(device_t, device_t, int, int *); + +/* + * Bus interface definitions. + */ +static device_method_t mv_pcib_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, mv_pcib_probe), + DEVMETHOD(device_attach, mv_pcib_attach), + + /* Bus interface */ + DEVMETHOD(bus_read_ivar, mv_pcib_read_ivar), + DEVMETHOD(bus_write_ivar, mv_pcib_write_ivar), + DEVMETHOD(bus_get_rman, mv_pcib_get_rman), + DEVMETHOD(bus_alloc_resource, mv_pcib_alloc_resource), + DEVMETHOD(bus_adjust_resource, mv_pcib_adjust_resource), + DEVMETHOD(bus_release_resource, mv_pcib_release_resource), + DEVMETHOD(bus_activate_resource, mv_pcib_activate_resource), + DEVMETHOD(bus_deactivate_resource, mv_pcib_deactivate_resource), + DEVMETHOD(bus_map_resource, mv_pcib_map_resource), + DEVMETHOD(bus_unmap_resource, mv_pcib_unmap_resource), + DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), + DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), + + /* pcib interface */ + DEVMETHOD(pcib_maxslots, mv_pcib_maxslots), + DEVMETHOD(pcib_read_config, mv_pcib_read_config), + DEVMETHOD(pcib_write_config, mv_pcib_write_config), + DEVMETHOD(pcib_route_interrupt, mv_pcib_route_interrupt), + DEVMETHOD(pcib_request_feature, pcib_request_feature_allow), + + DEVMETHOD(pcib_alloc_msi, mv_pcib_alloc_msi), + DEVMETHOD(pcib_release_msi, mv_pcib_release_msi), + DEVMETHOD(pcib_map_msi, mv_pcib_map_msi), + + /* OFW bus interface */ + DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat), + DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model), + DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name), + DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node), + DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type), + + DEVMETHOD_END +}; + +static driver_t mv_pcib_driver = { + "pcib", + mv_pcib_methods, + sizeof(struct mv_pcib_softc), +}; + +DRIVER_MODULE(mv_pcib, ofwbus, mv_pcib_driver, 0, 0); +DRIVER_MODULE(mv_pcib, pcib_ctrl, mv_pcib_driver, 0, 0); + +static struct mtx pcicfg_mtx; + +static int +mv_pcib_probe(device_t self) +{ + phandle_t node; + + node = ofw_bus_get_node(self); + if (!mv_fdt_is_type(node, "pci")) + return (ENXIO); + + if (!(ofw_bus_is_compatible(self, "mrvl,pcie") || + ofw_bus_is_compatible(self, "mrvl,pci") || + ofw_bus_node_is_compatible( + OF_parent(node), "marvell,armada-370-pcie"))) + return (ENXIO); + + if (!ofw_bus_status_okay(self)) + return (ENXIO); + + device_set_desc(self, "Marvell Integrated PCI/PCI-E Controller"); + return (BUS_PROBE_DEFAULT); +} + +static int +mv_pcib_attach(device_t self) +{ + struct mv_pcib_softc *sc; + phandle_t node, parnode; + uint32_t val, reg0; + int err, bus, devfn, port_id; + + sc = device_get_softc(self); + sc->sc_dev = self; + + node = ofw_bus_get_node(self); + parnode = OF_parent(node); + + if (OF_getencprop(node, "marvell,pcie-port", &(port_id), + sizeof(port_id)) <= 0) { + /* If port ID does not exist in the FDT set value to 0 */ + if (!OF_hasprop(node, "marvell,pcie-port")) + port_id = 0; + else + return(ENXIO); + } + + sc->ap_segment = port_id; + + if (ofw_bus_node_is_compatible(node, "mrvl,pcie")) { + sc->sc_type = MV_TYPE_PCIE; + sc->sc_win_target = MV_WIN_PCIE_TARGET(port_id); + sc->sc_mem_win_attr = MV_WIN_PCIE_MEM_ATTR(port_id); + sc->sc_io_win_attr = MV_WIN_PCIE_IO_ATTR(port_id); + sc->sc_skip_enable_procedure = 1; + } else if (ofw_bus_node_is_compatible(parnode, "marvell,armada-370-pcie")) { + sc->sc_type = MV_TYPE_PCIE; + sc->sc_win_target = MV_WIN_PCIE_TARGET_ARMADA38X(port_id); + sc->sc_mem_win_attr = MV_WIN_PCIE_MEM_ATTR_ARMADA38X(port_id); + sc->sc_io_win_attr = MV_WIN_PCIE_IO_ATTR_ARMADA38X(port_id); + sc->sc_enable_find_root_slot = 1; + } else if (ofw_bus_node_is_compatible(node, "mrvl,pci")) { + sc->sc_type = MV_TYPE_PCI; + sc->sc_win_target = MV_WIN_PCI_TARGET; + sc->sc_mem_win_attr = MV_WIN_PCI_MEM_ATTR; + sc->sc_io_win_attr = MV_WIN_PCI_IO_ATTR; + } else + return (ENXIO); + + /* + * Retrieve our mem-mapped registers range. + */ + sc->sc_rid = 0; + sc->sc_res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &sc->sc_rid, + RF_ACTIVE); + if (sc->sc_res == NULL) { + device_printf(self, "could not map memory\n"); + return (ENXIO); + } + sc->sc_bst = rman_get_bustag(sc->sc_res); + sc->sc_bsh = rman_get_bushandle(sc->sc_res); + + val = bus_space_read_4(sc->sc_bst, sc->sc_bsh, PCIE_REG_CONTROL); + sc->sc_mode = (val & PCIE_CONTROL_ROOT_CMPLX ? MV_MODE_ROOT : + MV_MODE_ENDPOINT); + + /* + * Get PCI interrupt info. + */ + if (sc->sc_mode == MV_MODE_ROOT) + ofw_bus_setup_iinfo(node, &sc->sc_pci_iinfo, sizeof(pcell_t)); + + /* + * Configure decode windows for PCI(E) access. + */ + if (mv_pcib_decode_win(node, sc) != 0) + return (ENXIO); + + mv_pcib_hw_cfginit(); + + /* + * Enable PCIE device. + */ + mv_pcib_enable(sc, port_id); + + /* + * Memory management. + */ + err = mv_pcib_mem_init(sc); + if (err) + return (err); + + /* + * Preliminary bus enumeration to find first linked devices and set + * appropriate bus number from which should start the actual enumeration + */ + for (bus = 0; bus < PCI_BUSMAX; bus++) { + for (devfn = 0; devfn < mv_pcib_maxslots(self); devfn++) { + reg0 = mv_pcib_read_config(self, bus, devfn, devfn & 0x7, 0x0, 4); + if (reg0 == (~0U)) + continue; /* no device */ + else { + sc->sc_busnr = bus; /* update bus number */ + break; + } + } + } + + if (sc->sc_mode == MV_MODE_ROOT) { + err = mv_pcib_init(sc, sc->sc_busnr, + mv_pcib_maxslots(sc->sc_dev)); + if (err) + goto error; + + device_add_child(self, "pci", DEVICE_UNIT_ANY); + } else { + sc->sc_devnr = 1; + bus_space_write_4(sc->sc_bst, sc->sc_bsh, + PCIE_REG_STATUS, 1 << PCIE_STATUS_DEV_OFFS); + device_add_child(self, "pci_ep", DEVICE_UNIT_ANY); + } + + mtx_init(&sc->sc_msi_mtx, "msi_mtx", NULL, MTX_DEF); + bus_attach_children(self); + return (0); + +error: + /* XXX SYS_RES_ should be released here */ + rman_fini(&sc->sc_mem_rman); + rman_fini(&sc->sc_io_rman); + + return (err); +} + +static void +mv_pcib_enable(struct mv_pcib_softc *sc, uint32_t unit) +{ + uint32_t val; + int timeout; + + if (sc->sc_skip_enable_procedure) + goto pcib_enable_root_mode; + + /* + * Check if PCIE device is enabled. + */ + if ((sc->sc_skip_enable_procedure == 0) && + (read_cpu_ctrl(CPU_CONTROL) & CPU_CONTROL_PCIE_DISABLE(unit))) { + write_cpu_ctrl(CPU_CONTROL, read_cpu_ctrl(CPU_CONTROL) & + ~(CPU_CONTROL_PCIE_DISABLE(unit))); + + timeout = PCIE_LINK_TIMEOUT; + val = bus_space_read_4(sc->sc_bst, sc->sc_bsh, + PCIE_REG_STATUS); + while (((val & PCIE_STATUS_LINK_DOWN) == 1) && (timeout > 0)) { + DELAY(1000); + timeout -= 1000; + val = bus_space_read_4(sc->sc_bst, sc->sc_bsh, + PCIE_REG_STATUS); + } + } + +pcib_enable_root_mode: + if (sc->sc_mode == MV_MODE_ROOT) { + /* + * Enable PCI bridge. + */ + val = bus_space_read_4(sc->sc_bst, sc->sc_bsh, PCIR_COMMAND); + val |= PCIM_CMD_SERRESPEN | PCIM_CMD_BUSMASTEREN | + PCIM_CMD_MEMEN | PCIM_CMD_PORTEN; + bus_space_write_4(sc->sc_bst, sc->sc_bsh, PCIR_COMMAND, val); + } +} + +static int +mv_pcib_mem_init(struct mv_pcib_softc *sc) +{ + int err; + + /* + * Memory management. + */ + sc->sc_mem_rman.rm_type = RMAN_ARRAY; + err = rman_init(&sc->sc_mem_rman); + if (err) + return (err); + + sc->sc_io_rman.rm_type = RMAN_ARRAY; + err = rman_init(&sc->sc_io_rman); + if (err) { + rman_fini(&sc->sc_mem_rman); + return (err); + } + + err = rman_manage_region(&sc->sc_mem_rman, sc->sc_mem_base, + sc->sc_mem_base + sc->sc_mem_size - 1); + if (err) + goto error; + + err = rman_manage_region(&sc->sc_io_rman, sc->sc_io_base, + sc->sc_io_base + sc->sc_io_size - 1); + if (err) + goto error; + + return (0); + +error: + rman_fini(&sc->sc_mem_rman); + rman_fini(&sc->sc_io_rman); + + return (err); +} + +static inline uint32_t +pcib_bit_get(uint32_t *map, uint32_t bit) +{ + uint32_t n = bit / BITS_PER_UINT32; + + bit = bit % BITS_PER_UINT32; + return (map[n] & (1 << bit)); +} + +static inline void +pcib_bit_set(uint32_t *map, uint32_t bit) +{ + uint32_t n = bit / BITS_PER_UINT32; + + bit = bit % BITS_PER_UINT32; + map[n] |= (1 << bit); +} + +static inline uint32_t +pcib_map_check(uint32_t *map, uint32_t start, uint32_t bits) +{ + uint32_t i; + + for (i = start; i < start + bits; i++) + if (pcib_bit_get(map, i)) + return (0); + + return (1); +} + +static inline void +pcib_map_set(uint32_t *map, uint32_t start, uint32_t bits) +{ + uint32_t i; + + for (i = start; i < start + bits; i++) + pcib_bit_set(map, i); +} + +/* + * The idea of this allocator is taken from ARM No-Cache memory + * management code (sys/arm/arm/vm_machdep.c). + */ +static bus_addr_t +pcib_alloc(struct mv_pcib_softc *sc, uint32_t smask) +{ + uint32_t bits, bits_limit, i, *map, min_alloc, size; + bus_addr_t addr = 0; + bus_addr_t base; + + if (smask & 1) { + base = sc->sc_io_base; + min_alloc = PCI_MIN_IO_ALLOC; + bits_limit = sc->sc_io_size / min_alloc; + map = sc->sc_io_map; + smask &= ~0x3; + } else { + base = sc->sc_mem_base; + min_alloc = PCI_MIN_MEM_ALLOC; + bits_limit = sc->sc_mem_size / min_alloc; + map = sc->sc_mem_map; + smask &= ~0xF; + } + + size = ~smask + 1; + bits = size / min_alloc; + + for (i = 0; i + bits <= bits_limit; i += bits) + if (pcib_map_check(map, i, bits)) { + pcib_map_set(map, i, bits); + addr = base + (i * min_alloc); + return (addr); + } + + return (addr); +} + +static int +mv_pcib_init_bar(struct mv_pcib_softc *sc, int bus, int slot, int func, + int barno) +{ + uint32_t addr, bar; + int reg, width; + + reg = PCIR_BAR(barno); + + /* + * Need to init the BAR register with 0xffffffff before correct + * value can be read. + */ + mv_pcib_write_config(sc->sc_dev, bus, slot, func, reg, ~0, 4); + bar = mv_pcib_read_config(sc->sc_dev, bus, slot, func, reg, 4); + if (bar == 0) + return (1); + + /* Calculate BAR size: 64 or 32 bit (in 32-bit units) */ + width = ((bar & 7) == 4) ? 2 : 1; + + addr = pcib_alloc(sc, bar); + if (!addr) + return (-1); + + if (bootverbose) + printf("PCI %u:%u:%u: reg %x: smask=%08x: addr=%08x\n", + bus, slot, func, reg, bar, addr); + + mv_pcib_write_config(sc->sc_dev, bus, slot, func, reg, addr, 4); + if (width == 2) + mv_pcib_write_config(sc->sc_dev, bus, slot, func, reg + 4, + 0, 4); + + return (width); +} + +static void +mv_pcib_init_bridge(struct mv_pcib_softc *sc, int bus, int slot, int func) +{ + bus_addr_t io_base, mem_base; + uint32_t io_limit, mem_limit; + int secbus; + + io_base = sc->sc_io_base; + io_limit = io_base + sc->sc_io_size - 1; + mem_base = sc->sc_mem_base; + mem_limit = mem_base + sc->sc_mem_size - 1; + + /* Configure I/O decode registers */ + mv_pcib_write_config(sc->sc_dev, bus, slot, func, PCIR_IOBASEL_1, + io_base >> 8, 1); + mv_pcib_write_config(sc->sc_dev, bus, slot, func, PCIR_IOBASEH_1, + io_base >> 16, 2); + mv_pcib_write_config(sc->sc_dev, bus, slot, func, PCIR_IOLIMITL_1, + io_limit >> 8, 1); + mv_pcib_write_config(sc->sc_dev, bus, slot, func, PCIR_IOLIMITH_1, + io_limit >> 16, 2); + + /* Configure memory decode registers */ + mv_pcib_write_config(sc->sc_dev, bus, slot, func, PCIR_MEMBASE_1, + mem_base >> 16, 2); + mv_pcib_write_config(sc->sc_dev, bus, slot, func, PCIR_MEMLIMIT_1, + mem_limit >> 16, 2); + + /* Disable memory prefetch decode */ + mv_pcib_write_config(sc->sc_dev, bus, slot, func, PCIR_PMBASEL_1, + 0x10, 2); + mv_pcib_write_config(sc->sc_dev, bus, slot, func, PCIR_PMBASEH_1, + 0x0, 4); + mv_pcib_write_config(sc->sc_dev, bus, slot, func, PCIR_PMLIMITL_1, + 0xF, 2); + mv_pcib_write_config(sc->sc_dev, bus, slot, func, PCIR_PMLIMITH_1, + 0x0, 4); + + secbus = mv_pcib_read_config(sc->sc_dev, bus, slot, func, + PCIR_SECBUS_1, 1); + + /* Configure buses behind the bridge */ + mv_pcib_init(sc, secbus, PCI_SLOTMAX); +} + +static int +mv_pcib_init(struct mv_pcib_softc *sc, int bus, int maxslot) +{ + int slot, func, maxfunc, error; + uint8_t hdrtype, command, class, subclass; + + for (slot = 0; slot <= maxslot; slot++) { + maxfunc = 0; + for (func = 0; func <= maxfunc; func++) { + hdrtype = mv_pcib_read_config(sc->sc_dev, bus, slot, + func, PCIR_HDRTYPE, 1); + + if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE) + continue; + + if (func == 0 && (hdrtype & PCIM_MFDEV)) + maxfunc = PCI_FUNCMAX; + + command = mv_pcib_read_config(sc->sc_dev, bus, slot, + func, PCIR_COMMAND, 1); + command &= ~(PCIM_CMD_MEMEN | PCIM_CMD_PORTEN); + mv_pcib_write_config(sc->sc_dev, bus, slot, func, + PCIR_COMMAND, command, 1); + + error = mv_pcib_init_all_bars(sc, bus, slot, func, + hdrtype); + + if (error) + return (error); + + command |= PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN | + PCIM_CMD_PORTEN; + mv_pcib_write_config(sc->sc_dev, bus, slot, func, + PCIR_COMMAND, command, 1); + + /* Handle PCI-PCI bridges */ + class = mv_pcib_read_config(sc->sc_dev, bus, slot, + func, PCIR_CLASS, 1); + subclass = mv_pcib_read_config(sc->sc_dev, bus, slot, + func, PCIR_SUBCLASS, 1); + + if (class != PCIC_BRIDGE || + subclass != PCIS_BRIDGE_PCI) + continue; + + mv_pcib_init_bridge(sc, bus, slot, func); + } + } + + /* Enable all ABCD interrupts */ + pcib_write_irq_mask(sc, (0xF << 24)); + + return (0); +} + +static int +mv_pcib_init_all_bars(struct mv_pcib_softc *sc, int bus, int slot, + int func, int hdrtype) +{ + int maxbar, bar, i; + + maxbar = (hdrtype & PCIM_HDRTYPE) ? 0 : 6; + bar = 0; + + /* Program the base address registers */ + while (bar < maxbar) { + i = mv_pcib_init_bar(sc, bus, slot, func, bar); + bar += i; + if (i < 0) { + device_printf(sc->sc_dev, + "PCI IO/Memory space exhausted\n"); + return (ENOMEM); + } + } + + return (0); +} + +static struct rman * +mv_pcib_get_rman(device_t dev, int type, u_int flags) +{ + struct mv_pcib_softc *sc = device_get_softc(dev); + + switch (type) { + case SYS_RES_IOPORT: + return (&sc->sc_io_rman); + case SYS_RES_MEMORY: + return (&sc->sc_mem_rman); + default: + return (NULL); + } +} + +static struct resource * +mv_pcib_alloc_resource(device_t dev, device_t child, int type, int *rid, + rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) +{ + struct mv_pcib_softc *sc = device_get_softc(dev); + + switch (type) { + case SYS_RES_IOPORT: + case SYS_RES_MEMORY: + break; + case PCI_RES_BUS: + return (pci_domain_alloc_bus(sc->ap_segment, child, rid, start, + end, count, flags)); + default: + return (BUS_ALLOC_RESOURCE(device_get_parent(dev), dev, + type, rid, start, end, count, flags)); + } + + if (RMAN_IS_DEFAULT_RANGE(start, end)) { + start = sc->sc_mem_base; + end = sc->sc_mem_base + sc->sc_mem_size - 1; + count = sc->sc_mem_size; + } + + if ((start < sc->sc_mem_base) || (start + count - 1 != end) || + (end > sc->sc_mem_base + sc->sc_mem_size - 1)) + return (NULL); + + return (bus_generic_rman_alloc_resource(dev, child, type, rid, + start, end, count, flags)); +} + +static int +mv_pcib_adjust_resource(device_t dev, device_t child, + struct resource *r, rman_res_t start, rman_res_t end) +{ + struct mv_pcib_softc *sc = device_get_softc(dev); + + switch (rman_get_type(r)) { + case SYS_RES_IOPORT: + case SYS_RES_MEMORY: + return (bus_generic_rman_adjust_resource(dev, child, r, start, + end)); + case PCI_RES_BUS: + return (pci_domain_adjust_bus(sc->ap_segment, child, r, start, + end)); + default: + return (bus_generic_adjust_resource(dev, child, r, start, end)); + } +} + +static int +mv_pcib_release_resource(device_t dev, device_t child, struct resource *res) +{ + struct mv_pcib_softc *sc = device_get_softc(dev); + + switch (rman_get_type(res)) { + case SYS_RES_IOPORT: + case SYS_RES_MEMORY: + return (bus_generic_rman_release_resource(dev, child, res)); + case PCI_RES_BUS: + return (pci_domain_release_bus(sc->ap_segment, child, res)); + default: + return (bus_generic_release_resource(dev, child, res)); + } +} + +static int +mv_pcib_activate_resource(device_t dev, device_t child, struct resource *r) +{ + struct mv_pcib_softc *sc = device_get_softc(dev); + + switch (rman_get_type(r)) { + case SYS_RES_IOPORT: + case SYS_RES_MEMORY: + return (bus_generic_rman_activate_resource(dev, child, r)); + case PCI_RES_BUS: + return (pci_domain_activate_bus(sc->ap_segment, child, r)); + default: + return (bus_generic_activate_resource(dev, child, r)); + } +} + +static int +mv_pcib_deactivate_resource(device_t dev, device_t child, struct resource *r) +{ + struct mv_pcib_softc *sc = device_get_softc(dev); + + switch (rman_get_type(r)) { + case SYS_RES_IOPORT: + case SYS_RES_MEMORY: + return (bus_generic_rman_deactivate_resource(dev, child, r)); + case PCI_RES_BUS: + return (pci_domain_deactivate_bus(sc->ap_segment, child, r)); + default: + return (bus_generic_deactivate_resource(dev, child, r)); + } +} + +static int +mv_pcib_map_resource(device_t dev, device_t child, struct resource *r, + struct resource_map_request *argsp, struct resource_map *map) +{ + struct resource_map_request args; + rman_res_t length, start; + int error; + + /* Resources must be active to be mapped. */ + if (!(rman_get_flags(r) & RF_ACTIVE)) + return (ENXIO); + + /* Mappings are only supported on I/O and memory resources. */ + switch (rman_get_type(r)) { + case SYS_RES_IOPORT: + case SYS_RES_MEMORY: + break; + default: + return (EINVAL); + } + + resource_init_map_request(&args); + error = resource_validate_map_request(r, argsp, &args, &start, &length); + if (error) + return (error); + + map->r_bustag = fdtbus_bs_tag; + map->r_bushandle = start; + map->r_size = length; + return (0); +} + +static int +mv_pcib_unmap_resource(device_t dev, device_t child, struct resource *r, + struct resource_map *map) +{ + switch (rman_get_type(r)) { + case SYS_RES_IOPORT: + case SYS_RES_MEMORY: + return (0); + default: + return (EINVAL); + } +} + +static int +mv_pcib_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) +{ + struct mv_pcib_softc *sc = device_get_softc(dev); + + switch (which) { + case PCIB_IVAR_BUS: + *result = sc->sc_busnr; + return (0); + case PCIB_IVAR_DOMAIN: + *result = device_get_unit(dev); + return (0); + } + + return (ENOENT); +} + +static int +mv_pcib_write_ivar(device_t dev, device_t child, int which, uintptr_t value) +{ + struct mv_pcib_softc *sc = device_get_softc(dev); + + switch (which) { + case PCIB_IVAR_BUS: + sc->sc_busnr = value; + return (0); + } + + return (ENOENT); +} + +static inline void +pcib_write_irq_mask(struct mv_pcib_softc *sc, uint32_t mask) +{ + + if (sc->sc_type != MV_TYPE_PCIE) + return; + + bus_space_write_4(sc->sc_bst, sc->sc_bsh, PCIE_REG_IRQ_MASK, mask); +} + +static void +mv_pcib_hw_cfginit(void) +{ + static int opened = 0; + + if (opened) + return; + + mtx_init(&pcicfg_mtx, "pcicfg", NULL, MTX_SPIN); + opened = 1; +} + +static uint32_t +mv_pcib_hw_cfgread(struct mv_pcib_softc *sc, u_int bus, u_int slot, + u_int func, u_int reg, int bytes) +{ + uint32_t addr, data, ca, cd; + + ca = (sc->sc_type != MV_TYPE_PCI) ? + PCIE_REG_CFG_ADDR : PCI_REG_CFG_ADDR; + cd = (sc->sc_type != MV_TYPE_PCI) ? + PCIE_REG_CFG_DATA : PCI_REG_CFG_DATA; + addr = PCI_CFG_ENA | PCI_CFG_BUS(bus) | PCI_CFG_DEV(slot) | + PCI_CFG_FUN(func) | PCI_CFG_PCIE_REG(reg); + + mtx_lock_spin(&pcicfg_mtx); + bus_space_write_4(sc->sc_bst, sc->sc_bsh, ca, addr); + + data = ~0; + switch (bytes) { + case 1: + data = bus_space_read_1(sc->sc_bst, sc->sc_bsh, + cd + (reg & 3)); + break; + case 2: + data = le16toh(bus_space_read_2(sc->sc_bst, sc->sc_bsh, + cd + (reg & 2))); + break; + case 4: + data = le32toh(bus_space_read_4(sc->sc_bst, sc->sc_bsh, + cd)); + break; + } + mtx_unlock_spin(&pcicfg_mtx); + return (data); +} + +static void +mv_pcib_hw_cfgwrite(struct mv_pcib_softc *sc, u_int bus, u_int slot, + u_int func, u_int reg, uint32_t data, int bytes) +{ + uint32_t addr, ca, cd; + + ca = (sc->sc_type != MV_TYPE_PCI) ? + PCIE_REG_CFG_ADDR : PCI_REG_CFG_ADDR; + cd = (sc->sc_type != MV_TYPE_PCI) ? + PCIE_REG_CFG_DATA : PCI_REG_CFG_DATA; + addr = PCI_CFG_ENA | PCI_CFG_BUS(bus) | PCI_CFG_DEV(slot) | + PCI_CFG_FUN(func) | PCI_CFG_PCIE_REG(reg); + + mtx_lock_spin(&pcicfg_mtx); + bus_space_write_4(sc->sc_bst, sc->sc_bsh, ca, addr); + + switch (bytes) { + case 1: + bus_space_write_1(sc->sc_bst, sc->sc_bsh, + cd + (reg & 3), data); + break; + case 2: + bus_space_write_2(sc->sc_bst, sc->sc_bsh, + cd + (reg & 2), htole16(data)); + break; + case 4: + bus_space_write_4(sc->sc_bst, sc->sc_bsh, + cd, htole32(data)); + break; + } + mtx_unlock_spin(&pcicfg_mtx); +} + +static int +mv_pcib_maxslots(device_t dev) +{ + struct mv_pcib_softc *sc = device_get_softc(dev); + + return ((sc->sc_type != MV_TYPE_PCI) ? 1 : PCI_SLOTMAX); +} + +static int +mv_pcib_root_slot(device_t dev, u_int bus, u_int slot, u_int func) +{ + struct mv_pcib_softc *sc = device_get_softc(dev); + uint32_t vendor, device; + + /* On platforms other than Armada38x, root link is always at slot 0 */ + if (!sc->sc_enable_find_root_slot) + return (slot == 0); + + vendor = mv_pcib_hw_cfgread(sc, bus, slot, func, PCIR_VENDOR, + PCIR_VENDOR_LENGTH); + device = mv_pcib_hw_cfgread(sc, bus, slot, func, PCIR_DEVICE, + PCIR_DEVICE_LENGTH) & MV_DEV_FAMILY_MASK; + + return (vendor == PCI_VENDORID_MRVL && device == MV_DEV_ARMADA38X); +} + +static uint32_t +mv_pcib_read_config(device_t dev, u_int bus, u_int slot, u_int func, + u_int reg, int bytes) +{ + struct mv_pcib_softc *sc = device_get_softc(dev); + + /* Return ~0 if link is inactive or trying to read from Root */ + if ((bus_space_read_4(sc->sc_bst, sc->sc_bsh, PCIE_REG_STATUS) & + PCIE_STATUS_LINK_DOWN) || mv_pcib_root_slot(dev, bus, slot, func)) + return (~0U); + + return (mv_pcib_hw_cfgread(sc, bus, slot, func, reg, bytes)); +} + +static void +mv_pcib_write_config(device_t dev, u_int bus, u_int slot, u_int func, + u_int reg, uint32_t val, int bytes) +{ + struct mv_pcib_softc *sc = device_get_softc(dev); + + /* Return if link is inactive or trying to write to Root */ + if ((bus_space_read_4(sc->sc_bst, sc->sc_bsh, PCIE_REG_STATUS) & + PCIE_STATUS_LINK_DOWN) || mv_pcib_root_slot(dev, bus, slot, func)) + return; + + mv_pcib_hw_cfgwrite(sc, bus, slot, func, reg, val, bytes); +} + +static int +mv_pcib_route_interrupt(device_t bus, device_t dev, int pin) +{ + struct mv_pcib_softc *sc; + struct ofw_pci_register reg; + uint32_t pintr, mintr[4]; + int icells; + phandle_t iparent; + + sc = device_get_softc(bus); + pintr = pin; + + /* Fabricate imap information in case this isn't an OFW device */ + bzero(®, sizeof(reg)); + reg.phys_hi = (pci_get_bus(dev) << OFW_PCI_PHYS_HI_BUSSHIFT) | + (pci_get_slot(dev) << OFW_PCI_PHYS_HI_DEVICESHIFT) | + (pci_get_function(dev) << OFW_PCI_PHYS_HI_FUNCTIONSHIFT); + + icells = ofw_bus_lookup_imap(ofw_bus_get_node(dev), &sc->sc_pci_iinfo, + ®, sizeof(reg), &pintr, sizeof(pintr), mintr, sizeof(mintr), + &iparent); + if (icells > 0) + return (ofw_bus_map_intr(dev, iparent, icells, mintr)); + + /* Maybe it's a real interrupt, not an intpin */ + if (pin > 4) + return (pin); + + device_printf(bus, "could not route pin %d for device %d.%d\n", + pin, pci_get_slot(dev), pci_get_function(dev)); + return (PCI_INVALID_IRQ); +} + +static int +mv_pcib_decode_win(phandle_t node, struct mv_pcib_softc *sc) +{ + struct mv_pci_range io_space, mem_space; + device_t dev; + int error; + + dev = sc->sc_dev; + + if ((error = mv_pci_ranges(node, &io_space, &mem_space)) != 0) { + device_printf(dev, "could not retrieve 'ranges' data\n"); + return (error); + } + + /* Configure CPU decoding windows */ + error = decode_win_cpu_set(sc->sc_win_target, + sc->sc_io_win_attr, io_space.base_parent, io_space.len, ~0); + if (error < 0) { + device_printf(dev, "could not set up CPU decode " + "window for PCI IO\n"); + return (ENXIO); + } + error = decode_win_cpu_set(sc->sc_win_target, + sc->sc_mem_win_attr, mem_space.base_parent, mem_space.len, + mem_space.base_parent); + if (error < 0) { + device_printf(dev, "could not set up CPU decode " + "windows for PCI MEM\n"); + return (ENXIO); + } + + sc->sc_io_base = io_space.base_parent; + sc->sc_io_size = io_space.len; + + sc->sc_mem_base = mem_space.base_parent; + sc->sc_mem_size = mem_space.len; + + return (0); +} + +static int +mv_pcib_map_msi(device_t dev, device_t child, int irq, uint64_t *addr, + uint32_t *data) +{ + struct mv_pcib_softc *sc; + + sc = device_get_softc(dev); + if (!sc->sc_msi_supported) + return (ENOTSUP); + + irq = irq - MSI_IRQ; + + /* validate parameters */ + if (isclr(&sc->sc_msi_bitmap, irq)) { + device_printf(dev, "invalid MSI 0x%x\n", irq); + return (EINVAL); + } + + mv_msi_data(irq, addr, data); + + debugf("%s: irq: %d addr: %jx data: %x\n", + __func__, irq, *addr, *data); + + return (0); +} + +static int +mv_pcib_alloc_msi(device_t dev, device_t child, int count, + int maxcount __unused, int *irqs) +{ + struct mv_pcib_softc *sc; + u_int start = 0, i; + + sc = device_get_softc(dev); + if (!sc->sc_msi_supported) + return (ENOTSUP); + + if (powerof2(count) == 0 || count > MSI_IRQ_NUM) + return (EINVAL); + + mtx_lock(&sc->sc_msi_mtx); + + for (start = 0; (start + count) < MSI_IRQ_NUM; start++) { + for (i = start; i < start + count; i++) { + if (isset(&sc->sc_msi_bitmap, i)) + break; + } + if (i == start + count) + break; + } + + if ((start + count) == MSI_IRQ_NUM) { + mtx_unlock(&sc->sc_msi_mtx); + return (ENXIO); + } + + for (i = start; i < start + count; i++) { + setbit(&sc->sc_msi_bitmap, i); + *irqs++ = MSI_IRQ + i; + } + debugf("%s: start: %x count: %x\n", __func__, start, count); + + mtx_unlock(&sc->sc_msi_mtx); + return (0); +} + +static int +mv_pcib_release_msi(device_t dev, device_t child, int count, int *irqs) +{ + struct mv_pcib_softc *sc; + u_int i; + + sc = device_get_softc(dev); + if(!sc->sc_msi_supported) + return (ENOTSUP); + + mtx_lock(&sc->sc_msi_mtx); + + for (i = 0; i < count; i++) + clrbit(&sc->sc_msi_bitmap, irqs[i] - MSI_IRQ); + + mtx_unlock(&sc->sc_msi_mtx); + return (0); +} |