/* $NetBSD: apple_pcie.c,v 1.6 2022/04/27 08:03:06 skrll Exp $ */ /* $OpenBSD: aplpcie.c,v 1.13 2022/04/06 18:59:26 naddy Exp $ */ /*- * Copyright (c) 2021 Jared McNeill * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Copyright (c) 2021 Mark Kettenis * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include __KERNEL_RCSID(0, "$NetBSD: apple_pcie.c,v 1.6 2022/04/27 08:03:06 skrll Exp $"); #include #include #include #include #include #include #include #include #include #include #include #include #include #define PCIE_CORE_LANE_CONF(port) (0x84000 + (port) * 0x4000) #define PCIE_CORE_LANE_CONF_REFCLK0REQ __BIT(0) #define PCIE_CORE_LANE_CONF_REFCLK1REQ __BIT(1) #define PCIE_CORE_LANE_CONF_REFCLK0ACK __BIT(2) #define PCIE_CORE_LANE_CONF_REFCLK1ACK __BIT(3) #define PCIE_CORE_LANE_CONF_REFCLK0EN __BIT(9) #define PCIE_CORE_LANE_CONF_REFCLK1EN __BIT(10) #define PCIE_CORE_LANE_CTRL(port) (0x84004 + (port) * 0x4000) #define PCIE_CORE_LANE_CTRL_CFGACC __BIT(15) #define PCIE_PORT_LTSSM_CTRL 0x0080 #define PCIE_PORT_LTSSM_CTRL_START __BIT(0) #define PCIE_PORT_MSI_CTRL 0x0124 #define PCIE_PORT_MSI_CTRL_EN __BIT(0) #define PCIE_PORT_MSI_CTRL_32 __SHIFTIN(5U, __BITS(7,4)) #define PCIE_PORT_MSI_REMAP 0x0128 #define PCIE_PORT_MSI_DOORBELL 0x0168 #define PCIE_PORT_LINK_STAT 0x0208 #define PCIE_PORT_LINK_STAT_UP __BIT(0) #define PCIE_PORT_APPCLK 0x0800 #define PCIE_PORT_APPCLK_EN __BIT(0) #define PCIE_PORT_APPCLK_CGDIS __BIT(8) #define PCIE_PORT_STAT 0x0804 #define PCIE_PORT_STAT_READY __BIT(0) #define PCIE_PORT_REFCLK 0x0810 #define PCIE_PORT_REFCLK_EN __BIT(0) #define PCIE_PORT_REFCLK_CGDIS __BIT(8) #define PCIE_PORT_PERST 0x0814 #define PCIE_PORT_PERST_DIS __BIT(0) extern struct bus_space arm_generic_bs_tag; struct apple_pcie_softc { struct pcihost_softc sc_pcihost; bus_space_tag_t sc_rc_bst; bus_space_handle_t sc_rc_bsh; int sc_phandle; struct arm_pci_msi sc_msi; u_int sc_msi_start; u_int sc_nmsi; struct pci_attach_args **sc_msi_pa; void **sc_msi_ih; uint64_t sc_msi_addr; }; static int apple_pcie_match(device_t, cfdata_t, void *); static void apple_pcie_attach(device_t, device_t, void *); static void apple_pcie_attach_hook(device_t, device_t, struct pcibus_attach_args *); static int apple_pcie_msi_init(struct apple_pcie_softc *); CFATTACH_DECL_NEW(apple_pcie, sizeof(struct apple_pcie_softc), apple_pcie_match, apple_pcie_attach, NULL, NULL); static const struct device_compatible_entry compat_data[] = { { .compat = "apple,pcie" }, DEVICE_COMPAT_EOL }; #define RREAD4(sc, reg) \ (bus_space_read_4((sc)->sc_rc_bst, (sc)->sc_rc_bsh, (reg))) #define RWRITE4(sc, reg, val) \ bus_space_write_4((sc)->sc_rc_bst, (sc)->sc_rc_bsh, (reg), (val)) #define RSET4(sc, reg, bits) \ RWRITE4((sc), (reg), RREAD4((sc), (reg)) | (bits)) #define RCLR4(sc, reg, bits) \ RWRITE4((sc), (reg), RREAD4((sc), (reg)) & ~(bits)) static void apple_pcie_setup_port(struct apple_pcie_softc *sc, int phandle) { const bus_space_tag_t bst = sc->sc_pcihost.sc_bst; const device_t dev = sc->sc_pcihost.sc_dev; const int parent = sc->sc_pcihost.sc_phandle; char regname[sizeof("portX")]; bus_space_handle_t bsh; bus_addr_t addr; bus_size_t size; int error; int timo; int len; const u_int *reg = fdtbus_get_prop(phandle, "reg", &len); if (len != 5 * sizeof(uint32_t)) { aprint_error(": couldn't get port number\n"); } u_int portno = __SHIFTOUT(be32toh(reg[0]), __BITS(13,11)); snprintf(regname, sizeof(regname), "port%u", portno); if (fdtbus_get_reg_byname(parent, regname, &addr, &size) != 0) { aprint_error(": couldn't get %s regs\n", regname); return; } error = bus_space_map(bst, addr, size, 0, &bsh); if (error != 0) { aprint_error(": couldn't map %s regs\n", regname); return; } #define PREAD4(bst, bsh, reg) \ bus_space_read_4((bst), (bsh), (reg)) #define PWRITE4(bst, bsh, reg, val) \ bus_space_write_4((bst), (bsh), (reg), (val)) #define PSET4(bst, bsh, reg, bits) \ PWRITE4((bst), (bsh), (reg), PREAD4((bst), (bsh), (reg)) | (bits)) #define PCLR4(bst, bsh, reg, bits) \ PWRITE4((bst), (bsh), (reg), PREAD4((bst), (bsh), (reg)) & ~(bits)) /* Doorbell address must be below 4GB */ KASSERT((sc->sc_msi_addr & ~0xffffffffUL) == 0); int pwren_gpiolen, reset_gpiolen; pwren_gpiolen = OF_getproplen(phandle, "pwren-gpios"); reset_gpiolen = OF_getproplen(phandle, "reset-gpios"); if (reset_gpiolen <= 0) return; /* * Set things up such that we can share the 32 available MSIs * across all ports. */ PWRITE4(bst, bsh, PCIE_PORT_MSI_CTRL, PCIE_PORT_MSI_CTRL_32 | PCIE_PORT_MSI_CTRL_EN); PWRITE4(bst, bsh, PCIE_PORT_MSI_REMAP, 0); PWRITE4(bst, bsh, PCIE_PORT_MSI_DOORBELL, __SHIFTOUT(sc->sc_msi_addr, __BITS(31, 0))); /* Check if the link is already up. */ uint32_t stat = PREAD4(bst, bsh, PCIE_PORT_LINK_STAT); if (stat & PCIE_PORT_LINK_STAT_UP) { aprint_debug_dev(dev, "link already up\n"); return; } aprint_debug_dev(dev, "bringing link up\n"); PSET4(bst, bsh, PCIE_PORT_APPCLK, PCIE_PORT_APPCLK_EN); struct fdtbus_gpio_pin *gpio_reset = fdtbus_gpio_acquire(phandle, "reset-gpios", GPIO_PIN_OUTPUT); if (gpio_reset == NULL) { aprint_debug_dev(dev, "failed to get reset-gpios\n"); return; } fdtbus_gpio_write(gpio_reset, 1); /* Power up the device if necessary. */ if (pwren_gpiolen > 0) { struct fdtbus_gpio_pin *gpio_pwren = fdtbus_gpio_acquire(phandle, "pwren-gpios", GPIO_PIN_OUTPUT); if (gpio_pwren == NULL) { aprint_debug_dev(dev, "failed to get pwren-gpios\n"); return; } fdtbus_gpio_write(gpio_pwren, 1); } /* Setup Refclk. */ RSET4(sc, PCIE_CORE_LANE_CTRL(portno), PCIE_CORE_LANE_CTRL_CFGACC); RSET4(sc, PCIE_CORE_LANE_CONF(portno), PCIE_CORE_LANE_CONF_REFCLK0REQ); for (timo = 500; timo > 0; timo--) { stat = RREAD4(sc, PCIE_CORE_LANE_CONF(portno)); if (stat & PCIE_CORE_LANE_CONF_REFCLK0ACK) break; delay(100); } RSET4(sc, PCIE_CORE_LANE_CONF(portno), PCIE_CORE_LANE_CONF_REFCLK1REQ); for (timo = 500; timo > 0; timo--) { stat = RREAD4(sc, PCIE_CORE_LANE_CONF(portno)); if (stat & PCIE_CORE_LANE_CONF_REFCLK1ACK) break; delay(100); } RCLR4(sc, PCIE_CORE_LANE_CTRL(portno), PCIE_CORE_LANE_CTRL_CFGACC); RSET4(sc, PCIE_CORE_LANE_CONF(portno), PCIE_CORE_LANE_CONF_REFCLK0EN | PCIE_CORE_LANE_CONF_REFCLK1EN); PSET4(bst, bsh, PCIE_PORT_REFCLK, PCIE_PORT_REFCLK_EN); /* * PERST# must remain asserted for at least 100us after the * reference clock becomes stable. But also has to remain * active at least 100ms after power up. */ if (pwren_gpiolen > 0) delay(100000); else delay(100); /* Deassert PERST#. */ PSET4(bst, bsh, PCIE_PORT_PERST, PCIE_PORT_PERST_DIS); fdtbus_gpio_write(gpio_reset, 0); for (timo = 2500; timo > 0; timo--) { stat = PREAD4(bst, bsh, PCIE_PORT_STAT); if (stat & PCIE_PORT_STAT_READY) break; delay(100); } if ((stat & PCIE_PORT_STAT_READY) == 0) { aprint_debug_dev(dev, "link up\n"); return; } PCLR4(bst, bsh, PCIE_PORT_REFCLK, PCIE_PORT_REFCLK_CGDIS); PCLR4(bst, bsh, PCIE_PORT_APPCLK, PCIE_PORT_APPCLK_CGDIS); /* Bring up the link. */ PWRITE4(bst, bsh, PCIE_PORT_LTSSM_CTRL, PCIE_PORT_LTSSM_CTRL_START); for (timo = 1000; timo > 0; timo--) { stat = PREAD4(bst, bsh, PCIE_PORT_LINK_STAT); if (stat & PCIE_PORT_LINK_STAT_UP) break; delay(100); } #undef PREAD4 #undef PWRITE4 #undef PCLR4 #undef PSET4 bus_space_unmap(bst, bsh, size); } static int apple_pcie_match(device_t parent, cfdata_t cf, void *aux) { struct fdt_attach_args * const faa = aux; return of_compatible_match(faa->faa_phandle, compat_data); } static void apple_pcie_attach(device_t parent, device_t self, void *aux) { struct apple_pcie_softc * const asc = device_private(self); struct pcihost_softc * const sc = &asc->sc_pcihost; struct fdt_attach_args * const faa = aux; const int phandle = faa->faa_phandle; bus_addr_t cs_addr, rc_addr; bus_size_t cs_size, rc_size; int error; if (fdtbus_get_reg_byname(phandle, "config", &cs_addr, &cs_size) != 0) { aprint_error(": couldn't get registers (%s)\n", "config"); return; } if (fdtbus_get_reg_byname(phandle, "rc", &rc_addr, &rc_size) != 0) { aprint_error(": couldn't get registers (%s)\n", "rc"); return; } sc->sc_dev = self; sc->sc_dmat = faa->faa_dmat; sc->sc_bst = asc->sc_rc_bst = faa->faa_bst; /* * Create a new bus tag for PCIe devices that does not inherit the * nonposted MMIO flag from the host controller. */ sc->sc_pci_bst = &arm_generic_bs_tag; sc->sc_phandle = phandle; error = bus_space_map(faa->faa_bst, cs_addr, cs_size, 0, &sc->sc_bsh); if (error) { aprint_error(": couldn't map registers (%s): %d\n", "config", error); return; } error = bus_space_map(asc->sc_rc_bst, rc_addr, rc_size, 0, &asc->sc_rc_bsh); if (error) { aprint_error(": couldn't map registers (%s): %d\n", "rc", error); return; } sc->sc_type = PCIHOST_ECAM; if (apple_pcie_msi_init(asc) == 0) { sc->sc_pci_flags |= PCI_FLAGS_MSI_OKAY; #if notyet sc->sc_pci_flags |= PCI_FLAGS_MSIX_OKAY; #endif } aprint_naive("\n"); aprint_normal(": Apple PCIe host controller\n"); for (int node = OF_child(phandle); node; node = OF_peer(node)) apple_pcie_setup_port(asc, node); /* * Must wait at least 100ms after link training completes * before sending a configuration request to a device * immediately below a port. */ delay(100000); pcihost_init(&sc->sc_pc, sc); sc->sc_pc.pc_attach_hook = apple_pcie_attach_hook; pcihost_init2(sc); } static void apple_pcie_attach_hook(device_t parent, device_t self, struct pcibus_attach_args *pba) { struct apple_pcie_softc *sc = pba->pba_pc->pc_conf_v; const int phandle = sc->sc_pcihost.sc_phandle; bus_dma_tag_t dmat; KASSERT(device_is_a(sc->sc_pcihost.sc_dev, "applepcie")); /* XXX this should be per-device, not per-bus */ const uint32_t rid = pba->pba_bus << 8; dmat = fdtbus_iommu_map_pci(phandle, rid, sc->sc_pcihost.sc_dmat); pba->pba_dmat = pba->pba_dmat64 = dmat; } static int apple_pcie_msi_alloc_msi(struct apple_pcie_softc *sc, int count, const struct pci_attach_args *pa) { struct pci_attach_args *new_pa; int msi, n; for (msi = 0; msi < sc->sc_nmsi; msi += n) { /* Look for first empty slot */ if (sc->sc_msi_pa[msi] != NULL) { /* skip the used entry */ n = 1; continue; } /* Now check that 'count' entries are also empty */ for (n = 1; n < count && msi + n < sc->sc_nmsi; n++) { if (sc->sc_msi_pa[msi + n] != NULL) { break; } } /* * If 'count' empty entries weren't found then the search * continues. */ if (n != count) continue; for (n = 0; n < count; n++) { new_pa = kmem_alloc(sizeof(*new_pa), KM_SLEEP); memcpy(new_pa, pa, sizeof(*new_pa)); sc->sc_msi_pa[msi + n] = new_pa; } return msi; } return -1; } static void apple_pcie_msi_free_msi(struct apple_pcie_softc *sc, int msi) { struct pci_attach_args *pa; pa = sc->sc_msi_pa[msi]; sc->sc_msi_pa[msi] = NULL; if (pa != NULL) { kmem_free(pa, sizeof(*pa)); } } static int apple_pcie_msi_available_msi(struct apple_pcie_softc *sc) { int msi, n; for (n = 0, msi = 0; msi < sc->sc_nmsi; msi++) { if (sc->sc_msi_pa[msi] == NULL) { n++; } } return n; } static void apple_pcie_msi_msi_enable(struct apple_pcie_softc *sc, int msi, int count) { const struct pci_attach_args *pa = sc->sc_msi_pa[msi]; pci_chipset_tag_t pc = pa->pa_pc; pcitag_t tag = pa->pa_tag; pcireg_t ctl; int off; if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL)) panic("apple_pcie_msi_msi_enable: device is not MSI-capable"); ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); ctl &= ~PCI_MSI_CTL_MSI_ENABLE; pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); ctl &= ~PCI_MSI_CTL_MME_MASK; ctl |= __SHIFTIN(ilog2(count), PCI_MSI_CTL_MME_MASK); pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); const uint64_t addr = sc->sc_msi_addr; const uint32_t data = msi; ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); if (ctl & PCI_MSI_CTL_64BIT_ADDR) { pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_LO, __SHIFTOUT(addr, __BITS(31, 0))); pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_HI, __SHIFTOUT(addr, __BITS(63, 32))); pci_conf_write(pc, tag, off + PCI_MSI_MDATA64, data); } else { pci_conf_write(pc, tag, off + PCI_MSI_MADDR, __SHIFTOUT(addr, __BITS(31, 0))); pci_conf_write(pc, tag, off + PCI_MSI_MDATA, data); } ctl |= PCI_MSI_CTL_MSI_ENABLE; pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); } static void apple_pcie_msi_msi_disable(struct apple_pcie_softc *sc, int msi) { const struct pci_attach_args *pa = sc->sc_msi_pa[msi]; pci_chipset_tag_t pc = pa->pa_pc; pcitag_t tag = pa->pa_tag; pcireg_t ctl; int off; if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL)) panic("apple_pcie_msi_msi_disable: device is not MSI-capable"); ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); ctl &= ~PCI_MSI_CTL_MSI_ENABLE; pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); } static void apple_pcie_msi_msix_enable(struct apple_pcie_softc *sc, int msi, int msix_vec, bus_space_tag_t bst, bus_space_handle_t bsh) { const struct pci_attach_args *pa = sc->sc_msi_pa[msi]; pci_chipset_tag_t pc = pa->pa_pc; pcitag_t tag = pa->pa_tag; pcireg_t ctl; uint32_t val; int off; if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL)) panic("apple_pcie_msi_msix_enable: device is not MSI-X-capable"); ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); ctl &= ~PCI_MSIX_CTL_ENABLE; pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); const uint64_t addr = sc->sc_msi_addr; const uint32_t data = msi; const uint64_t entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec; bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_LO, __SHIFTOUT(addr, __BITS(31, 0))); bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_HI, __SHIFTOUT(addr, __BITS(63, 32))); bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_DATA, data); val = bus_space_read_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL); val &= ~PCI_MSIX_VECTCTL_MASK; bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, val); ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); ctl |= PCI_MSIX_CTL_ENABLE; pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); } static void apple_pcie_msi_msix_disable(struct apple_pcie_softc *sc, int msi) { const struct pci_attach_args *pa = sc->sc_msi_pa[msi]; pci_chipset_tag_t pc = pa->pa_pc; pcitag_t tag = pa->pa_tag; pcireg_t ctl; int off; if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL)) panic("apple_pcie_msi_msix_disable: device is not MSI-X-capable"); ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); ctl &= ~PCI_MSIX_CTL_ENABLE; pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); } static pci_intr_handle_t * apple_pcie_msi_msi_alloc(struct arm_pci_msi *msi, int *count, const struct pci_attach_args *pa, bool exact) { struct apple_pcie_softc * const sc = msi->msi_priv; pci_intr_handle_t *vectors; int n, off; if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSI, &off, NULL)) return NULL; const int avail = apple_pcie_msi_available_msi(sc); if (avail == 0) return NULL; if (exact && *count > avail) return NULL; while (*count > avail) { (*count) >>= 1; } if (*count == 0) return NULL; const int msi_base = apple_pcie_msi_alloc_msi(sc, *count, pa); if (msi_base == -1) return NULL; vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP); for (n = 0; n < *count; n++) { const int msino = msi_base + n; vectors[n] = ARM_PCI_INTR_MSI | __SHIFTIN(msino, ARM_PCI_INTR_IRQ) | __SHIFTIN(n, ARM_PCI_INTR_MSI_VEC) | __SHIFTIN(msi->msi_id, ARM_PCI_INTR_FRAME); } apple_pcie_msi_msi_enable(sc, msi_base, *count); return vectors; } static pci_intr_handle_t * apple_pcie_msi_msix_alloc(struct arm_pci_msi *msi, u_int *table_indexes, int *count, const struct pci_attach_args *pa, bool exact) { struct apple_pcie_softc * const sc = msi->msi_priv; pci_intr_handle_t *vectors; bus_space_tag_t bst; bus_space_handle_t bsh; bus_size_t bsz; uint32_t table_offset, table_size; int n, off, bar, error; pcireg_t tbl; if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, &off, NULL)) return NULL; const int avail = apple_pcie_msi_available_msi(sc); if (exact && *count > avail) return NULL; while (*count > avail) { (*count) >>= 1; } if (*count == 0) return NULL; tbl = pci_conf_read(pa->pa_pc, pa->pa_tag, off + PCI_MSIX_TBLOFFSET); bar = PCI_BAR0 + (4 * (tbl & PCI_MSIX_TBLBIR_MASK)); table_offset = tbl & PCI_MSIX_TBLOFFSET_MASK; table_size = pci_msix_count(pa->pa_pc, pa->pa_tag) * PCI_MSIX_TABLE_ENTRY_SIZE; if (table_size == 0) return NULL; error = pci_mapreg_submap(pa, bar, pci_mapreg_type(pa->pa_pc, pa->pa_tag, bar), BUS_SPACE_MAP_LINEAR, roundup(table_size, PAGE_SIZE), table_offset, &bst, &bsh, NULL, &bsz); if (error) return NULL; const int msi_base = apple_pcie_msi_alloc_msi(sc, *count, pa); if (msi_base == -1) { bus_space_unmap(bst, bsh, bsz); return NULL; } vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP); for (n = 0; n < *count; n++) { const int msino = msi_base + n; const int msix_vec = table_indexes ? table_indexes[n] : n; vectors[msix_vec] = ARM_PCI_INTR_MSIX | __SHIFTIN(msino, ARM_PCI_INTR_IRQ) | __SHIFTIN(msix_vec, ARM_PCI_INTR_MSI_VEC) | __SHIFTIN(msi->msi_id, ARM_PCI_INTR_FRAME); apple_pcie_msi_msix_enable(sc, msino, msix_vec, bst, bsh); } bus_space_unmap(bst, bsh, bsz); return vectors; } static void * apple_pcie_msi_intr_establish(struct arm_pci_msi *msi, pci_intr_handle_t ih, int ipl, int (*func)(void *), void *arg, const char *xname) { struct apple_pcie_softc * const sc = msi->msi_priv; const int msino = __SHIFTOUT(ih, ARM_PCI_INTR_IRQ); const int mpsafe = (ih & ARM_PCI_INTR_MPSAFE) ? FDT_INTR_MPSAFE : 0; KASSERT(sc->sc_msi_ih[msino] == NULL); sc->sc_msi_ih[msino] = intr_establish_xname(sc->sc_msi_start + msino, ipl, IST_LEVEL | (mpsafe ? IST_MPSAFE : 0), func, arg, xname); return sc->sc_msi_ih[msino]; } static void apple_pcie_msi_intr_release(struct arm_pci_msi *msi, pci_intr_handle_t *pih, int count) { struct apple_pcie_softc * const sc = msi->msi_priv; int n; for (n = 0; n < count; n++) { const int msino = __SHIFTOUT(pih[n], ARM_PCI_INTR_IRQ); if (pih[n] & ARM_PCI_INTR_MSIX) apple_pcie_msi_msix_disable(sc, msino); if (pih[n] & ARM_PCI_INTR_MSI) apple_pcie_msi_msi_disable(sc, msino); apple_pcie_msi_free_msi(sc, msino); if (sc->sc_msi_ih[msino] != NULL) { intr_disestablish(sc->sc_msi_ih[msino]); sc->sc_msi_ih[msino] = NULL; } } } static int apple_pcie_msi_init(struct apple_pcie_softc *sc) { struct arm_pci_msi *msi = &sc->sc_msi; const int phandle = sc->sc_pcihost.sc_phandle; int len; const u_int *data = fdtbus_get_prop(phandle, "msi-ranges", &len); switch (len) { case 8: /* two cells: start and count */ sc->sc_msi_start = be32toh(data[0]); sc->sc_nmsi = be32toh(data[1]); break; case 20: /* 5 cells: xref, specifier (3 cells), and count */ sc->sc_msi_start = be32toh(data[2]); sc->sc_nmsi = be32toh(data[4]); break; default: aprint_error_dev(sc->sc_pcihost.sc_dev, "WARNING: bad msi-ranges property, MSI not enabled!\n"); return ENXIO; } sc->sc_msi_pa = kmem_zalloc(sizeof(*sc->sc_msi_pa) * sc->sc_nmsi, KM_SLEEP); sc->sc_msi_ih = kmem_zalloc(sizeof(*sc->sc_msi_ih) * sc->sc_nmsi, KM_SLEEP); if (of_getprop_uint64(phandle, "msi-doorbell", &sc->sc_msi_addr)) { sc->sc_msi_addr = 0xffff000ULL; } msi->msi_dev = sc->sc_pcihost.sc_dev; msi->msi_priv = sc; msi->msi_alloc = apple_pcie_msi_msi_alloc; msi->msix_alloc = apple_pcie_msi_msix_alloc; msi->msi_intr_establish = apple_pcie_msi_intr_establish; msi->msi_intr_release = apple_pcie_msi_intr_release; return arm_pci_msi_add(msi); }