diff options
Diffstat (limited to 'drivers/pci/controller')
42 files changed, 5233 insertions, 1111 deletions
diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 6012f3059acd..70e078238899 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -174,14 +174,14 @@ config PCIE_IPROC_MSI PCIe controller config PCIE_ALTERA - bool "Altera PCIe controller" + tristate "Altera PCIe controller" depends on ARM || NIOS2 || ARM64 || COMPILE_TEST help Say Y here if you want to enable PCIe controller support on Altera FPGA. config PCIE_ALTERA_MSI - bool "Altera PCIe MSI feature" + tristate "Altera PCIe MSI feature" depends on PCIE_ALTERA depends on PCI_MSI_IRQ_DOMAIN help @@ -267,6 +267,7 @@ config PCIE_TANGO_SMP8759 config VMD depends on PCI_MSI && X86_64 && SRCU + select X86_DEV_DMA_OPS tristate "Intel Volume Management Device Driver" ---help--- Adds support for the Intel Volume Management Device (VMD). VMD is a @@ -280,5 +281,12 @@ config VMD To compile this driver as a module, choose M here: the module will be called vmd. +config PCI_HYPERV_INTERFACE + tristate "Hyper-V PCI Interface" + depends on X86 && HYPERV && PCI_MSI && PCI_MSI_IRQ_DOMAIN && X86_64 + help + The Hyper-V PCI Interface is a helper driver allows other drivers to + have a common interface with the Hyper-V PCI frontend driver. + source "drivers/pci/controller/dwc/Kconfig" endmenu diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile index d56a507495c5..a2a22c9d91af 100644 --- a/drivers/pci/controller/Makefile +++ b/drivers/pci/controller/Makefile @@ -4,6 +4,7 @@ obj-$(CONFIG_PCIE_CADENCE_HOST) += pcie-cadence-host.o obj-$(CONFIG_PCIE_CADENCE_EP) += pcie-cadence-ep.o obj-$(CONFIG_PCI_FTPCI100) += pci-ftpci100.o obj-$(CONFIG_PCI_HYPERV) += pci-hyperv.o +obj-$(CONFIG_PCI_HYPERV_INTERFACE) += pci-hyperv-intf.o obj-$(CONFIG_PCI_MVEBU) += pci-mvebu.o obj-$(CONFIG_PCI_AARDVARK) += pci-aardvark.o obj-$(CONFIG_PCI_TEGRA) += pci-tegra.o diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig index 6ea74b1c0d94..0ba988b5b5bc 100644 --- a/drivers/pci/controller/dwc/Kconfig +++ b/drivers/pci/controller/dwc/Kconfig @@ -90,7 +90,7 @@ config PCI_EXYNOS config PCI_IMX6 bool "Freescale i.MX6/7/8 PCIe controller" - depends on SOC_IMX6Q || SOC_IMX7D || (ARM64 && ARCH_MXC) || COMPILE_TEST + depends on ARCH_MXC || COMPILE_TEST depends on PCI_MSI_IRQ_DOMAIN select PCIE_DW_HOST @@ -103,24 +103,57 @@ config PCIE_SPEAR13XX Say Y here if you want PCIe support on SPEAr13XX SoCs. config PCI_KEYSTONE - bool "TI Keystone PCIe controller" - depends on ARCH_KEYSTONE || (ARM && COMPILE_TEST) + bool + +config PCI_KEYSTONE_HOST + bool "PCI Keystone Host Mode" + depends on ARCH_KEYSTONE || ARCH_K3 || ((ARM || ARM64) && COMPILE_TEST) depends on PCI_MSI_IRQ_DOMAIN select PCIE_DW_HOST + select PCI_KEYSTONE + default y help - Say Y here if you want to enable PCI controller support on Keystone - SoCs. The PCI controller on Keystone is based on DesignWare hardware - and therefore the driver re-uses the DesignWare core functions to - implement the driver. + Enables support for the PCIe controller in the Keystone SoC to + work in host mode. The PCI controller on Keystone is based on + DesignWare hardware and therefore the driver re-uses the + DesignWare core functions to implement the driver. + +config PCI_KEYSTONE_EP + bool "PCI Keystone Endpoint Mode" + depends on ARCH_KEYSTONE || ARCH_K3 || ((ARM || ARM64) && COMPILE_TEST) + depends on PCI_ENDPOINT + select PCIE_DW_EP + select PCI_KEYSTONE + help + Enables support for the PCIe controller in the Keystone SoC to + work in endpoint mode. The PCI controller on Keystone is based + on DesignWare hardware and therefore the driver re-uses the + DesignWare core functions to implement the driver. config PCI_LAYERSCAPE - bool "Freescale Layerscape PCIe controller" + bool "Freescale Layerscape PCIe controller - Host mode" depends on OF && (ARM || ARCH_LAYERSCAPE || COMPILE_TEST) depends on PCI_MSI_IRQ_DOMAIN select MFD_SYSCON select PCIE_DW_HOST help - Say Y here if you want PCIe controller support on Layerscape SoCs. + Say Y here if you want to enable PCIe controller support on Layerscape + SoCs to work in Host mode. + This controller can work either as EP or RC. The RCW[HOST_AGT_PEX] + determines which PCIe controller works in EP mode and which PCIe + controller works in RC mode. + +config PCI_LAYERSCAPE_EP + bool "Freescale Layerscape PCIe controller - Endpoint mode" + depends on OF && (ARM || ARCH_LAYERSCAPE || COMPILE_TEST) + depends on PCI_ENDPOINT + select PCIE_DW_EP + help + Say Y here if you want to enable PCIe controller support on Layerscape + SoCs to work in Endpoint mode. + This controller can work either as EP or RC. The RCW[HOST_AGT_PEX] + determines which PCIe controller works in EP mode and which PCIe + controller works in RC mode. config PCI_HISI depends on OF && (ARM64 || COMPILE_TEST) @@ -203,6 +236,16 @@ config PCI_MESON and therefore the driver re-uses the DesignWare core functions to implement the driver. +config PCIE_TEGRA194 + tristate "NVIDIA Tegra194 (and later) PCIe controller" + depends on ARCH_TEGRA_194_SOC || COMPILE_TEST + depends on PCI_MSI_IRQ_DOMAIN + select PCIE_DW_HOST + select PHY_TEGRA194_P2U + help + Say Y here if you want support for DesignWare core based PCIe host + controller found in NVIDIA Tegra194 SoC. + config PCIE_UNIPHIER bool "Socionext UniPhier PCIe controllers" depends on ARCH_UNIPHIER || COMPILE_TEST @@ -213,4 +256,16 @@ config PCIE_UNIPHIER Say Y here if you want PCIe controller support on UniPhier SoCs. This driver supports LD20 and PXs3 SoCs. +config PCIE_AL + bool "Amazon Annapurna Labs PCIe controller" + depends on OF && (ARM64 || COMPILE_TEST) + depends on PCI_MSI_IRQ_DOMAIN + select PCIE_DW_HOST + help + Say Y here to enable support of the Amazon's Annapurna Labs PCIe + controller IP on Amazon SoCs. The PCIe controller uses the DesignWare + core plus Annapurna Labs proprietary hardware wrappers. This is + required only for DT-based platforms. ACPI platforms with the + Annapurna Labs PCIe controller don't need to enable this. + endmenu diff --git a/drivers/pci/controller/dwc/Makefile b/drivers/pci/controller/dwc/Makefile index b5f3b83cc2b3..69faff371f11 100644 --- a/drivers/pci/controller/dwc/Makefile +++ b/drivers/pci/controller/dwc/Makefile @@ -8,13 +8,15 @@ obj-$(CONFIG_PCI_EXYNOS) += pci-exynos.o obj-$(CONFIG_PCI_IMX6) += pci-imx6.o obj-$(CONFIG_PCIE_SPEAR13XX) += pcie-spear13xx.o obj-$(CONFIG_PCI_KEYSTONE) += pci-keystone.o -obj-$(CONFIG_PCI_LAYERSCAPE) += pci-layerscape.o pci-layerscape-ep.o +obj-$(CONFIG_PCI_LAYERSCAPE) += pci-layerscape.o +obj-$(CONFIG_PCI_LAYERSCAPE_EP) += pci-layerscape-ep.o obj-$(CONFIG_PCIE_QCOM) += pcie-qcom.o obj-$(CONFIG_PCIE_ARMADA_8K) += pcie-armada8k.o obj-$(CONFIG_PCIE_ARTPEC6) += pcie-artpec6.o obj-$(CONFIG_PCIE_KIRIN) += pcie-kirin.o obj-$(CONFIG_PCIE_HISI_STB) += pcie-histb.o obj-$(CONFIG_PCI_MESON) += pci-meson.o +obj-$(CONFIG_PCIE_TEGRA194) += pcie-tegra194.o obj-$(CONFIG_PCIE_UNIPHIER) += pcie-uniphier.o # The following drivers are for devices that use the generic ACPI @@ -28,5 +30,6 @@ obj-$(CONFIG_PCIE_UNIPHIER) += pcie-uniphier.o # depending on whether ACPI, the DT driver, or both are enabled. ifdef CONFIG_PCI +obj-$(CONFIG_ARM64) += pcie-al.o obj-$(CONFIG_ARM64) += pcie-hisi.o endif diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c index ae84a69ae63a..4234ddb4722f 100644 --- a/drivers/pci/controller/dwc/pci-dra7xx.c +++ b/drivers/pci/controller/dwc/pci-dra7xx.c @@ -26,6 +26,7 @@ #include <linux/types.h> #include <linux/mfd/syscon.h> #include <linux/regmap.h> +#include <linux/gpio/consumer.h> #include "../../pci.h" #include "pcie-designware.h" @@ -247,6 +248,7 @@ static int dra7xx_pcie_init_irq_domain(struct pcie_port *pp) dra7xx->irq_domain = irq_domain_add_linear(pcie_intc_node, PCI_NUM_INTX, &intx_domain_ops, pp); + of_node_put(pcie_intc_node); if (!dra7xx->irq_domain) { dev_err(dev, "Failed to get a INTx IRQ domain\n"); return -ENODEV; @@ -406,7 +408,7 @@ dra7xx_pcie_get_features(struct dw_pcie_ep *ep) return &dra7xx_pcie_epc_features; } -static struct dw_pcie_ep_ops pcie_ep_ops = { +static const struct dw_pcie_ep_ops pcie_ep_ops = { .ep_init = dra7xx_pcie_ep_init, .raise_irq = dra7xx_pcie_raise_irq, .get_features = dra7xx_pcie_get_features, diff --git a/drivers/pci/controller/dwc/pci-exynos.c b/drivers/pci/controller/dwc/pci-exynos.c index cee5f2f590e2..14a6ba4067fb 100644 --- a/drivers/pci/controller/dwc/pci-exynos.c +++ b/drivers/pci/controller/dwc/pci-exynos.c @@ -465,7 +465,7 @@ static int __init exynos_pcie_probe(struct platform_device *pdev) ep->phy = devm_of_phy_get(dev, np, NULL); if (IS_ERR(ep->phy)) { - if (PTR_ERR(ep->phy) == -EPROBE_DEFER) + if (PTR_ERR(ep->phy) != -ENODEV) return PTR_ERR(ep->phy); ep->phy = NULL; diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index 3d627f94a166..acfbd34032a8 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -52,10 +52,12 @@ enum imx6_pcie_variants { #define IMX6_PCIE_FLAG_IMX6_PHY BIT(0) #define IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE BIT(1) +#define IMX6_PCIE_FLAG_SUPPORTS_SUSPEND BIT(2) struct imx6_pcie_drvdata { enum imx6_pcie_variants variant; u32 flags; + int dbi_length; }; struct imx6_pcie { @@ -89,9 +91,8 @@ struct imx6_pcie { }; /* Parameters for the waiting for PCIe PHY PLL to lock on i.MX7 */ -#define PHY_PLL_LOCK_WAIT_MAX_RETRIES 2000 -#define PHY_PLL_LOCK_WAIT_USLEEP_MIN 50 #define PHY_PLL_LOCK_WAIT_USLEEP_MAX 200 +#define PHY_PLL_LOCK_WAIT_TIMEOUT (2000 * PHY_PLL_LOCK_WAIT_USLEEP_MAX) /* PCIe Root Complex registers (memory-mapped) */ #define PCIE_RC_IMX6_MSI_CAP 0x50 @@ -104,34 +105,29 @@ struct imx6_pcie { /* PCIe Port Logic registers (memory-mapped) */ #define PL_OFFSET 0x700 -#define PCIE_PL_PFLR (PL_OFFSET + 0x08) -#define PCIE_PL_PFLR_LINK_STATE_MASK (0x3f << 16) -#define PCIE_PL_PFLR_FORCE_LINK (1 << 15) -#define PCIE_PHY_DEBUG_R0 (PL_OFFSET + 0x28) -#define PCIE_PHY_DEBUG_R1 (PL_OFFSET + 0x2c) #define PCIE_PHY_CTRL (PL_OFFSET + 0x114) -#define PCIE_PHY_CTRL_DATA_LOC 0 -#define PCIE_PHY_CTRL_CAP_ADR_LOC 16 -#define PCIE_PHY_CTRL_CAP_DAT_LOC 17 -#define PCIE_PHY_CTRL_WR_LOC 18 -#define PCIE_PHY_CTRL_RD_LOC 19 +#define PCIE_PHY_CTRL_DATA(x) FIELD_PREP(GENMASK(15, 0), (x)) +#define PCIE_PHY_CTRL_CAP_ADR BIT(16) +#define PCIE_PHY_CTRL_CAP_DAT BIT(17) +#define PCIE_PHY_CTRL_WR BIT(18) +#define PCIE_PHY_CTRL_RD BIT(19) #define PCIE_PHY_STAT (PL_OFFSET + 0x110) -#define PCIE_PHY_STAT_ACK_LOC 16 +#define PCIE_PHY_STAT_ACK BIT(16) #define PCIE_LINK_WIDTH_SPEED_CONTROL 0x80C /* PHY registers (not memory-mapped) */ #define PCIE_PHY_ATEOVRD 0x10 -#define PCIE_PHY_ATEOVRD_EN (0x1 << 2) +#define PCIE_PHY_ATEOVRD_EN BIT(2) #define PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT 0 #define PCIE_PHY_ATEOVRD_REF_CLKDIV_MASK 0x1 #define PCIE_PHY_MPLL_OVRD_IN_LO 0x11 #define PCIE_PHY_MPLL_MULTIPLIER_SHIFT 2 #define PCIE_PHY_MPLL_MULTIPLIER_MASK 0x7f -#define PCIE_PHY_MPLL_MULTIPLIER_OVRD (0x1 << 9) +#define PCIE_PHY_MPLL_MULTIPLIER_OVRD BIT(9) #define PCIE_PHY_RX_ASIC_OUT 0x100D #define PCIE_PHY_RX_ASIC_OUT_VALID (1 << 0) @@ -154,19 +150,19 @@ struct imx6_pcie { #define PCIE_PHY_CMN_REG26_ATT_MODE 0xBC #define PHY_RX_OVRD_IN_LO 0x1005 -#define PHY_RX_OVRD_IN_LO_RX_DATA_EN (1 << 5) -#define PHY_RX_OVRD_IN_LO_RX_PLL_EN (1 << 3) +#define PHY_RX_OVRD_IN_LO_RX_DATA_EN BIT(5) +#define PHY_RX_OVRD_IN_LO_RX_PLL_EN BIT(3) -static int pcie_phy_poll_ack(struct imx6_pcie *imx6_pcie, int exp_val) +static int pcie_phy_poll_ack(struct imx6_pcie *imx6_pcie, bool exp_val) { struct dw_pcie *pci = imx6_pcie->pci; - u32 val; + bool val; u32 max_iterations = 10; u32 wait_counter = 0; do { - val = dw_pcie_readl_dbi(pci, PCIE_PHY_STAT); - val = (val >> PCIE_PHY_STAT_ACK_LOC) & 0x1; + val = dw_pcie_readl_dbi(pci, PCIE_PHY_STAT) & + PCIE_PHY_STAT_ACK; wait_counter++; if (val == exp_val) @@ -184,27 +180,27 @@ static int pcie_phy_wait_ack(struct imx6_pcie *imx6_pcie, int addr) u32 val; int ret; - val = addr << PCIE_PHY_CTRL_DATA_LOC; + val = PCIE_PHY_CTRL_DATA(addr); dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val); - val |= (0x1 << PCIE_PHY_CTRL_CAP_ADR_LOC); + val |= PCIE_PHY_CTRL_CAP_ADR; dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val); - ret = pcie_phy_poll_ack(imx6_pcie, 1); + ret = pcie_phy_poll_ack(imx6_pcie, true); if (ret) return ret; - val = addr << PCIE_PHY_CTRL_DATA_LOC; + val = PCIE_PHY_CTRL_DATA(addr); dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val); - return pcie_phy_poll_ack(imx6_pcie, 0); + return pcie_phy_poll_ack(imx6_pcie, false); } /* Read from the 16-bit PCIe PHY control registers (not memory-mapped) */ -static int pcie_phy_read(struct imx6_pcie *imx6_pcie, int addr, int *data) +static int pcie_phy_read(struct imx6_pcie *imx6_pcie, int addr, u16 *data) { struct dw_pcie *pci = imx6_pcie->pci; - u32 val, phy_ctl; + u32 phy_ctl; int ret; ret = pcie_phy_wait_ack(imx6_pcie, addr); @@ -212,23 +208,22 @@ static int pcie_phy_read(struct imx6_pcie *imx6_pcie, int addr, int *data) return ret; /* assert Read signal */ - phy_ctl = 0x1 << PCIE_PHY_CTRL_RD_LOC; + phy_ctl = PCIE_PHY_CTRL_RD; dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, phy_ctl); - ret = pcie_phy_poll_ack(imx6_pcie, 1); + ret = pcie_phy_poll_ack(imx6_pcie, true); if (ret) return ret; - val = dw_pcie_readl_dbi(pci, PCIE_PHY_STAT); - *data = val & 0xffff; + *data = dw_pcie_readl_dbi(pci, PCIE_PHY_STAT); /* deassert Read signal */ dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, 0x00); - return pcie_phy_poll_ack(imx6_pcie, 0); + return pcie_phy_poll_ack(imx6_pcie, false); } -static int pcie_phy_write(struct imx6_pcie *imx6_pcie, int addr, int data) +static int pcie_phy_write(struct imx6_pcie *imx6_pcie, int addr, u16 data) { struct dw_pcie *pci = imx6_pcie->pci; u32 var; @@ -240,41 +235,41 @@ static int pcie_phy_write(struct imx6_pcie *imx6_pcie, int addr, int data) if (ret) return ret; - var = data << PCIE_PHY_CTRL_DATA_LOC; + var = PCIE_PHY_CTRL_DATA(data); dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var); /* capture data */ - var |= (0x1 << PCIE_PHY_CTRL_CAP_DAT_LOC); + var |= PCIE_PHY_CTRL_CAP_DAT; dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var); - ret = pcie_phy_poll_ack(imx6_pcie, 1); + ret = pcie_phy_poll_ack(imx6_pcie, true); if (ret) return ret; /* deassert cap data */ - var = data << PCIE_PHY_CTRL_DATA_LOC; + var = PCIE_PHY_CTRL_DATA(data); dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var); /* wait for ack de-assertion */ - ret = pcie_phy_poll_ack(imx6_pcie, 0); + ret = pcie_phy_poll_ack(imx6_pcie, false); if (ret) return ret; /* assert wr signal */ - var = 0x1 << PCIE_PHY_CTRL_WR_LOC; + var = PCIE_PHY_CTRL_WR; dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var); /* wait for ack */ - ret = pcie_phy_poll_ack(imx6_pcie, 1); + ret = pcie_phy_poll_ack(imx6_pcie, true); if (ret) return ret; /* deassert wr signal */ - var = data << PCIE_PHY_CTRL_DATA_LOC; + var = PCIE_PHY_CTRL_DATA(data); dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var); /* wait for ack de-assertion */ - ret = pcie_phy_poll_ack(imx6_pcie, 0); + ret = pcie_phy_poll_ack(imx6_pcie, false); if (ret) return ret; @@ -285,7 +280,7 @@ static int pcie_phy_write(struct imx6_pcie *imx6_pcie, int addr, int data) static void imx6_pcie_reset_phy(struct imx6_pcie *imx6_pcie) { - u32 tmp; + u16 tmp; if (!(imx6_pcie->drvdata->flags & IMX6_PCIE_FLAG_IMX6_PHY)) return; @@ -455,7 +450,7 @@ static int imx6_pcie_enable_ref_clk(struct imx6_pcie *imx6_pcie) * reset time is too short, cannot meet the requirement. * add one ~10us delay here. */ - udelay(10); + usleep_range(10, 100); regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_REF_CLK_EN, 1 << 16); break; @@ -488,20 +483,14 @@ static int imx6_pcie_enable_ref_clk(struct imx6_pcie *imx6_pcie) static void imx7d_pcie_wait_for_phy_pll_lock(struct imx6_pcie *imx6_pcie) { u32 val; - unsigned int retries; struct device *dev = imx6_pcie->pci->dev; - for (retries = 0; retries < PHY_PLL_LOCK_WAIT_MAX_RETRIES; retries++) { - regmap_read(imx6_pcie->iomuxc_gpr, IOMUXC_GPR22, &val); - - if (val & IMX7D_GPR22_PCIE_PHY_PLL_LOCKED) - return; - - usleep_range(PHY_PLL_LOCK_WAIT_USLEEP_MIN, - PHY_PLL_LOCK_WAIT_USLEEP_MAX); - } - - dev_err(dev, "PCIe PLL lock timeout\n"); + if (regmap_read_poll_timeout(imx6_pcie->iomuxc_gpr, + IOMUXC_GPR22, val, + val & IMX7D_GPR22_PCIE_PHY_PLL_LOCKED, + PHY_PLL_LOCK_WAIT_USLEEP_MAX, + PHY_PLL_LOCK_WAIT_TIMEOUT)) + dev_err(dev, "PCIe PLL lock timeout\n"); } static void imx6_pcie_deassert_core_reset(struct imx6_pcie *imx6_pcie) @@ -687,7 +676,7 @@ static int imx6_setup_phy_mpll(struct imx6_pcie *imx6_pcie) { unsigned long phy_rate = clk_get_rate(imx6_pcie->pcie_phy); int mult, div; - u32 val; + u16 val; if (!(imx6_pcie->drvdata->flags & IMX6_PCIE_FLAG_IMX6_PHY)) return 0; @@ -730,21 +719,6 @@ static int imx6_setup_phy_mpll(struct imx6_pcie *imx6_pcie) return 0; } -static int imx6_pcie_wait_for_link(struct imx6_pcie *imx6_pcie) -{ - struct dw_pcie *pci = imx6_pcie->pci; - struct device *dev = pci->dev; - - /* check if the link is up or not */ - if (!dw_pcie_wait_for_link(pci)) - return 0; - - dev_dbg(dev, "DEBUG_R0: 0x%08x, DEBUG_R1: 0x%08x\n", - dw_pcie_readl_dbi(pci, PCIE_PHY_DEBUG_R0), - dw_pcie_readl_dbi(pci, PCIE_PHY_DEBUG_R1)); - return -ETIMEDOUT; -} - static int imx6_pcie_wait_for_speed_change(struct imx6_pcie *imx6_pcie) { struct dw_pcie *pci = imx6_pcie->pci; @@ -761,7 +735,7 @@ static int imx6_pcie_wait_for_speed_change(struct imx6_pcie *imx6_pcie) } dev_err(dev, "Speed change timeout\n"); - return -EINVAL; + return -ETIMEDOUT; } static void imx6_pcie_ltssm_enable(struct device *dev) @@ -803,7 +777,7 @@ static int imx6_pcie_establish_link(struct imx6_pcie *imx6_pcie) /* Start LTSSM. */ imx6_pcie_ltssm_enable(dev); - ret = imx6_pcie_wait_for_link(imx6_pcie); + ret = dw_pcie_wait_for_link(pci); if (ret) goto err_reset_phy; @@ -841,7 +815,7 @@ static int imx6_pcie_establish_link(struct imx6_pcie *imx6_pcie) } /* Make sure link training is finished as well! */ - ret = imx6_pcie_wait_for_link(imx6_pcie); + ret = dw_pcie_wait_for_link(pci); if (ret) { dev_err(dev, "Failed to bring link up!\n"); goto err_reset_phy; @@ -856,8 +830,8 @@ static int imx6_pcie_establish_link(struct imx6_pcie *imx6_pcie) err_reset_phy: dev_dbg(dev, "PHY DEBUG_R0=0x%08x DEBUG_R1=0x%08x\n", - dw_pcie_readl_dbi(pci, PCIE_PHY_DEBUG_R0), - dw_pcie_readl_dbi(pci, PCIE_PHY_DEBUG_R1)); + dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG0), + dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG1)); imx6_pcie_reset_phy(imx6_pcie); return ret; } @@ -993,17 +967,11 @@ static void imx6_pcie_clk_disable(struct imx6_pcie *imx6_pcie) } } -static inline bool imx6_pcie_supports_suspend(struct imx6_pcie *imx6_pcie) -{ - return (imx6_pcie->drvdata->variant == IMX7D || - imx6_pcie->drvdata->variant == IMX6SX); -} - static int imx6_pcie_suspend_noirq(struct device *dev) { struct imx6_pcie *imx6_pcie = dev_get_drvdata(dev); - if (!imx6_pcie_supports_suspend(imx6_pcie)) + if (!(imx6_pcie->drvdata->flags & IMX6_PCIE_FLAG_SUPPORTS_SUSPEND)) return 0; imx6_pcie_pm_turnoff(imx6_pcie); @@ -1019,7 +987,7 @@ static int imx6_pcie_resume_noirq(struct device *dev) struct imx6_pcie *imx6_pcie = dev_get_drvdata(dev); struct pcie_port *pp = &imx6_pcie->pci->pp; - if (!imx6_pcie_supports_suspend(imx6_pcie)) + if (!(imx6_pcie->drvdata->flags & IMX6_PCIE_FLAG_SUPPORTS_SUSPEND)) return 0; imx6_pcie_assert_core_reset(imx6_pcie); @@ -1206,8 +1174,8 @@ static int imx6_pcie_probe(struct platform_device *pdev) imx6_pcie->vpcie = devm_regulator_get_optional(&pdev->dev, "vpcie"); if (IS_ERR(imx6_pcie->vpcie)) { - if (PTR_ERR(imx6_pcie->vpcie) == -EPROBE_DEFER) - return -EPROBE_DEFER; + if (PTR_ERR(imx6_pcie->vpcie) != -ENODEV) + return PTR_ERR(imx6_pcie->vpcie); imx6_pcie->vpcie = NULL; } @@ -1245,11 +1213,13 @@ static const struct imx6_pcie_drvdata drvdata[] = { .variant = IMX6Q, .flags = IMX6_PCIE_FLAG_IMX6_PHY | IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE, + .dbi_length = 0x200, }, [IMX6SX] = { .variant = IMX6SX, .flags = IMX6_PCIE_FLAG_IMX6_PHY | - IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE, + IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE | + IMX6_PCIE_FLAG_SUPPORTS_SUSPEND, }, [IMX6QP] = { .variant = IMX6QP, @@ -1258,6 +1228,7 @@ static const struct imx6_pcie_drvdata drvdata[] = { }, [IMX7D] = { .variant = IMX7D, + .flags = IMX6_PCIE_FLAG_SUPPORTS_SUSPEND, }, [IMX8MQ] = { .variant = IMX8MQ, @@ -1279,11 +1250,43 @@ static struct platform_driver imx6_pcie_driver = { .of_match_table = imx6_pcie_of_match, .suppress_bind_attrs = true, .pm = &imx6_pcie_pm_ops, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, }, .probe = imx6_pcie_probe, .shutdown = imx6_pcie_shutdown, }; +static void imx6_pcie_quirk(struct pci_dev *dev) +{ + struct pci_bus *bus = dev->bus; + struct pcie_port *pp = bus->sysdata; + + /* Bus parent is the PCI bridge, its parent is this platform driver */ + if (!bus->dev.parent || !bus->dev.parent->parent) + return; + + /* Make sure we only quirk devices associated with this driver */ + if (bus->dev.parent->parent->driver != &imx6_pcie_driver.driver) + return; + + if (bus->number == pp->root_bus_nr) { + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct imx6_pcie *imx6_pcie = to_imx6_pcie(pci); + + /* + * Limit config length to avoid the kernel reading beyond + * the register set and causing an abort on i.MX 6Quad + */ + if (imx6_pcie->drvdata->dbi_length) { + dev->cfg_size = imx6_pcie->drvdata->dbi_length; + dev_info(&dev->dev, "Limiting cfg_size to %d\n", + dev->cfg_size); + } + } +} +DECLARE_PCI_FIXUP_CLASS_HEADER(PCI_VENDOR_ID_SYNOPSYS, 0xabcd, + PCI_CLASS_BRIDGE_PCI, 8, imx6_pcie_quirk); + static int __init imx6_pcie_init(void) { #ifdef CONFIG_ARM diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c index 14f2b0b4ed5e..af677254a072 100644 --- a/drivers/pci/controller/dwc/pci-keystone.c +++ b/drivers/pci/controller/dwc/pci-keystone.c @@ -11,6 +11,7 @@ #include <linux/clk.h> #include <linux/delay.h> +#include <linux/gpio/consumer.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/irqchip/chained_irq.h> @@ -18,6 +19,7 @@ #include <linux/mfd/syscon.h> #include <linux/msi.h> #include <linux/of.h> +#include <linux/of_device.h> #include <linux/of_irq.h> #include <linux/of_pci.h> #include <linux/phy/phy.h> @@ -26,6 +28,7 @@ #include <linux/resource.h> #include <linux/signal.h> +#include "../../pci.h" #include "pcie-designware.h" #define PCIE_VENDORID_MASK 0xffff @@ -44,28 +47,34 @@ #define CFG_TYPE1 BIT(24) #define OB_SIZE 0x030 -#define SPACE0_REMOTE_CFG_OFFSET 0x1000 #define OB_OFFSET_INDEX(n) (0x200 + (8 * (n))) #define OB_OFFSET_HI(n) (0x204 + (8 * (n))) #define OB_ENABLEN BIT(0) #define OB_WIN_SIZE 8 /* 8MB */ +#define PCIE_LEGACY_IRQ_ENABLE_SET(n) (0x188 + (0x10 * ((n) - 1))) +#define PCIE_LEGACY_IRQ_ENABLE_CLR(n) (0x18c + (0x10 * ((n) - 1))) +#define PCIE_EP_IRQ_SET 0x64 +#define PCIE_EP_IRQ_CLR 0x68 +#define INT_ENABLE BIT(0) + /* IRQ register defines */ #define IRQ_EOI 0x050 -#define IRQ_STATUS 0x184 -#define IRQ_ENABLE_SET 0x188 -#define IRQ_ENABLE_CLR 0x18c #define MSI_IRQ 0x054 -#define MSI0_IRQ_STATUS 0x104 -#define MSI0_IRQ_ENABLE_SET 0x108 -#define MSI0_IRQ_ENABLE_CLR 0x10c -#define IRQ_STATUS 0x184 +#define MSI_IRQ_STATUS(n) (0x104 + ((n) << 4)) +#define MSI_IRQ_ENABLE_SET(n) (0x108 + ((n) << 4)) +#define MSI_IRQ_ENABLE_CLR(n) (0x10c + ((n) << 4)) #define MSI_IRQ_OFFSET 4 +#define IRQ_STATUS(n) (0x184 + ((n) << 4)) +#define IRQ_ENABLE_SET(n) (0x188 + ((n) << 4)) +#define INTx_EN BIT(0) + #define ERR_IRQ_STATUS 0x1c4 #define ERR_IRQ_ENABLE_SET 0x1c8 #define ERR_AER BIT(5) /* ECRC error */ +#define AM6_ERR_AER BIT(4) /* AM6 ECRC error */ #define ERR_AXI BIT(4) /* AXI tag lookup fatal error */ #define ERR_CORR BIT(3) /* Correctable error */ #define ERR_NONFATAL BIT(2) /* Non-fatal error */ @@ -74,25 +83,45 @@ #define ERR_IRQ_ALL (ERR_AER | ERR_AXI | ERR_CORR | \ ERR_NONFATAL | ERR_FATAL | ERR_SYS) -#define MAX_MSI_HOST_IRQS 8 /* PCIE controller device IDs */ #define PCIE_RC_K2HK 0xb008 #define PCIE_RC_K2E 0xb009 #define PCIE_RC_K2L 0xb00a #define PCIE_RC_K2G 0xb00b +#define KS_PCIE_DEV_TYPE_MASK (0x3 << 1) +#define KS_PCIE_DEV_TYPE(mode) ((mode) << 1) + +#define EP 0x0 +#define LEG_EP 0x1 +#define RC 0x2 + +#define EXP_CAP_ID_OFFSET 0x70 + +#define KS_PCIE_SYSCLOCKOUTEN BIT(0) + +#define AM654_PCIE_DEV_TYPE_MASK 0x3 +#define AM654_WIN_SIZE SZ_64K + +#define APP_ADDR_SPACE_0 (16 * SZ_1K) + #define to_keystone_pcie(x) dev_get_drvdata((x)->dev) +struct ks_pcie_of_data { + enum dw_pcie_device_mode mode; + const struct dw_pcie_host_ops *host_ops; + const struct dw_pcie_ep_ops *ep_ops; + unsigned int version; +}; + struct keystone_pcie { struct dw_pcie *pci; /* PCI Device ID */ u32 device_id; - int num_legacy_host_irqs; int legacy_host_irqs[PCI_NUM_INTX]; struct device_node *legacy_intc_np; - int num_msi_host_irqs; - int msi_host_irqs[MAX_MSI_HOST_IRQS]; + int msi_host_irq; int num_lanes; u32 num_viewport; struct phy **phy; @@ -101,28 +130,12 @@ struct keystone_pcie { struct irq_domain *legacy_irq_domain; struct device_node *np; - int error_irq; - /* Application register space */ void __iomem *va_app_base; /* DT 1st resource */ struct resource app; + bool is_am6; }; -static inline void update_reg_offset_bit_pos(u32 offset, u32 *reg_offset, - u32 *bit_pos) -{ - *reg_offset = offset % 8; - *bit_pos = offset >> 3; -} - -static phys_addr_t ks_pcie_get_msi_addr(struct pcie_port *pp) -{ - struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); - - return ks_pcie->app.start + MSI_IRQ; -} - static u32 ks_pcie_app_readl(struct keystone_pcie *ks_pcie, u32 offset) { return readl(ks_pcie->va_app_base + offset); @@ -134,81 +147,114 @@ static void ks_pcie_app_writel(struct keystone_pcie *ks_pcie, u32 offset, writel(val, ks_pcie->va_app_base + offset); } -static void ks_pcie_handle_msi_irq(struct keystone_pcie *ks_pcie, int offset) +static void ks_pcie_msi_irq_ack(struct irq_data *data) { - struct dw_pcie *pci = ks_pcie->pci; - struct pcie_port *pp = &pci->pp; - struct device *dev = pci->dev; - u32 pending, vector; - int src, virq; + struct pcie_port *pp = irq_data_get_irq_chip_data(data); + struct keystone_pcie *ks_pcie; + u32 irq = data->hwirq; + struct dw_pcie *pci; + u32 reg_offset; + u32 bit_pos; - pending = ks_pcie_app_readl(ks_pcie, MSI0_IRQ_STATUS + (offset << 4)); + pci = to_dw_pcie_from_pp(pp); + ks_pcie = to_keystone_pcie(pci); - /* - * MSI0 status bit 0-3 shows vectors 0, 8, 16, 24, MSI1 status bit - * shows 1, 9, 17, 25 and so forth - */ - for (src = 0; src < 4; src++) { - if (BIT(src) & pending) { - vector = offset + (src << 3); - virq = irq_linear_revmap(pp->irq_domain, vector); - dev_dbg(dev, "irq: bit %d, vector %d, virq %d\n", - src, vector, virq); - generic_handle_irq(virq); - } - } + reg_offset = irq % 8; + bit_pos = irq >> 3; + + ks_pcie_app_writel(ks_pcie, MSI_IRQ_STATUS(reg_offset), + BIT(bit_pos)); + ks_pcie_app_writel(ks_pcie, IRQ_EOI, reg_offset + MSI_IRQ_OFFSET); } -static void ks_pcie_msi_irq_ack(int irq, struct pcie_port *pp) +static void ks_pcie_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) { - u32 reg_offset, bit_pos; + struct pcie_port *pp = irq_data_get_irq_chip_data(data); struct keystone_pcie *ks_pcie; struct dw_pcie *pci; + u64 msi_target; pci = to_dw_pcie_from_pp(pp); ks_pcie = to_keystone_pcie(pci); - update_reg_offset_bit_pos(irq, ®_offset, &bit_pos); - ks_pcie_app_writel(ks_pcie, MSI0_IRQ_STATUS + (reg_offset << 4), - BIT(bit_pos)); - ks_pcie_app_writel(ks_pcie, IRQ_EOI, reg_offset + MSI_IRQ_OFFSET); + msi_target = ks_pcie->app.start + MSI_IRQ; + msg->address_lo = lower_32_bits(msi_target); + msg->address_hi = upper_32_bits(msi_target); + msg->data = data->hwirq; + + dev_dbg(pci->dev, "msi#%d address_hi %#x address_lo %#x\n", + (int)data->hwirq, msg->address_hi, msg->address_lo); } -static void ks_pcie_msi_set_irq(struct pcie_port *pp, int irq) +static int ks_pcie_msi_set_affinity(struct irq_data *irq_data, + const struct cpumask *mask, bool force) { - u32 reg_offset, bit_pos; - struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); + return -EINVAL; +} - update_reg_offset_bit_pos(irq, ®_offset, &bit_pos); - ks_pcie_app_writel(ks_pcie, MSI0_IRQ_ENABLE_SET + (reg_offset << 4), +static void ks_pcie_msi_mask(struct irq_data *data) +{ + struct pcie_port *pp = irq_data_get_irq_chip_data(data); + struct keystone_pcie *ks_pcie; + u32 irq = data->hwirq; + struct dw_pcie *pci; + unsigned long flags; + u32 reg_offset; + u32 bit_pos; + + raw_spin_lock_irqsave(&pp->lock, flags); + + pci = to_dw_pcie_from_pp(pp); + ks_pcie = to_keystone_pcie(pci); + + reg_offset = irq % 8; + bit_pos = irq >> 3; + + ks_pcie_app_writel(ks_pcie, MSI_IRQ_ENABLE_CLR(reg_offset), BIT(bit_pos)); + + raw_spin_unlock_irqrestore(&pp->lock, flags); } -static void ks_pcie_msi_clear_irq(struct pcie_port *pp, int irq) +static void ks_pcie_msi_unmask(struct irq_data *data) { - u32 reg_offset, bit_pos; - struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); + struct pcie_port *pp = irq_data_get_irq_chip_data(data); + struct keystone_pcie *ks_pcie; + u32 irq = data->hwirq; + struct dw_pcie *pci; + unsigned long flags; + u32 reg_offset; + u32 bit_pos; + + raw_spin_lock_irqsave(&pp->lock, flags); - update_reg_offset_bit_pos(irq, ®_offset, &bit_pos); - ks_pcie_app_writel(ks_pcie, MSI0_IRQ_ENABLE_CLR + (reg_offset << 4), + pci = to_dw_pcie_from_pp(pp); + ks_pcie = to_keystone_pcie(pci); + + reg_offset = irq % 8; + bit_pos = irq >> 3; + + ks_pcie_app_writel(ks_pcie, MSI_IRQ_ENABLE_SET(reg_offset), BIT(bit_pos)); + + raw_spin_unlock_irqrestore(&pp->lock, flags); } +static struct irq_chip ks_pcie_msi_irq_chip = { + .name = "KEYSTONE-PCI-MSI", + .irq_ack = ks_pcie_msi_irq_ack, + .irq_compose_msi_msg = ks_pcie_compose_msi_msg, + .irq_set_affinity = ks_pcie_msi_set_affinity, + .irq_mask = ks_pcie_msi_mask, + .irq_unmask = ks_pcie_msi_unmask, +}; + static int ks_pcie_msi_host_init(struct pcie_port *pp) { + pp->msi_irq_chip = &ks_pcie_msi_irq_chip; return dw_pcie_allocate_domains(pp); } -static void ks_pcie_enable_legacy_irqs(struct keystone_pcie *ks_pcie) -{ - int i; - - for (i = 0; i < PCI_NUM_INTX; i++) - ks_pcie_app_writel(ks_pcie, IRQ_ENABLE_SET + (i << 4), 0x1); -} - static void ks_pcie_handle_legacy_irq(struct keystone_pcie *ks_pcie, int offset) { @@ -217,7 +263,7 @@ static void ks_pcie_handle_legacy_irq(struct keystone_pcie *ks_pcie, u32 pending; int virq; - pending = ks_pcie_app_readl(ks_pcie, IRQ_STATUS + (offset << 4)); + pending = ks_pcie_app_readl(ks_pcie, IRQ_STATUS(offset)); if (BIT(0) & pending) { virq = irq_linear_revmap(ks_pcie->legacy_irq_domain, offset); @@ -229,6 +275,14 @@ static void ks_pcie_handle_legacy_irq(struct keystone_pcie *ks_pcie, ks_pcie_app_writel(ks_pcie, IRQ_EOI, offset); } +/* + * Dummy function so that DW core doesn't configure MSI + */ +static int ks_pcie_am654_msi_host_init(struct pcie_port *pp) +{ + return 0; +} + static void ks_pcie_enable_error_irq(struct keystone_pcie *ks_pcie) { ks_pcie_app_writel(ks_pcie, ERR_IRQ_ENABLE_SET, ERR_IRQ_ALL); @@ -255,10 +309,10 @@ static irqreturn_t ks_pcie_handle_error_irq(struct keystone_pcie *ks_pcie) if (reg & ERR_CORR) dev_dbg(dev, "Correctable Error\n"); - if (reg & ERR_AXI) + if (!ks_pcie->is_am6 && (reg & ERR_AXI)) dev_err(dev, "AXI tag lookup fatal Error\n"); - if (reg & ERR_AER) + if (reg & ERR_AER || (ks_pcie->is_am6 && (reg & AM6_ERR_AER))) dev_err(dev, "ECRC Error\n"); ks_pcie_app_writel(ks_pcie, ERR_IRQ_STATUS, reg); @@ -356,6 +410,9 @@ static void ks_pcie_setup_rc_app_regs(struct keystone_pcie *ks_pcie) dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_1, 0); ks_pcie_clear_dbi_mode(ks_pcie); + if (ks_pcie->is_am6) + return; + val = ilog2(OB_WIN_SIZE); ks_pcie_app_writel(ks_pcie, OB_SIZE, val); @@ -445,68 +502,33 @@ static int ks_pcie_link_up(struct dw_pcie *pci) return (val == PORT_LOGIC_LTSSM_STATE_L0); } -static void ks_pcie_initiate_link_train(struct keystone_pcie *ks_pcie) +static void ks_pcie_stop_link(struct dw_pcie *pci) { + struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); u32 val; /* Disable Link training */ val = ks_pcie_app_readl(ks_pcie, CMD_STATUS); val &= ~LTSSM_EN_VAL; ks_pcie_app_writel(ks_pcie, CMD_STATUS, LTSSM_EN_VAL | val); - - /* Initiate Link Training */ - val = ks_pcie_app_readl(ks_pcie, CMD_STATUS); - ks_pcie_app_writel(ks_pcie, CMD_STATUS, LTSSM_EN_VAL | val); } -/** - * ks_pcie_dw_host_init() - initialize host for v3_65 dw hardware - * - * Ioremap the register resources, initialize legacy irq domain - * and call dw_pcie_v3_65_host_init() API to initialize the Keystone - * PCI host controller. - */ -static int __init ks_pcie_dw_host_init(struct keystone_pcie *ks_pcie) +static int ks_pcie_start_link(struct dw_pcie *pci) { - struct dw_pcie *pci = ks_pcie->pci; - struct pcie_port *pp = &pci->pp; + struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); struct device *dev = pci->dev; - struct platform_device *pdev = to_platform_device(dev); - struct resource *res; - - /* Index 0 is the config reg. space address */ - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - pci->dbi_base = devm_pci_remap_cfg_resource(dev, res); - if (IS_ERR(pci->dbi_base)) - return PTR_ERR(pci->dbi_base); - - /* - * We set these same and is used in pcie rd/wr_other_conf - * functions - */ - pp->va_cfg0_base = pci->dbi_base + SPACE0_REMOTE_CFG_OFFSET; - pp->va_cfg1_base = pp->va_cfg0_base; - - /* Index 1 is the application reg. space address */ - res = platform_get_resource(pdev, IORESOURCE_MEM, 1); - ks_pcie->va_app_base = devm_ioremap_resource(dev, res); - if (IS_ERR(ks_pcie->va_app_base)) - return PTR_ERR(ks_pcie->va_app_base); - - ks_pcie->app = *res; + u32 val; - /* Create legacy IRQ domain */ - ks_pcie->legacy_irq_domain = - irq_domain_add_linear(ks_pcie->legacy_intc_np, - PCI_NUM_INTX, - &ks_pcie_legacy_irq_domain_ops, - NULL); - if (!ks_pcie->legacy_irq_domain) { - dev_err(dev, "Failed to add irq domain for legacy irqs\n"); - return -EINVAL; + if (dw_pcie_link_up(pci)) { + dev_dbg(dev, "link is already up\n"); + return 0; } - return dw_pcie_host_init(pp); + /* Initiate Link Training */ + val = ks_pcie_app_readl(ks_pcie, CMD_STATUS); + ks_pcie_app_writel(ks_pcie, CMD_STATUS, LTSSM_EN_VAL | val); + + return 0; } static void ks_pcie_quirk(struct pci_dev *dev) @@ -552,34 +574,16 @@ static void ks_pcie_quirk(struct pci_dev *dev) } DECLARE_PCI_FIXUP_ENABLE(PCI_ANY_ID, PCI_ANY_ID, ks_pcie_quirk); -static int ks_pcie_establish_link(struct keystone_pcie *ks_pcie) -{ - struct dw_pcie *pci = ks_pcie->pci; - struct device *dev = pci->dev; - - if (dw_pcie_link_up(pci)) { - dev_info(dev, "Link already up\n"); - return 0; - } - - ks_pcie_initiate_link_train(ks_pcie); - - /* check if the link is up or not */ - if (!dw_pcie_wait_for_link(pci)) - return 0; - - dev_err(dev, "phy link never came up\n"); - return -ETIMEDOUT; -} - static void ks_pcie_msi_irq_handler(struct irq_desc *desc) { - unsigned int irq = irq_desc_get_irq(desc); + unsigned int irq = desc->irq_data.hwirq; struct keystone_pcie *ks_pcie = irq_desc_get_handler_data(desc); - u32 offset = irq - ks_pcie->msi_host_irqs[0]; + u32 offset = irq - ks_pcie->msi_host_irq; struct dw_pcie *pci = ks_pcie->pci; + struct pcie_port *pp = &pci->pp; struct device *dev = pci->dev; struct irq_chip *chip = irq_desc_get_chip(desc); + u32 vector, virq, reg, pos; dev_dbg(dev, "%s, irq %d\n", __func__, irq); @@ -589,7 +593,23 @@ static void ks_pcie_msi_irq_handler(struct irq_desc *desc) * ack operation. */ chained_irq_enter(chip, desc); - ks_pcie_handle_msi_irq(ks_pcie, offset); + + reg = ks_pcie_app_readl(ks_pcie, MSI_IRQ_STATUS(offset)); + /* + * MSI0 status bit 0-3 shows vectors 0, 8, 16, 24, MSI1 status bit + * shows 1, 9, 17, 25 and so forth + */ + for (pos = 0; pos < 4; pos++) { + if (!(reg & BIT(pos))) + continue; + + vector = offset + (pos << 3); + virq = irq_linear_revmap(pp->irq_domain, vector); + dev_dbg(dev, "irq: bit %d, vector %d, virq %d\n", pos, vector, + virq); + generic_handle_irq(virq); + } + chained_irq_exit(chip, desc); } @@ -622,89 +642,119 @@ static void ks_pcie_legacy_irq_handler(struct irq_desc *desc) chained_irq_exit(chip, desc); } -static int ks_pcie_get_irq_controller_info(struct keystone_pcie *ks_pcie, - char *controller, int *num_irqs) +static int ks_pcie_config_msi_irq(struct keystone_pcie *ks_pcie) { - int temp, max_host_irqs, legacy = 1, *host_irqs; struct device *dev = ks_pcie->pci->dev; - struct device_node *np_pcie = dev->of_node, **np_temp; - - if (!strcmp(controller, "msi-interrupt-controller")) - legacy = 0; + struct device_node *np = ks_pcie->np; + struct device_node *intc_np; + struct irq_data *irq_data; + int irq_count, irq, ret, i; - if (legacy) { - np_temp = &ks_pcie->legacy_intc_np; - max_host_irqs = PCI_NUM_INTX; - host_irqs = &ks_pcie->legacy_host_irqs[0]; - } else { - np_temp = &ks_pcie->msi_intc_np; - max_host_irqs = MAX_MSI_HOST_IRQS; - host_irqs = &ks_pcie->msi_host_irqs[0]; - } + if (!IS_ENABLED(CONFIG_PCI_MSI)) + return 0; - /* interrupt controller is in a child node */ - *np_temp = of_get_child_by_name(np_pcie, controller); - if (!(*np_temp)) { - dev_err(dev, "Node for %s is absent\n", controller); + intc_np = of_get_child_by_name(np, "msi-interrupt-controller"); + if (!intc_np) { + if (ks_pcie->is_am6) + return 0; + dev_warn(dev, "msi-interrupt-controller node is absent\n"); return -EINVAL; } - temp = of_irq_count(*np_temp); - if (!temp) { - dev_err(dev, "No IRQ entries in %s\n", controller); - of_node_put(*np_temp); - return -EINVAL; + irq_count = of_irq_count(intc_np); + if (!irq_count) { + dev_err(dev, "No IRQ entries in msi-interrupt-controller\n"); + ret = -EINVAL; + goto err; } - if (temp > max_host_irqs) - dev_warn(dev, "Too many %s interrupts defined %u\n", - (legacy ? "legacy" : "MSI"), temp); - - /* - * support upto max_host_irqs. In dt from index 0 to 3 (legacy) or 0 to - * 7 (MSI) - */ - for (temp = 0; temp < max_host_irqs; temp++) { - host_irqs[temp] = irq_of_parse_and_map(*np_temp, temp); - if (!host_irqs[temp]) - break; - } + for (i = 0; i < irq_count; i++) { + irq = irq_of_parse_and_map(intc_np, i); + if (!irq) { + ret = -EINVAL; + goto err; + } - of_node_put(*np_temp); + if (!ks_pcie->msi_host_irq) { + irq_data = irq_get_irq_data(irq); + if (!irq_data) { + ret = -EINVAL; + goto err; + } + ks_pcie->msi_host_irq = irq_data->hwirq; + } - if (temp) { - *num_irqs = temp; - return 0; + irq_set_chained_handler_and_data(irq, ks_pcie_msi_irq_handler, + ks_pcie); } - return -EINVAL; + of_node_put(intc_np); + return 0; + +err: + of_node_put(intc_np); + return ret; } -static void ks_pcie_setup_interrupts(struct keystone_pcie *ks_pcie) +static int ks_pcie_config_legacy_irq(struct keystone_pcie *ks_pcie) { - int i; + struct device *dev = ks_pcie->pci->dev; + struct irq_domain *legacy_irq_domain; + struct device_node *np = ks_pcie->np; + struct device_node *intc_np; + int irq_count, irq, ret = 0, i; + + intc_np = of_get_child_by_name(np, "legacy-interrupt-controller"); + if (!intc_np) { + /* + * Since legacy interrupts are modeled as edge-interrupts in + * AM6, keep it disabled for now. + */ + if (ks_pcie->is_am6) + return 0; + dev_warn(dev, "legacy-interrupt-controller node is absent\n"); + return -EINVAL; + } - /* Legacy IRQ */ - for (i = 0; i < ks_pcie->num_legacy_host_irqs; i++) { - irq_set_chained_handler_and_data(ks_pcie->legacy_host_irqs[i], + irq_count = of_irq_count(intc_np); + if (!irq_count) { + dev_err(dev, "No IRQ entries in legacy-interrupt-controller\n"); + ret = -EINVAL; + goto err; + } + + for (i = 0; i < irq_count; i++) { + irq = irq_of_parse_and_map(intc_np, i); + if (!irq) { + ret = -EINVAL; + goto err; + } + ks_pcie->legacy_host_irqs[i] = irq; + + irq_set_chained_handler_and_data(irq, ks_pcie_legacy_irq_handler, ks_pcie); } - ks_pcie_enable_legacy_irqs(ks_pcie); - /* MSI IRQ */ - if (IS_ENABLED(CONFIG_PCI_MSI)) { - for (i = 0; i < ks_pcie->num_msi_host_irqs; i++) { - irq_set_chained_handler_and_data(ks_pcie->msi_host_irqs[i], - ks_pcie_msi_irq_handler, - ks_pcie); - } + legacy_irq_domain = + irq_domain_add_linear(intc_np, PCI_NUM_INTX, + &ks_pcie_legacy_irq_domain_ops, NULL); + if (!legacy_irq_domain) { + dev_err(dev, "Failed to add irq domain for legacy irqs\n"); + ret = -EINVAL; + goto err; } + ks_pcie->legacy_irq_domain = legacy_irq_domain; + + for (i = 0; i < PCI_NUM_INTX; i++) + ks_pcie_app_writel(ks_pcie, IRQ_ENABLE_SET(i), INTx_EN); - if (ks_pcie->error_irq > 0) - ks_pcie_enable_error_irq(ks_pcie); +err: + of_node_put(intc_np); + return ret; } +#ifdef CONFIG_ARM /* * When a PCI device does not exist during config cycles, keystone host gets a * bus error instead of returning 0xffffffff. This handler always returns 0 @@ -724,6 +774,7 @@ static int ks_pcie_fault(unsigned long addr, unsigned int fsr, return 0; } +#endif static int __init ks_pcie_init_id(struct keystone_pcie *ks_pcie) { @@ -742,8 +793,10 @@ static int __init ks_pcie_init_id(struct keystone_pcie *ks_pcie) if (ret) return ret; + dw_pcie_dbi_ro_wr_en(pci); dw_pcie_writew_dbi(pci, PCI_VENDOR_ID, id & PCIE_VENDORID_MASK); dw_pcie_writew_dbi(pci, PCI_DEVICE_ID, id >> PCIE_DEVICEID_SHIFT); + dw_pcie_dbi_ro_wr_dis(pci); return 0; } @@ -754,11 +807,18 @@ static int __init ks_pcie_host_init(struct pcie_port *pp) struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); int ret; + ret = ks_pcie_config_legacy_irq(ks_pcie); + if (ret) + return ret; + + ret = ks_pcie_config_msi_irq(ks_pcie); + if (ret) + return ret; + dw_pcie_setup_rc(pp); - ks_pcie_establish_link(ks_pcie); + ks_pcie_stop_link(pci); ks_pcie_setup_rc_app_regs(ks_pcie); - ks_pcie_setup_interrupts(ks_pcie); writew(PCI_IO_RANGE_TYPE_32 | (PCI_IO_RANGE_TYPE_32 << 8), pci->dbi_base + PCI_IO_BASE); @@ -766,12 +826,17 @@ static int __init ks_pcie_host_init(struct pcie_port *pp) if (ret < 0) return ret; +#ifdef CONFIG_ARM /* * PCIe access errors that result into OCP errors are caught by ARM as * "External aborts" */ hook_fault_code(17, ks_pcie_fault, SIGBUS, 0, "Asynchronous external abort"); +#endif + + ks_pcie_start_link(pci); + dw_pcie_wait_for_link(pci); return 0; } @@ -780,14 +845,15 @@ static const struct dw_pcie_host_ops ks_pcie_host_ops = { .rd_other_conf = ks_pcie_rd_other_conf, .wr_other_conf = ks_pcie_wr_other_conf, .host_init = ks_pcie_host_init, - .msi_set_irq = ks_pcie_msi_set_irq, - .msi_clear_irq = ks_pcie_msi_clear_irq, - .get_msi_addr = ks_pcie_get_msi_addr, .msi_host_init = ks_pcie_msi_host_init, - .msi_irq_ack = ks_pcie_msi_irq_ack, .scan_bus = ks_pcie_v3_65_scan_bus, }; +static const struct dw_pcie_host_ops ks_pcie_am654_host_ops = { + .host_init = ks_pcie_host_init, + .msi_host_init = ks_pcie_am654_msi_host_init, +}; + static irqreturn_t ks_pcie_err_irq_handler(int irq, void *priv) { struct keystone_pcie *ks_pcie = priv; @@ -801,41 +867,17 @@ static int __init ks_pcie_add_pcie_port(struct keystone_pcie *ks_pcie, struct dw_pcie *pci = ks_pcie->pci; struct pcie_port *pp = &pci->pp; struct device *dev = &pdev->dev; + struct resource *res; int ret; - ret = ks_pcie_get_irq_controller_info(ks_pcie, - "legacy-interrupt-controller", - &ks_pcie->num_legacy_host_irqs); - if (ret) - return ret; - - if (IS_ENABLED(CONFIG_PCI_MSI)) { - ret = ks_pcie_get_irq_controller_info(ks_pcie, - "msi-interrupt-controller", - &ks_pcie->num_msi_host_irqs); - if (ret) - return ret; - } + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config"); + pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res); + if (IS_ERR(pp->va_cfg0_base)) + return PTR_ERR(pp->va_cfg0_base); - /* - * Index 0 is the platform interrupt for error interrupt - * from RC. This is optional. - */ - ks_pcie->error_irq = irq_of_parse_and_map(ks_pcie->np, 0); - if (ks_pcie->error_irq <= 0) - dev_info(dev, "no error IRQ defined\n"); - else { - ret = request_irq(ks_pcie->error_irq, ks_pcie_err_irq_handler, - IRQF_SHARED, "pcie-error-irq", ks_pcie); - if (ret < 0) { - dev_err(dev, "failed to request error IRQ %d\n", - ks_pcie->error_irq); - return ret; - } - } + pp->va_cfg1_base = pp->va_cfg0_base; - pp->ops = &ks_pcie_host_ops; - ret = ks_pcie_dw_host_init(ks_pcie); + ret = dw_pcie_host_init(pp); if (ret) { dev_err(dev, "failed to initialize host\n"); return ret; @@ -844,18 +886,139 @@ static int __init ks_pcie_add_pcie_port(struct keystone_pcie *ks_pcie, return 0; } -static const struct of_device_id ks_pcie_of_match[] = { - { - .type = "pci", - .compatible = "ti,keystone-pcie", - }, - { }, -}; +static u32 ks_pcie_am654_read_dbi2(struct dw_pcie *pci, void __iomem *base, + u32 reg, size_t size) +{ + struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); + u32 val; + + ks_pcie_set_dbi_mode(ks_pcie); + dw_pcie_read(base + reg, size, &val); + ks_pcie_clear_dbi_mode(ks_pcie); + return val; +} + +static void ks_pcie_am654_write_dbi2(struct dw_pcie *pci, void __iomem *base, + u32 reg, size_t size, u32 val) +{ + struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); + + ks_pcie_set_dbi_mode(ks_pcie); + dw_pcie_write(base + reg, size, val); + ks_pcie_clear_dbi_mode(ks_pcie); +} static const struct dw_pcie_ops ks_pcie_dw_pcie_ops = { + .start_link = ks_pcie_start_link, + .stop_link = ks_pcie_stop_link, .link_up = ks_pcie_link_up, + .read_dbi2 = ks_pcie_am654_read_dbi2, + .write_dbi2 = ks_pcie_am654_write_dbi2, +}; + +static void ks_pcie_am654_ep_init(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + int flags; + + ep->page_size = AM654_WIN_SIZE; + flags = PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_32; + dw_pcie_writel_dbi2(pci, PCI_BASE_ADDRESS_0, APP_ADDR_SPACE_0 - 1); + dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, flags); +} + +static void ks_pcie_am654_raise_legacy_irq(struct keystone_pcie *ks_pcie) +{ + struct dw_pcie *pci = ks_pcie->pci; + u8 int_pin; + + int_pin = dw_pcie_readb_dbi(pci, PCI_INTERRUPT_PIN); + if (int_pin == 0 || int_pin > 4) + return; + + ks_pcie_app_writel(ks_pcie, PCIE_LEGACY_IRQ_ENABLE_SET(int_pin), + INT_ENABLE); + ks_pcie_app_writel(ks_pcie, PCIE_EP_IRQ_SET, INT_ENABLE); + mdelay(1); + ks_pcie_app_writel(ks_pcie, PCIE_EP_IRQ_CLR, INT_ENABLE); + ks_pcie_app_writel(ks_pcie, PCIE_LEGACY_IRQ_ENABLE_CLR(int_pin), + INT_ENABLE); +} + +static int ks_pcie_am654_raise_irq(struct dw_pcie_ep *ep, u8 func_no, + enum pci_epc_irq_type type, + u16 interrupt_num) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); + + switch (type) { + case PCI_EPC_IRQ_LEGACY: + ks_pcie_am654_raise_legacy_irq(ks_pcie); + break; + case PCI_EPC_IRQ_MSI: + dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num); + break; + default: + dev_err(pci->dev, "UNKNOWN IRQ type\n"); + return -EINVAL; + } + + return 0; +} + +static const struct pci_epc_features ks_pcie_am654_epc_features = { + .linkup_notifier = false, + .msi_capable = true, + .msix_capable = false, + .reserved_bar = 1 << BAR_0 | 1 << BAR_1, + .bar_fixed_64bit = 1 << BAR_0, + .bar_fixed_size[2] = SZ_1M, + .bar_fixed_size[3] = SZ_64K, + .bar_fixed_size[4] = 256, + .bar_fixed_size[5] = SZ_1M, + .align = SZ_1M, }; +static const struct pci_epc_features* +ks_pcie_am654_get_features(struct dw_pcie_ep *ep) +{ + return &ks_pcie_am654_epc_features; +} + +static const struct dw_pcie_ep_ops ks_pcie_am654_ep_ops = { + .ep_init = ks_pcie_am654_ep_init, + .raise_irq = ks_pcie_am654_raise_irq, + .get_features = &ks_pcie_am654_get_features, +}; + +static int __init ks_pcie_add_pcie_ep(struct keystone_pcie *ks_pcie, + struct platform_device *pdev) +{ + int ret; + struct dw_pcie_ep *ep; + struct resource *res; + struct device *dev = &pdev->dev; + struct dw_pcie *pci = ks_pcie->pci; + + ep = &pci->ep; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space"); + if (!res) + return -EINVAL; + + ep->phys_base = res->start; + ep->addr_size = resource_size(res); + + ret = dw_pcie_ep_init(ep); + if (ret) { + dev_err(dev, "failed to initialize endpoint\n"); + return ret; + } + + return 0; +} + static void ks_pcie_disable_phy(struct keystone_pcie *ks_pcie) { int num_lanes = ks_pcie->num_lanes; @@ -873,6 +1036,10 @@ static int ks_pcie_enable_phy(struct keystone_pcie *ks_pcie) int num_lanes = ks_pcie->num_lanes; for (i = 0; i < num_lanes; i++) { + ret = phy_reset(ks_pcie->phy[i]); + if (ret < 0) + goto err_phy; + ret = phy_init(ks_pcie->phy[i]); if (ret < 0) goto err_phy; @@ -895,20 +1062,161 @@ err_phy: return ret; } +static int ks_pcie_set_mode(struct device *dev) +{ + struct device_node *np = dev->of_node; + struct regmap *syscon; + u32 val; + u32 mask; + int ret = 0; + + syscon = syscon_regmap_lookup_by_phandle(np, "ti,syscon-pcie-mode"); + if (IS_ERR(syscon)) + return 0; + + mask = KS_PCIE_DEV_TYPE_MASK | KS_PCIE_SYSCLOCKOUTEN; + val = KS_PCIE_DEV_TYPE(RC) | KS_PCIE_SYSCLOCKOUTEN; + + ret = regmap_update_bits(syscon, 0, mask, val); + if (ret) { + dev_err(dev, "failed to set pcie mode\n"); + return ret; + } + + return 0; +} + +static int ks_pcie_am654_set_mode(struct device *dev, + enum dw_pcie_device_mode mode) +{ + struct device_node *np = dev->of_node; + struct regmap *syscon; + u32 val; + u32 mask; + int ret = 0; + + syscon = syscon_regmap_lookup_by_phandle(np, "ti,syscon-pcie-mode"); + if (IS_ERR(syscon)) + return 0; + + mask = AM654_PCIE_DEV_TYPE_MASK; + + switch (mode) { + case DW_PCIE_RC_TYPE: + val = RC; + break; + case DW_PCIE_EP_TYPE: + val = EP; + break; + default: + dev_err(dev, "INVALID device type %d\n", mode); + return -EINVAL; + } + + ret = regmap_update_bits(syscon, 0, mask, val); + if (ret) { + dev_err(dev, "failed to set pcie mode\n"); + return ret; + } + + return 0; +} + +static void ks_pcie_set_link_speed(struct dw_pcie *pci, int link_speed) +{ + u32 val; + + dw_pcie_dbi_ro_wr_en(pci); + + val = dw_pcie_readl_dbi(pci, EXP_CAP_ID_OFFSET + PCI_EXP_LNKCAP); + if ((val & PCI_EXP_LNKCAP_SLS) != link_speed) { + val &= ~((u32)PCI_EXP_LNKCAP_SLS); + val |= link_speed; + dw_pcie_writel_dbi(pci, EXP_CAP_ID_OFFSET + PCI_EXP_LNKCAP, + val); + } + + val = dw_pcie_readl_dbi(pci, EXP_CAP_ID_OFFSET + PCI_EXP_LNKCTL2); + if ((val & PCI_EXP_LNKCAP_SLS) != link_speed) { + val &= ~((u32)PCI_EXP_LNKCAP_SLS); + val |= link_speed; + dw_pcie_writel_dbi(pci, EXP_CAP_ID_OFFSET + PCI_EXP_LNKCTL2, + val); + } + + dw_pcie_dbi_ro_wr_dis(pci); +} + +static const struct ks_pcie_of_data ks_pcie_rc_of_data = { + .host_ops = &ks_pcie_host_ops, + .version = 0x365A, +}; + +static const struct ks_pcie_of_data ks_pcie_am654_rc_of_data = { + .host_ops = &ks_pcie_am654_host_ops, + .mode = DW_PCIE_RC_TYPE, + .version = 0x490A, +}; + +static const struct ks_pcie_of_data ks_pcie_am654_ep_of_data = { + .ep_ops = &ks_pcie_am654_ep_ops, + .mode = DW_PCIE_EP_TYPE, + .version = 0x490A, +}; + +static const struct of_device_id ks_pcie_of_match[] = { + { + .type = "pci", + .data = &ks_pcie_rc_of_data, + .compatible = "ti,keystone-pcie", + }, + { + .data = &ks_pcie_am654_rc_of_data, + .compatible = "ti,am654-pcie-rc", + }, + { + .data = &ks_pcie_am654_ep_of_data, + .compatible = "ti,am654-pcie-ep", + }, + { }, +}; + static int __init ks_pcie_probe(struct platform_device *pdev) { + const struct dw_pcie_host_ops *host_ops; + const struct dw_pcie_ep_ops *ep_ops; struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; + const struct ks_pcie_of_data *data; + const struct of_device_id *match; + enum dw_pcie_device_mode mode; struct dw_pcie *pci; struct keystone_pcie *ks_pcie; struct device_link **link; + struct gpio_desc *gpiod; + void __iomem *atu_base; + struct resource *res; + unsigned int version; + void __iomem *base; u32 num_viewport; struct phy **phy; + int link_speed; u32 num_lanes; char name[10]; int ret; + int irq; int i; + match = of_match_device(of_match_ptr(ks_pcie_of_match), dev); + data = (struct ks_pcie_of_data *)match->data; + if (!data) + return -EINVAL; + + version = data->version; + host_ops = data->host_ops; + ep_ops = data->ep_ops; + mode = data->mode; + ks_pcie = devm_kzalloc(dev, sizeof(*ks_pcie), GFP_KERNEL); if (!ks_pcie) return -ENOMEM; @@ -917,12 +1225,38 @@ static int __init ks_pcie_probe(struct platform_device *pdev) if (!pci) return -ENOMEM; + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "app"); + ks_pcie->va_app_base = devm_ioremap_resource(dev, res); + if (IS_ERR(ks_pcie->va_app_base)) + return PTR_ERR(ks_pcie->va_app_base); + + ks_pcie->app = *res; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbics"); + base = devm_pci_remap_cfg_resource(dev, res); + if (IS_ERR(base)) + return PTR_ERR(base); + + if (of_device_is_compatible(np, "ti,am654-pcie-rc")) + ks_pcie->is_am6 = true; + + pci->dbi_base = base; + pci->dbi_base2 = base; pci->dev = dev; pci->ops = &ks_pcie_dw_pcie_ops; + pci->version = version; + + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + dev_err(dev, "missing IRQ resource: %d\n", irq); + return irq; + } - ret = of_property_read_u32(np, "num-viewport", &num_viewport); + ret = request_irq(irq, ks_pcie_err_irq_handler, IRQF_SHARED, + "ks-pcie-error-irq", ks_pcie); if (ret < 0) { - dev_err(dev, "unable to read *num-viewport* property\n"); + dev_err(dev, "failed to request error IRQ %d\n", + irq); return ret; } @@ -960,9 +1294,17 @@ static int __init ks_pcie_probe(struct platform_device *pdev) ks_pcie->pci = pci; ks_pcie->link = link; ks_pcie->num_lanes = num_lanes; - ks_pcie->num_viewport = num_viewport; ks_pcie->phy = phy; + gpiod = devm_gpiod_get_optional(dev, "reset", + GPIOD_OUT_LOW); + if (IS_ERR(gpiod)) { + ret = PTR_ERR(gpiod); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to get reset GPIO\n"); + goto err_link; + } + ret = ks_pcie_enable_phy(ks_pcie); if (ret) { dev_err(dev, "failed to enable phy\n"); @@ -977,9 +1319,79 @@ static int __init ks_pcie_probe(struct platform_device *pdev) goto err_get_sync; } - ret = ks_pcie_add_pcie_port(ks_pcie, pdev); - if (ret < 0) - goto err_get_sync; + if (pci->version >= 0x480A) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "atu"); + atu_base = devm_ioremap_resource(dev, res); + if (IS_ERR(atu_base)) { + ret = PTR_ERR(atu_base); + goto err_get_sync; + } + + pci->atu_base = atu_base; + + ret = ks_pcie_am654_set_mode(dev, mode); + if (ret < 0) + goto err_get_sync; + } else { + ret = ks_pcie_set_mode(dev); + if (ret < 0) + goto err_get_sync; + } + + link_speed = of_pci_get_max_link_speed(np); + if (link_speed < 0) + link_speed = 2; + + ks_pcie_set_link_speed(pci, link_speed); + + switch (mode) { + case DW_PCIE_RC_TYPE: + if (!IS_ENABLED(CONFIG_PCI_KEYSTONE_HOST)) { + ret = -ENODEV; + goto err_get_sync; + } + + ret = of_property_read_u32(np, "num-viewport", &num_viewport); + if (ret < 0) { + dev_err(dev, "unable to read *num-viewport* property\n"); + return ret; + } + + /* + * "Power Sequencing and Reset Signal Timings" table in + * PCI EXPRESS CARD ELECTROMECHANICAL SPECIFICATION, REV. 2.0 + * indicates PERST# should be deasserted after minimum of 100us + * once REFCLK is stable. The REFCLK to the connector in RC + * mode is selected while enabling the PHY. So deassert PERST# + * after 100 us. + */ + if (gpiod) { + usleep_range(100, 200); + gpiod_set_value_cansleep(gpiod, 1); + } + + ks_pcie->num_viewport = num_viewport; + pci->pp.ops = host_ops; + ret = ks_pcie_add_pcie_port(ks_pcie, pdev); + if (ret < 0) + goto err_get_sync; + break; + case DW_PCIE_EP_TYPE: + if (!IS_ENABLED(CONFIG_PCI_KEYSTONE_EP)) { + ret = -ENODEV; + goto err_get_sync; + } + + pci->ep.ops = ep_ops; + ret = ks_pcie_add_pcie_ep(ks_pcie, pdev); + if (ret < 0) + goto err_get_sync; + break; + default: + dev_err(dev, "INVALID device type %d\n", mode); + } + + ks_pcie_enable_error_irq(ks_pcie); return 0; diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c index a42c9c3ae1cc..ca9aa4501e7e 100644 --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c @@ -44,6 +44,7 @@ static const struct pci_epc_features ls_pcie_epc_features = { .linkup_notifier = false, .msi_capable = true, .msix_capable = false, + .bar_fixed_64bit = (1 << BAR_2) | (1 << BAR_4), }; static const struct pci_epc_features* @@ -79,7 +80,7 @@ static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } } -static struct dw_pcie_ep_ops pcie_ep_ops = { +static const struct dw_pcie_ep_ops pcie_ep_ops = { .ep_init = ls_pcie_ep_init, .raise_irq = ls_pcie_ep_raise_irq, .get_features = ls_pcie_ep_get_features, diff --git a/drivers/pci/controller/dwc/pci-layerscape.c b/drivers/pci/controller/dwc/pci-layerscape.c index ce45bde29bf8..3a5fa26d5e56 100644 --- a/drivers/pci/controller/dwc/pci-layerscape.c +++ b/drivers/pci/controller/dwc/pci-layerscape.c @@ -201,6 +201,7 @@ static int ls_pcie_msi_host_init(struct pcie_port *pp) return -EINVAL; } + of_node_put(msi_node); return 0; } diff --git a/drivers/pci/controller/dwc/pcie-al.c b/drivers/pci/controller/dwc/pcie-al.c new file mode 100644 index 000000000000..1eeda2f6371f --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-al.c @@ -0,0 +1,458 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for Amazon's Annapurna Labs IP (used in chips + * such as Graviton and Alpine) + * + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Author: Jonathan Chocron <jonnyc@amazon.com> + */ + +#include <linux/pci.h> +#include <linux/pci-ecam.h> +#include <linux/pci-acpi.h> +#include "../../pci.h" + +#if defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS) + +struct al_pcie_acpi { + void __iomem *dbi_base; +}; + +static void __iomem *al_pcie_map_bus(struct pci_bus *bus, unsigned int devfn, + int where) +{ + struct pci_config_window *cfg = bus->sysdata; + struct al_pcie_acpi *pcie = cfg->priv; + void __iomem *dbi_base = pcie->dbi_base; + + if (bus->number == cfg->busr.start) { + /* + * The DW PCIe core doesn't filter out transactions to other + * devices/functions on the root bus num, so we do this here. + */ + if (PCI_SLOT(devfn) > 0) + return NULL; + else + return dbi_base + where; + } + + return pci_ecam_map_bus(bus, devfn, where); +} + +static int al_pcie_init(struct pci_config_window *cfg) +{ + struct device *dev = cfg->parent; + struct acpi_device *adev = to_acpi_device(dev); + struct acpi_pci_root *root = acpi_driver_data(adev); + struct al_pcie_acpi *al_pcie; + struct resource *res; + int ret; + + al_pcie = devm_kzalloc(dev, sizeof(*al_pcie), GFP_KERNEL); + if (!al_pcie) + return -ENOMEM; + + res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL); + if (!res) + return -ENOMEM; + + ret = acpi_get_rc_resources(dev, "AMZN0001", root->segment, res); + if (ret) { + dev_err(dev, "can't get rc dbi base address for SEG %d\n", + root->segment); + return ret; + } + + dev_dbg(dev, "Root port dbi res: %pR\n", res); + + al_pcie->dbi_base = devm_pci_remap_cfg_resource(dev, res); + if (IS_ERR(al_pcie->dbi_base)) { + long err = PTR_ERR(al_pcie->dbi_base); + + dev_err(dev, "couldn't remap dbi base %pR (err:%ld)\n", + res, err); + return err; + } + + cfg->priv = al_pcie; + + return 0; +} + +struct pci_ecam_ops al_pcie_ops = { + .bus_shift = 20, + .init = al_pcie_init, + .pci_ops = { + .map_bus = al_pcie_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, + } +}; + +#endif /* defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS) */ + +#ifdef CONFIG_PCIE_AL + +#include <linux/of_pci.h> +#include "pcie-designware.h" + +#define AL_PCIE_REV_ID_2 2 +#define AL_PCIE_REV_ID_3 3 +#define AL_PCIE_REV_ID_4 4 + +#define AXI_BASE_OFFSET 0x0 + +#define DEVICE_ID_OFFSET 0x16c + +#define DEVICE_REV_ID 0x0 +#define DEVICE_REV_ID_DEV_ID_MASK GENMASK(31, 16) + +#define DEVICE_REV_ID_DEV_ID_X4 0 +#define DEVICE_REV_ID_DEV_ID_X8 2 +#define DEVICE_REV_ID_DEV_ID_X16 4 + +#define OB_CTRL_REV1_2_OFFSET 0x0040 +#define OB_CTRL_REV3_5_OFFSET 0x0030 + +#define CFG_TARGET_BUS 0x0 +#define CFG_TARGET_BUS_MASK_MASK GENMASK(7, 0) +#define CFG_TARGET_BUS_BUSNUM_MASK GENMASK(15, 8) + +#define CFG_CONTROL 0x4 +#define CFG_CONTROL_SUBBUS_MASK GENMASK(15, 8) +#define CFG_CONTROL_SEC_BUS_MASK GENMASK(23, 16) + +struct al_pcie_reg_offsets { + unsigned int ob_ctrl; +}; + +struct al_pcie_target_bus_cfg { + u8 reg_val; + u8 reg_mask; + u8 ecam_mask; +}; + +struct al_pcie { + struct dw_pcie *pci; + void __iomem *controller_base; /* base of PCIe unit (not DW core) */ + struct device *dev; + resource_size_t ecam_size; + unsigned int controller_rev_id; + struct al_pcie_reg_offsets reg_offsets; + struct al_pcie_target_bus_cfg target_bus_cfg; +}; + +#define PCIE_ECAM_DEVFN(x) (((x) & 0xff) << 12) + +#define to_al_pcie(x) dev_get_drvdata((x)->dev) + +static inline u32 al_pcie_controller_readl(struct al_pcie *pcie, u32 offset) +{ + return readl_relaxed(pcie->controller_base + offset); +} + +static inline void al_pcie_controller_writel(struct al_pcie *pcie, u32 offset, + u32 val) +{ + writel_relaxed(val, pcie->controller_base + offset); +} + +static int al_pcie_rev_id_get(struct al_pcie *pcie, unsigned int *rev_id) +{ + u32 dev_rev_id_val; + u32 dev_id_val; + + dev_rev_id_val = al_pcie_controller_readl(pcie, AXI_BASE_OFFSET + + DEVICE_ID_OFFSET + + DEVICE_REV_ID); + dev_id_val = FIELD_GET(DEVICE_REV_ID_DEV_ID_MASK, dev_rev_id_val); + + switch (dev_id_val) { + case DEVICE_REV_ID_DEV_ID_X4: + *rev_id = AL_PCIE_REV_ID_2; + break; + case DEVICE_REV_ID_DEV_ID_X8: + *rev_id = AL_PCIE_REV_ID_3; + break; + case DEVICE_REV_ID_DEV_ID_X16: + *rev_id = AL_PCIE_REV_ID_4; + break; + default: + dev_err(pcie->dev, "Unsupported dev_id_val (0x%x)\n", + dev_id_val); + return -EINVAL; + } + + dev_dbg(pcie->dev, "dev_id_val: 0x%x\n", dev_id_val); + + return 0; +} + +static int al_pcie_reg_offsets_set(struct al_pcie *pcie) +{ + switch (pcie->controller_rev_id) { + case AL_PCIE_REV_ID_2: + pcie->reg_offsets.ob_ctrl = OB_CTRL_REV1_2_OFFSET; + break; + case AL_PCIE_REV_ID_3: + case AL_PCIE_REV_ID_4: + pcie->reg_offsets.ob_ctrl = OB_CTRL_REV3_5_OFFSET; + break; + default: + dev_err(pcie->dev, "Unsupported controller rev_id: 0x%x\n", + pcie->controller_rev_id); + return -EINVAL; + } + + return 0; +} + +static inline void al_pcie_target_bus_set(struct al_pcie *pcie, + u8 target_bus, + u8 mask_target_bus) +{ + u32 reg; + + reg = FIELD_PREP(CFG_TARGET_BUS_MASK_MASK, mask_target_bus) | + FIELD_PREP(CFG_TARGET_BUS_BUSNUM_MASK, target_bus); + + al_pcie_controller_writel(pcie, AXI_BASE_OFFSET + + pcie->reg_offsets.ob_ctrl + CFG_TARGET_BUS, + reg); +} + +static void __iomem *al_pcie_conf_addr_map(struct al_pcie *pcie, + unsigned int busnr, + unsigned int devfn) +{ + struct al_pcie_target_bus_cfg *target_bus_cfg = &pcie->target_bus_cfg; + unsigned int busnr_ecam = busnr & target_bus_cfg->ecam_mask; + unsigned int busnr_reg = busnr & target_bus_cfg->reg_mask; + struct pcie_port *pp = &pcie->pci->pp; + void __iomem *pci_base_addr; + + pci_base_addr = (void __iomem *)((uintptr_t)pp->va_cfg0_base + + (busnr_ecam << 20) + + PCIE_ECAM_DEVFN(devfn)); + + if (busnr_reg != target_bus_cfg->reg_val) { + dev_dbg(pcie->pci->dev, "Changing target bus busnum val from 0x%x to 0x%x\n", + target_bus_cfg->reg_val, busnr_reg); + target_bus_cfg->reg_val = busnr_reg; + al_pcie_target_bus_set(pcie, + target_bus_cfg->reg_val, + target_bus_cfg->reg_mask); + } + + return pci_base_addr; +} + +static int al_pcie_rd_other_conf(struct pcie_port *pp, struct pci_bus *bus, + unsigned int devfn, int where, int size, + u32 *val) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct al_pcie *pcie = to_al_pcie(pci); + unsigned int busnr = bus->number; + void __iomem *pci_addr; + int rc; + + pci_addr = al_pcie_conf_addr_map(pcie, busnr, devfn); + + rc = dw_pcie_read(pci_addr + where, size, val); + + dev_dbg(pci->dev, "%d-byte config read from %04x:%02x:%02x.%d offset 0x%x (pci_addr: 0x%px) - val:0x%x\n", + size, pci_domain_nr(bus), bus->number, + PCI_SLOT(devfn), PCI_FUNC(devfn), where, + (pci_addr + where), *val); + + return rc; +} + +static int al_pcie_wr_other_conf(struct pcie_port *pp, struct pci_bus *bus, + unsigned int devfn, int where, int size, + u32 val) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct al_pcie *pcie = to_al_pcie(pci); + unsigned int busnr = bus->number; + void __iomem *pci_addr; + int rc; + + pci_addr = al_pcie_conf_addr_map(pcie, busnr, devfn); + + rc = dw_pcie_write(pci_addr + where, size, val); + + dev_dbg(pci->dev, "%d-byte config write to %04x:%02x:%02x.%d offset 0x%x (pci_addr: 0x%px) - val:0x%x\n", + size, pci_domain_nr(bus), bus->number, + PCI_SLOT(devfn), PCI_FUNC(devfn), where, + (pci_addr + where), val); + + return rc; +} + +static void al_pcie_config_prepare(struct al_pcie *pcie) +{ + struct al_pcie_target_bus_cfg *target_bus_cfg; + struct pcie_port *pp = &pcie->pci->pp; + unsigned int ecam_bus_mask; + u32 cfg_control_offset; + u8 subordinate_bus; + u8 secondary_bus; + u32 cfg_control; + u32 reg; + + target_bus_cfg = &pcie->target_bus_cfg; + + ecam_bus_mask = (pcie->ecam_size >> 20) - 1; + if (ecam_bus_mask > 255) { + dev_warn(pcie->dev, "ECAM window size is larger than 256MB. Cutting off at 256\n"); + ecam_bus_mask = 255; + } + + /* This portion is taken from the transaction address */ + target_bus_cfg->ecam_mask = ecam_bus_mask; + /* This portion is taken from the cfg_target_bus reg */ + target_bus_cfg->reg_mask = ~target_bus_cfg->ecam_mask; + target_bus_cfg->reg_val = pp->busn->start & target_bus_cfg->reg_mask; + + al_pcie_target_bus_set(pcie, target_bus_cfg->reg_val, + target_bus_cfg->reg_mask); + + secondary_bus = pp->busn->start + 1; + subordinate_bus = pp->busn->end; + + /* Set the valid values of secondary and subordinate buses */ + cfg_control_offset = AXI_BASE_OFFSET + pcie->reg_offsets.ob_ctrl + + CFG_CONTROL; + + cfg_control = al_pcie_controller_readl(pcie, cfg_control_offset); + + reg = cfg_control & + ~(CFG_CONTROL_SEC_BUS_MASK | CFG_CONTROL_SUBBUS_MASK); + + reg |= FIELD_PREP(CFG_CONTROL_SUBBUS_MASK, subordinate_bus) | + FIELD_PREP(CFG_CONTROL_SEC_BUS_MASK, secondary_bus); + + al_pcie_controller_writel(pcie, cfg_control_offset, reg); +} + +static int al_pcie_host_init(struct pcie_port *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct al_pcie *pcie = to_al_pcie(pci); + int rc; + + rc = al_pcie_rev_id_get(pcie, &pcie->controller_rev_id); + if (rc) + return rc; + + rc = al_pcie_reg_offsets_set(pcie); + if (rc) + return rc; + + al_pcie_config_prepare(pcie); + + return 0; +} + +static const struct dw_pcie_host_ops al_pcie_host_ops = { + .rd_other_conf = al_pcie_rd_other_conf, + .wr_other_conf = al_pcie_wr_other_conf, + .host_init = al_pcie_host_init, +}; + +static int al_add_pcie_port(struct pcie_port *pp, + struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + int ret; + + pp->ops = &al_pcie_host_ops; + + ret = dw_pcie_host_init(pp); + if (ret) { + dev_err(dev, "failed to initialize host\n"); + return ret; + } + + return 0; +} + +static const struct dw_pcie_ops dw_pcie_ops = { +}; + +static int al_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct resource *controller_res; + struct resource *ecam_res; + struct resource *dbi_res; + struct al_pcie *al_pcie; + struct dw_pcie *pci; + + al_pcie = devm_kzalloc(dev, sizeof(*al_pcie), GFP_KERNEL); + if (!al_pcie) + return -ENOMEM; + + pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); + if (!pci) + return -ENOMEM; + + pci->dev = dev; + pci->ops = &dw_pcie_ops; + + al_pcie->pci = pci; + al_pcie->dev = dev; + + dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi"); + pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_res); + if (IS_ERR(pci->dbi_base)) { + dev_err(dev, "couldn't remap dbi base %pR\n", dbi_res); + return PTR_ERR(pci->dbi_base); + } + + ecam_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config"); + if (!ecam_res) { + dev_err(dev, "couldn't find 'config' reg in DT\n"); + return -ENOENT; + } + al_pcie->ecam_size = resource_size(ecam_res); + + controller_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "controller"); + al_pcie->controller_base = devm_ioremap_resource(dev, controller_res); + if (IS_ERR(al_pcie->controller_base)) { + dev_err(dev, "couldn't remap controller base %pR\n", + controller_res); + return PTR_ERR(al_pcie->controller_base); + } + + dev_dbg(dev, "From DT: dbi_base: %pR, controller_base: %pR\n", + dbi_res, controller_res); + + platform_set_drvdata(pdev, al_pcie); + + return al_add_pcie_port(&pci->pp, pdev); +} + +static const struct of_device_id al_pcie_of_match[] = { + { .compatible = "amazon,al-alpine-v2-pcie", + }, + { .compatible = "amazon,al-alpine-v3-pcie", + }, + {}, +}; + +static struct platform_driver al_pcie_driver = { + .driver = { + .name = "al-pcie", + .of_match_table = al_pcie_of_match, + .suppress_bind_attrs = true, + }, + .probe = al_pcie_probe, +}; +builtin_platform_driver(al_pcie_driver); + +#endif /* CONFIG_PCIE_AL*/ diff --git a/drivers/pci/controller/dwc/pcie-armada8k.c b/drivers/pci/controller/dwc/pcie-armada8k.c index 0c389a30ef5d..49596547e8c2 100644 --- a/drivers/pci/controller/dwc/pcie-armada8k.c +++ b/drivers/pci/controller/dwc/pcie-armada8k.c @@ -25,10 +25,14 @@ #include "pcie-designware.h" +#define ARMADA8K_PCIE_MAX_LANES PCIE_LNK_X4 + struct armada8k_pcie { struct dw_pcie *pci; struct clk *clk; struct clk *clk_reg; + struct phy *phy[ARMADA8K_PCIE_MAX_LANES]; + unsigned int phy_count; }; #define PCIE_VENDOR_REGS_OFFSET 0x8000 @@ -55,7 +59,7 @@ struct armada8k_pcie { #define PCIE_ARUSER_REG (PCIE_VENDOR_REGS_OFFSET + 0x5C) #define PCIE_AWUSER_REG (PCIE_VENDOR_REGS_OFFSET + 0x60) /* - * AR/AW Cache defauls: Normal memory, Write-Back, Read / Write + * AR/AW Cache defaults: Normal memory, Write-Back, Read / Write * allocate */ #define ARCACHE_DEFAULT_VALUE 0x3511 @@ -67,6 +71,75 @@ struct armada8k_pcie { #define to_armada8k_pcie(x) dev_get_drvdata((x)->dev) +static void armada8k_pcie_disable_phys(struct armada8k_pcie *pcie) +{ + int i; + + for (i = 0; i < ARMADA8K_PCIE_MAX_LANES; i++) { + phy_power_off(pcie->phy[i]); + phy_exit(pcie->phy[i]); + } +} + +static int armada8k_pcie_enable_phys(struct armada8k_pcie *pcie) +{ + int ret; + int i; + + for (i = 0; i < ARMADA8K_PCIE_MAX_LANES; i++) { + ret = phy_init(pcie->phy[i]); + if (ret) + return ret; + + ret = phy_set_mode_ext(pcie->phy[i], PHY_MODE_PCIE, + pcie->phy_count); + if (ret) { + phy_exit(pcie->phy[i]); + return ret; + } + + ret = phy_power_on(pcie->phy[i]); + if (ret) { + phy_exit(pcie->phy[i]); + return ret; + } + } + + return 0; +} + +static int armada8k_pcie_setup_phys(struct armada8k_pcie *pcie) +{ + struct dw_pcie *pci = pcie->pci; + struct device *dev = pci->dev; + struct device_node *node = dev->of_node; + int ret = 0; + int i; + + for (i = 0; i < ARMADA8K_PCIE_MAX_LANES; i++) { + pcie->phy[i] = devm_of_phy_get_by_index(dev, node, i); + if (IS_ERR(pcie->phy[i])) { + if (PTR_ERR(pcie->phy[i]) != -ENODEV) + return PTR_ERR(pcie->phy[i]); + + pcie->phy[i] = NULL; + continue; + } + + pcie->phy_count++; + } + + /* Old bindings miss the PHY handle, so just warn if there is no PHY */ + if (!pcie->phy_count) + dev_warn(dev, "No available PHY\n"); + + ret = armada8k_pcie_enable_phys(pcie); + if (ret) + dev_err(dev, "Failed to initialize PHY(s) (%d)\n", ret); + + return ret; +} + static int armada8k_pcie_link_up(struct dw_pcie *pci) { u32 reg; @@ -249,14 +322,20 @@ static int armada8k_pcie_probe(struct platform_device *pdev) goto fail_clkreg; } + ret = armada8k_pcie_setup_phys(pcie); + if (ret) + goto fail_clkreg; + platform_set_drvdata(pdev, pcie); ret = armada8k_add_pcie_port(pcie, pdev); if (ret) - goto fail_clkreg; + goto disable_phy; return 0; +disable_phy: + armada8k_pcie_disable_phys(pcie); fail_clkreg: clk_disable_unprepare(pcie->clk_reg); fail: diff --git a/drivers/pci/controller/dwc/pcie-artpec6.c b/drivers/pci/controller/dwc/pcie-artpec6.c index dba83abfe764..d00252bd8fae 100644 --- a/drivers/pci/controller/dwc/pcie-artpec6.c +++ b/drivers/pci/controller/dwc/pcie-artpec6.c @@ -444,7 +444,7 @@ static int artpec6_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no, return 0; } -static struct dw_pcie_ep_ops pcie_ep_ops = { +static const struct dw_pcie_ep_ops pcie_ep_ops = { .ep_init = artpec6_pcie_ep_init, .raise_irq = artpec6_pcie_raise_irq, }; diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index 24f5a775ad34..3dd2e2697294 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -40,39 +40,6 @@ void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) __dw_pcie_ep_reset_bar(pci, bar, 0); } -static u8 __dw_pcie_ep_find_next_cap(struct dw_pcie *pci, u8 cap_ptr, - u8 cap) -{ - u8 cap_id, next_cap_ptr; - u16 reg; - - reg = dw_pcie_readw_dbi(pci, cap_ptr); - next_cap_ptr = (reg & 0xff00) >> 8; - cap_id = (reg & 0x00ff); - - if (!next_cap_ptr || cap_id > PCI_CAP_ID_MAX) - return 0; - - if (cap_id == cap) - return cap_ptr; - - return __dw_pcie_ep_find_next_cap(pci, next_cap_ptr, cap); -} - -static u8 dw_pcie_ep_find_capability(struct dw_pcie *pci, u8 cap) -{ - u8 next_cap_ptr; - u16 reg; - - reg = dw_pcie_readw_dbi(pci, PCI_CAPABILITY_LIST); - next_cap_ptr = (reg & 0x00ff); - - if (!next_cap_ptr) - return 0; - - return __dw_pcie_ep_find_next_cap(pci, next_cap_ptr, cap); -} - static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no, struct pci_epf_header *hdr) { @@ -397,6 +364,7 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, { struct dw_pcie *pci = to_dw_pcie_from_ep(ep); struct pci_epc *epc = ep->epc; + unsigned int aligned_offset; u16 msg_ctrl, msg_data; u32 msg_addr_lower, msg_addr_upper, reg; u64 msg_addr; @@ -422,13 +390,15 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, reg = ep->msi_cap + PCI_MSI_DATA_32; msg_data = dw_pcie_readw_dbi(pci, reg); } - msg_addr = ((u64) msg_addr_upper) << 32 | msg_addr_lower; + aligned_offset = msg_addr_lower & (epc->mem->page_size - 1); + msg_addr = ((u64)msg_addr_upper) << 32 | + (msg_addr_lower & ~aligned_offset); ret = dw_pcie_ep_map_addr(epc, func_no, ep->msi_mem_phys, msg_addr, epc->mem->page_size); if (ret) return ret; - writel(msg_data | (interrupt_num - 1), ep->msi_mem); + writel(msg_data | (interrupt_num - 1), ep->msi_mem + aligned_offset); dw_pcie_ep_unmap_addr(epc, func_no, ep->msi_mem_phys); @@ -504,10 +474,33 @@ void dw_pcie_ep_exit(struct dw_pcie_ep *ep) pci_epc_mem_exit(epc); } +static unsigned int dw_pcie_ep_find_ext_capability(struct dw_pcie *pci, int cap) +{ + u32 header; + int pos = PCI_CFG_SPACE_SIZE; + + while (pos) { + header = dw_pcie_readl_dbi(pci, pos); + if (PCI_EXT_CAP_ID(header) == cap) + return pos; + + pos = PCI_EXT_CAP_NEXT(header); + if (!pos) + break; + } + + return 0; +} + int dw_pcie_ep_init(struct dw_pcie_ep *ep) { + int i; int ret; + u32 reg; void *addr; + u8 hdr_type; + unsigned int nbars; + unsigned int offset; struct pci_epc *epc; struct dw_pcie *pci = to_dw_pcie_from_ep(ep); struct device *dev = pci->dev; @@ -517,10 +510,6 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) dev_err(dev, "dbi_base/dbi_base2 is not populated\n"); return -EINVAL; } - if (pci->iatu_unroll_enabled && !pci->atu_base) { - dev_err(dev, "atu_base is not populated\n"); - return -EINVAL; - } ret = of_property_read_u32(np, "num-ib-windows", &ep->num_ib_windows); if (ret < 0) { @@ -574,6 +563,13 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) if (ep->ops->ep_init) ep->ops->ep_init(ep); + hdr_type = dw_pcie_readb_dbi(pci, PCI_HEADER_TYPE); + if (hdr_type != PCI_HEADER_TYPE_NORMAL) { + dev_err(pci->dev, "PCIe controller is not set to EP mode (hdr_type:0x%x)!\n", + hdr_type); + return -EIO; + } + ret = of_property_read_u8(np, "max-functions", &epc->max_functions); if (ret < 0) epc->max_functions = 1; @@ -591,9 +587,21 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) dev_err(dev, "Failed to reserve memory for MSI/MSI-X\n"); return -ENOMEM; } - ep->msi_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSI); + ep->msi_cap = dw_pcie_find_capability(pci, PCI_CAP_ID_MSI); + + ep->msix_cap = dw_pcie_find_capability(pci, PCI_CAP_ID_MSIX); - ep->msix_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSIX); + offset = dw_pcie_ep_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR); + if (offset) { + reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL); + nbars = (reg & PCI_REBAR_CTRL_NBAR_MASK) >> + PCI_REBAR_CTRL_NBAR_SHIFT; + + dw_pcie_dbi_ro_wr_en(pci); + for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL) + dw_pcie_writel_dbi(pci, offset + PCI_REBAR_CAP, 0x0); + dw_pcie_dbi_ro_wr_dis(pci); + } dw_pcie_setup(pci); diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c index 25087d3c9a82..0f36a926059a 100644 --- a/drivers/pci/controller/dwc/pcie-designware-host.c +++ b/drivers/pci/controller/dwc/pcie-designware-host.c @@ -126,18 +126,12 @@ static void dw_pci_setup_msi_msg(struct irq_data *d, struct msi_msg *msg) struct dw_pcie *pci = to_dw_pcie_from_pp(pp); u64 msi_target; - if (pp->ops->get_msi_addr) - msi_target = pp->ops->get_msi_addr(pp); - else - msi_target = (u64)pp->msi_data; + msi_target = (u64)pp->msi_data; msg->address_lo = lower_32_bits(msi_target); msg->address_hi = upper_32_bits(msi_target); - if (pp->ops->get_msi_data) - msg->data = pp->ops->get_msi_data(pp, d->hwirq); - else - msg->data = d->hwirq; + msg->data = d->hwirq; dev_dbg(pci->dev, "msi#%d address_hi %#x address_lo %#x\n", (int)d->hwirq, msg->address_hi, msg->address_lo); @@ -157,17 +151,13 @@ static void dw_pci_bottom_mask(struct irq_data *d) raw_spin_lock_irqsave(&pp->lock, flags); - if (pp->ops->msi_clear_irq) { - pp->ops->msi_clear_irq(pp, d->hwirq); - } else { - ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL; - res = ctrl * MSI_REG_CTRL_BLOCK_SIZE; - bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL; + ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL; + res = ctrl * MSI_REG_CTRL_BLOCK_SIZE; + bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL; - pp->irq_mask[ctrl] |= BIT(bit); - dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK + res, 4, - pp->irq_mask[ctrl]); - } + pp->irq_mask[ctrl] |= BIT(bit); + dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK + res, 4, + pp->irq_mask[ctrl]); raw_spin_unlock_irqrestore(&pp->lock, flags); } @@ -180,17 +170,13 @@ static void dw_pci_bottom_unmask(struct irq_data *d) raw_spin_lock_irqsave(&pp->lock, flags); - if (pp->ops->msi_set_irq) { - pp->ops->msi_set_irq(pp, d->hwirq); - } else { - ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL; - res = ctrl * MSI_REG_CTRL_BLOCK_SIZE; - bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL; + ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL; + res = ctrl * MSI_REG_CTRL_BLOCK_SIZE; + bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL; - pp->irq_mask[ctrl] &= ~BIT(bit); - dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK + res, 4, - pp->irq_mask[ctrl]); - } + pp->irq_mask[ctrl] &= ~BIT(bit); + dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK + res, 4, + pp->irq_mask[ctrl]); raw_spin_unlock_irqrestore(&pp->lock, flags); } @@ -199,20 +185,12 @@ static void dw_pci_bottom_ack(struct irq_data *d) { struct pcie_port *pp = irq_data_get_irq_chip_data(d); unsigned int res, bit, ctrl; - unsigned long flags; ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL; res = ctrl * MSI_REG_CTRL_BLOCK_SIZE; bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL; - raw_spin_lock_irqsave(&pp->lock, flags); - dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_STATUS + res, 4, BIT(bit)); - - if (pp->ops->msi_irq_ack) - pp->ops->msi_irq_ack(d->hwirq, pp); - - raw_spin_unlock_irqrestore(&pp->lock, flags); } static struct irq_chip dw_pci_msi_bottom_irq_chip = { @@ -245,7 +223,7 @@ static int dw_pcie_irq_domain_alloc(struct irq_domain *domain, for (i = 0; i < nr_irqs; i++) irq_domain_set_info(domain, virq + i, bit + i, - &dw_pci_msi_bottom_irq_chip, + pp->msi_irq_chip, pp, handle_edge_irq, NULL, NULL); @@ -298,25 +276,31 @@ int dw_pcie_allocate_domains(struct pcie_port *pp) void dw_pcie_free_msi(struct pcie_port *pp) { - irq_set_chained_handler(pp->msi_irq, NULL); - irq_set_handler_data(pp->msi_irq, NULL); + if (pp->msi_irq) { + irq_set_chained_handler(pp->msi_irq, NULL); + irq_set_handler_data(pp->msi_irq, NULL); + } irq_domain_remove(pp->msi_domain); irq_domain_remove(pp->irq_domain); + + if (pp->msi_page) + __free_page(pp->msi_page); } void dw_pcie_msi_init(struct pcie_port *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct device *dev = pci->dev; - struct page *page; u64 msi_target; - page = alloc_page(GFP_KERNEL); - pp->msi_data = dma_map_page(dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE); + pp->msi_page = alloc_page(GFP_KERNEL); + pp->msi_data = dma_map_page(dev, pp->msi_page, 0, PAGE_SIZE, + DMA_FROM_DEVICE); if (dma_mapping_error(dev, pp->msi_data)) { dev_err(dev, "Failed to map MSI data\n"); - __free_page(page); + __free_page(pp->msi_page); + pp->msi_page = NULL; return; } msi_target = (u64)pp->msi_data; @@ -327,6 +311,7 @@ void dw_pcie_msi_init(struct pcie_port *pp) dw_pcie_wr_own_conf(pp, PCIE_MSI_ADDR_HI, 4, upper_32_bits(msi_target)); } +EXPORT_SYMBOL_GPL(dw_pcie_msi_init); int dw_pcie_host_init(struct pcie_port *pp) { @@ -335,9 +320,10 @@ int dw_pcie_host_init(struct pcie_port *pp) struct device_node *np = dev->of_node; struct platform_device *pdev = to_platform_device(dev); struct resource_entry *win, *tmp; - struct pci_bus *bus, *child; + struct pci_bus *child; struct pci_host_bridge *bridge; struct resource *cfg_res; + u32 hdr_type; int ret; raw_spin_lock_init(&pci->pp.lock); @@ -352,7 +338,7 @@ int dw_pcie_host_init(struct pcie_port *pp) dev_err(dev, "Missing *config* reg space\n"); } - bridge = pci_alloc_host_bridge(0); + bridge = devm_pci_alloc_host_bridge(dev, 0); if (!bridge) return -ENOMEM; @@ -363,7 +349,7 @@ int dw_pcie_host_init(struct pcie_port *pp) ret = devm_request_pci_bus_resources(dev, &bridge->windows); if (ret) - goto error; + return ret; /* Get the I/O and memory ranges from DT */ resource_list_for_each_entry_safe(win, tmp, &bridge->windows) { @@ -407,8 +393,7 @@ int dw_pcie_host_init(struct pcie_port *pp) resource_size(pp->cfg)); if (!pci->dbi_base) { dev_err(dev, "Error with ioremap\n"); - ret = -ENOMEM; - goto error; + return -ENOMEM; } } @@ -419,8 +404,7 @@ int dw_pcie_host_init(struct pcie_port *pp) pp->cfg0_base, pp->cfg0_size); if (!pp->va_cfg0_base) { dev_err(dev, "Error with ioremap in function\n"); - ret = -ENOMEM; - goto error; + return -ENOMEM; } } @@ -430,8 +414,7 @@ int dw_pcie_host_init(struct pcie_port *pp) pp->cfg1_size); if (!pp->va_cfg1_base) { dev_err(dev, "Error with ioremap\n"); - ret = -ENOMEM; - goto error; + return -ENOMEM; } } @@ -439,7 +422,7 @@ int dw_pcie_host_init(struct pcie_port *pp) if (ret) pci->num_viewport = 2; - if (IS_ENABLED(CONFIG_PCI_MSI) && pci_msi_enabled()) { + if (pci_msi_enabled()) { /* * If a specific SoC driver needs to change the * default number of vectors, it needs to implement @@ -454,14 +437,16 @@ int dw_pcie_host_init(struct pcie_port *pp) pp->num_vectors == 0) { dev_err(dev, "Invalid number of vectors\n"); - goto error; + return -EINVAL; } } if (!pp->ops->msi_host_init) { + pp->msi_irq_chip = &dw_pci_msi_bottom_irq_chip; + ret = dw_pcie_allocate_domains(pp); if (ret) - goto error; + return ret; if (pp->msi_irq) irq_set_chained_handler_and_data(pp->msi_irq, @@ -470,14 +455,29 @@ int dw_pcie_host_init(struct pcie_port *pp) } else { ret = pp->ops->msi_host_init(pp); if (ret < 0) - goto error; + return ret; } } if (pp->ops->host_init) { ret = pp->ops->host_init(pp); if (ret) - goto error; + goto err_free_msi; + } + + ret = dw_pcie_rd_own_conf(pp, PCI_HEADER_TYPE, 1, &hdr_type); + if (ret != PCIBIOS_SUCCESSFUL) { + dev_err(pci->dev, "Failed reading PCI_HEADER_TYPE cfg space reg (ret: 0x%x)\n", + ret); + ret = pcibios_err_to_errno(ret); + goto err_free_msi; + } + if (hdr_type != PCI_HEADER_TYPE_BRIDGE) { + dev_err(pci->dev, + "PCIe controller is not set to bridge type (hdr_type: 0x%x)!\n", + hdr_type); + ret = -EIO; + goto err_free_msi; } pp->root_bus_nr = pp->busn->start; @@ -491,26 +491,37 @@ int dw_pcie_host_init(struct pcie_port *pp) ret = pci_scan_root_bus_bridge(bridge); if (ret) - goto error; + goto err_free_msi; - bus = bridge->bus; + pp->root_bus = bridge->bus; if (pp->ops->scan_bus) pp->ops->scan_bus(pp); - pci_bus_size_bridges(bus); - pci_bus_assign_resources(bus); + pci_bus_size_bridges(pp->root_bus); + pci_bus_assign_resources(pp->root_bus); - list_for_each_entry(child, &bus->children, node) + list_for_each_entry(child, &pp->root_bus->children, node) pcie_bus_configure_settings(child); - pci_bus_add_devices(bus); + pci_bus_add_devices(pp->root_bus); return 0; -error: - pci_free_host_bridge(bridge); +err_free_msi: + if (pci_msi_enabled() && !pp->ops->msi_host_init) + dw_pcie_free_msi(pp); return ret; } +EXPORT_SYMBOL_GPL(dw_pcie_host_init); + +void dw_pcie_host_deinit(struct pcie_port *pp) +{ + pci_stop_root_bus(pp->root_bus); + pci_remove_root_bus(pp->root_bus); + if (pci_msi_enabled() && !pp->ops->msi_host_init) + dw_pcie_free_msi(pp); +} +EXPORT_SYMBOL_GPL(dw_pcie_host_deinit); static int dw_pcie_access_other_conf(struct pcie_port *pp, struct pci_bus *bus, u32 devfn, int where, int size, u32 *val, @@ -628,35 +639,32 @@ static struct pci_ops dw_pcie_ops = { .write = dw_pcie_wr_conf, }; -static u8 dw_pcie_iatu_unroll_enabled(struct dw_pcie *pci) -{ - u32 val; - - val = dw_pcie_readl_dbi(pci, PCIE_ATU_VIEWPORT); - if (val == 0xffffffff) - return 1; - - return 0; -} - void dw_pcie_setup_rc(struct pcie_port *pp) { u32 val, ctrl, num_ctrls; struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - dw_pcie_setup(pci); + /* + * Enable DBI read-only registers for writing/updating configuration. + * Write permission gets disabled towards the end of this function. + */ + dw_pcie_dbi_ro_wr_en(pci); - num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL; + dw_pcie_setup(pci); - /* Initialize IRQ Status array */ - for (ctrl = 0; ctrl < num_ctrls; ctrl++) { - pp->irq_mask[ctrl] = ~0; - dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK + - (ctrl * MSI_REG_CTRL_BLOCK_SIZE), - 4, pp->irq_mask[ctrl]); - dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_ENABLE + - (ctrl * MSI_REG_CTRL_BLOCK_SIZE), - 4, ~0); + if (!pp->ops->msi_host_init) { + num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL; + + /* Initialize IRQ Status array */ + for (ctrl = 0; ctrl < num_ctrls; ctrl++) { + pp->irq_mask[ctrl] = ~0; + dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK + + (ctrl * MSI_REG_CTRL_BLOCK_SIZE), + 4, pp->irq_mask[ctrl]); + dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_ENABLE + + (ctrl * MSI_REG_CTRL_BLOCK_SIZE), + 4, ~0); + } } /* Setup RC BARs */ @@ -664,12 +672,10 @@ void dw_pcie_setup_rc(struct pcie_port *pp) dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_1, 0x00000000); /* Setup interrupt pins */ - dw_pcie_dbi_ro_wr_en(pci); val = dw_pcie_readl_dbi(pci, PCI_INTERRUPT_LINE); val &= 0xffff00ff; val |= 0x00000100; dw_pcie_writel_dbi(pci, PCI_INTERRUPT_LINE, val); - dw_pcie_dbi_ro_wr_dis(pci); /* Setup bus numbers */ val = dw_pcie_readl_dbi(pci, PCI_PRIMARY_BUS); @@ -690,14 +696,6 @@ void dw_pcie_setup_rc(struct pcie_port *pp) * we should not program the ATU here. */ if (!pp->ops->rd_other_conf) { - /* Get iATU unroll support */ - pci->iatu_unroll_enabled = dw_pcie_iatu_unroll_enabled(pci); - dev_dbg(pci->dev, "iATU unroll: %s\n", - pci->iatu_unroll_enabled ? "enabled" : "disabled"); - - if (pci->iatu_unroll_enabled && !pci->atu_base) - pci->atu_base = pci->dbi_base + DEFAULT_DBI_ATU_OFFSET; - dw_pcie_prog_outbound_atu(pci, PCIE_ATU_REGION_INDEX0, PCIE_ATU_TYPE_MEM, pp->mem_base, pp->mem_bus_addr, pp->mem_size); @@ -709,14 +707,13 @@ void dw_pcie_setup_rc(struct pcie_port *pp) dw_pcie_wr_own_conf(pp, PCI_BASE_ADDRESS_0, 4, 0); - /* Enable write permission for the DBI read-only register */ - dw_pcie_dbi_ro_wr_en(pci); /* Program correct class for RC */ dw_pcie_wr_own_conf(pp, PCI_CLASS_DEVICE, 2, PCI_CLASS_BRIDGE_PCI); - /* Better disable write permission right after the update */ - dw_pcie_dbi_ro_wr_dis(pci); dw_pcie_rd_own_conf(pp, PCIE_LINK_WIDTH_SPEED_CONTROL, 4, &val); val |= PORT_LOGIC_SPEED_CHANGE; dw_pcie_wr_own_conf(pp, PCIE_LINK_WIDTH_SPEED_CONTROL, 4, val); + + dw_pcie_dbi_ro_wr_dis(pci); } +EXPORT_SYMBOL_GPL(dw_pcie_setup_rc); diff --git a/drivers/pci/controller/dwc/pcie-designware-plat.c b/drivers/pci/controller/dwc/pcie-designware-plat.c index 932dbd0b34b6..b58fdcbc664b 100644 --- a/drivers/pci/controller/dwc/pcie-designware-plat.c +++ b/drivers/pci/controller/dwc/pcie-designware-plat.c @@ -106,7 +106,7 @@ dw_plat_pcie_get_features(struct dw_pcie_ep *ep) return &dw_plat_pcie_epc_features; } -static struct dw_pcie_ep_ops pcie_ep_ops = { +static const struct dw_pcie_ep_ops pcie_ep_ops = { .ep_init = dw_plat_pcie_ep_init, .raise_irq = dw_plat_pcie_ep_raise_irq, .get_features = dw_plat_pcie_get_features, diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index 31f6331ca46f..820488dfeaed 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -14,11 +14,85 @@ #include "pcie-designware.h" -/* PCIe Port Logic registers */ -#define PLR_OFFSET 0x700 -#define PCIE_PHY_DEBUG_R1 (PLR_OFFSET + 0x2c) -#define PCIE_PHY_DEBUG_R1_LINK_UP (0x1 << 4) -#define PCIE_PHY_DEBUG_R1_LINK_IN_TRAINING (0x1 << 29) +/* + * These interfaces resemble the pci_find_*capability() interfaces, but these + * are for configuring host controllers, which are bridges *to* PCI devices but + * are not PCI devices themselves. + */ +static u8 __dw_pcie_find_next_cap(struct dw_pcie *pci, u8 cap_ptr, + u8 cap) +{ + u8 cap_id, next_cap_ptr; + u16 reg; + + if (!cap_ptr) + return 0; + + reg = dw_pcie_readw_dbi(pci, cap_ptr); + cap_id = (reg & 0x00ff); + + if (cap_id > PCI_CAP_ID_MAX) + return 0; + + if (cap_id == cap) + return cap_ptr; + + next_cap_ptr = (reg & 0xff00) >> 8; + return __dw_pcie_find_next_cap(pci, next_cap_ptr, cap); +} + +u8 dw_pcie_find_capability(struct dw_pcie *pci, u8 cap) +{ + u8 next_cap_ptr; + u16 reg; + + reg = dw_pcie_readw_dbi(pci, PCI_CAPABILITY_LIST); + next_cap_ptr = (reg & 0x00ff); + + return __dw_pcie_find_next_cap(pci, next_cap_ptr, cap); +} +EXPORT_SYMBOL_GPL(dw_pcie_find_capability); + +static u16 dw_pcie_find_next_ext_capability(struct dw_pcie *pci, u16 start, + u8 cap) +{ + u32 header; + int ttl; + int pos = PCI_CFG_SPACE_SIZE; + + /* minimum 8 bytes per capability */ + ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8; + + if (start) + pos = start; + + header = dw_pcie_readl_dbi(pci, pos); + /* + * If we have no capabilities, this is indicated by cap ID, + * cap version and next pointer all being 0. + */ + if (header == 0) + return 0; + + while (ttl-- > 0) { + if (PCI_EXT_CAP_ID(header) == cap && pos != start) + return pos; + + pos = PCI_EXT_CAP_NEXT(header); + if (pos < PCI_CFG_SPACE_SIZE) + break; + + header = dw_pcie_readl_dbi(pci, pos); + } + + return 0; +} + +u16 dw_pcie_find_ext_capability(struct dw_pcie *pci, u8 cap) +{ + return dw_pcie_find_next_ext_capability(pci, 0, cap); +} +EXPORT_SYMBOL_GPL(dw_pcie_find_ext_capability); int dw_pcie_read(void __iomem *addr, int size, u32 *val) { @@ -40,6 +114,7 @@ int dw_pcie_read(void __iomem *addr, int size, u32 *val) return PCIBIOS_SUCCESSFUL; } +EXPORT_SYMBOL_GPL(dw_pcie_read); int dw_pcie_write(void __iomem *addr, int size, u32 val) { @@ -57,37 +132,96 @@ int dw_pcie_write(void __iomem *addr, int size, u32 val) return PCIBIOS_SUCCESSFUL; } +EXPORT_SYMBOL_GPL(dw_pcie_write); -u32 __dw_pcie_read_dbi(struct dw_pcie *pci, void __iomem *base, u32 reg, - size_t size) +u32 dw_pcie_read_dbi(struct dw_pcie *pci, u32 reg, size_t size) { int ret; u32 val; if (pci->ops->read_dbi) - return pci->ops->read_dbi(pci, base, reg, size); + return pci->ops->read_dbi(pci, pci->dbi_base, reg, size); - ret = dw_pcie_read(base + reg, size, &val); + ret = dw_pcie_read(pci->dbi_base + reg, size, &val); if (ret) dev_err(pci->dev, "Read DBI address failed\n"); return val; } +EXPORT_SYMBOL_GPL(dw_pcie_read_dbi); -void __dw_pcie_write_dbi(struct dw_pcie *pci, void __iomem *base, u32 reg, - size_t size, u32 val) +void dw_pcie_write_dbi(struct dw_pcie *pci, u32 reg, size_t size, u32 val) { int ret; if (pci->ops->write_dbi) { - pci->ops->write_dbi(pci, base, reg, size, val); + pci->ops->write_dbi(pci, pci->dbi_base, reg, size, val); return; } - ret = dw_pcie_write(base + reg, size, val); + ret = dw_pcie_write(pci->dbi_base + reg, size, val); if (ret) dev_err(pci->dev, "Write DBI address failed\n"); } +EXPORT_SYMBOL_GPL(dw_pcie_write_dbi); + +u32 dw_pcie_read_dbi2(struct dw_pcie *pci, u32 reg, size_t size) +{ + int ret; + u32 val; + + if (pci->ops->read_dbi2) + return pci->ops->read_dbi2(pci, pci->dbi_base2, reg, size); + + ret = dw_pcie_read(pci->dbi_base2 + reg, size, &val); + if (ret) + dev_err(pci->dev, "read DBI address failed\n"); + + return val; +} + +void dw_pcie_write_dbi2(struct dw_pcie *pci, u32 reg, size_t size, u32 val) +{ + int ret; + + if (pci->ops->write_dbi2) { + pci->ops->write_dbi2(pci, pci->dbi_base2, reg, size, val); + return; + } + + ret = dw_pcie_write(pci->dbi_base2 + reg, size, val); + if (ret) + dev_err(pci->dev, "write DBI address failed\n"); +} + +u32 dw_pcie_read_atu(struct dw_pcie *pci, u32 reg, size_t size) +{ + int ret; + u32 val; + + if (pci->ops->read_dbi) + return pci->ops->read_dbi(pci, pci->atu_base, reg, size); + + ret = dw_pcie_read(pci->atu_base + reg, size, &val); + if (ret) + dev_err(pci->dev, "Read ATU address failed\n"); + + return val; +} + +void dw_pcie_write_atu(struct dw_pcie *pci, u32 reg, size_t size, u32 val) +{ + int ret; + + if (pci->ops->write_dbi) { + pci->ops->write_dbi(pci, pci->atu_base, reg, size, val); + return; + } + + ret = dw_pcie_write(pci->atu_base + reg, size, val); + if (ret) + dev_err(pci->dev, "Write ATU address failed\n"); +} static u32 dw_pcie_readl_ob_unroll(struct dw_pcie *pci, u32 index, u32 reg) { @@ -322,10 +456,11 @@ int dw_pcie_wait_for_link(struct dw_pcie *pci) usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX); } - dev_err(pci->dev, "Phy link never came up\n"); + dev_info(pci->dev, "Phy link never came up\n"); return -ETIMEDOUT; } +EXPORT_SYMBOL_GPL(dw_pcie_wait_for_link); int dw_pcie_link_up(struct dw_pcie *pci) { @@ -334,9 +469,20 @@ int dw_pcie_link_up(struct dw_pcie *pci) if (pci->ops->link_up) return pci->ops->link_up(pci); - val = readl(pci->dbi_base + PCIE_PHY_DEBUG_R1); - return ((val & PCIE_PHY_DEBUG_R1_LINK_UP) && - (!(val & PCIE_PHY_DEBUG_R1_LINK_IN_TRAINING))); + val = readl(pci->dbi_base + PCIE_PORT_DEBUG1); + return ((val & PCIE_PORT_DEBUG1_LINK_UP) && + (!(val & PCIE_PORT_DEBUG1_LINK_IN_TRAINING))); +} + +static u8 dw_pcie_iatu_unroll_enabled(struct dw_pcie *pci) +{ + u32 val; + + val = dw_pcie_readl_dbi(pci, PCIE_ATU_VIEWPORT); + if (val == 0xffffffff) + return 1; + + return 0; } void dw_pcie_setup(struct dw_pcie *pci) @@ -347,9 +493,21 @@ void dw_pcie_setup(struct dw_pcie *pci) struct device *dev = pci->dev; struct device_node *np = dev->of_node; + if (pci->version >= 0x480A || (!pci->version && + dw_pcie_iatu_unroll_enabled(pci))) { + pci->iatu_unroll_enabled = true; + if (!pci->atu_base) + pci->atu_base = pci->dbi_base + DEFAULT_DBI_ATU_OFFSET; + } + dev_dbg(pci->dev, "iATU unroll: %s\n", pci->iatu_unroll_enabled ? + "enabled" : "disabled"); + + ret = of_property_read_u32(np, "num-lanes", &lanes); - if (ret) - lanes = 0; + if (ret) { + dev_dbg(pci->dev, "property num-lanes isn't found\n"); + return; + } /* Set the number of lanes */ val = dw_pcie_readl_dbi(pci, PCIE_PORT_LINK_CONTROL); @@ -391,4 +549,11 @@ void dw_pcie_setup(struct dw_pcie *pci) break; } dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val); + + if (of_property_read_bool(np, "snps,enable-cdm-check")) { + val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS); + val |= PCIE_PL_CHK_REG_CHK_REG_CONTINUOUS | + PCIE_PL_CHK_REG_CHK_REG_START; + dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, val); + } } diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index 377f4c0b52da..5a18e94e52c8 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -41,6 +41,9 @@ #define PCIE_PORT_DEBUG0 0x728 #define PORT_LOGIC_LTSSM_STATE_MASK 0x1f #define PORT_LOGIC_LTSSM_STATE_L0 0x11 +#define PCIE_PORT_DEBUG1 0x72C +#define PCIE_PORT_DEBUG1_LINK_UP BIT(4) +#define PCIE_PORT_DEBUG1_LINK_IN_TRAINING BIT(29) #define PCIE_LINK_WIDTH_SPEED_CONTROL 0x80C #define PORT_LOGIC_SPEED_CHANGE BIT(17) @@ -83,6 +86,15 @@ #define PCIE_MISC_CONTROL_1_OFF 0x8BC #define PCIE_DBI_RO_WR_EN BIT(0) +#define PCIE_PL_CHK_REG_CONTROL_STATUS 0xB20 +#define PCIE_PL_CHK_REG_CHK_REG_START BIT(0) +#define PCIE_PL_CHK_REG_CHK_REG_CONTINUOUS BIT(1) +#define PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR BIT(16) +#define PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR BIT(17) +#define PCIE_PL_CHK_REG_CHK_REG_COMPLETE BIT(18) + +#define PCIE_PL_CHK_REG_ERR_ADDR 0xB28 + /* * iATU Unroll-specific register definitions * From 4.80 core version the address translation will be made by unroll @@ -145,14 +157,9 @@ struct dw_pcie_host_ops { int (*wr_other_conf)(struct pcie_port *pp, struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val); int (*host_init)(struct pcie_port *pp); - void (*msi_set_irq)(struct pcie_port *pp, int irq); - void (*msi_clear_irq)(struct pcie_port *pp, int irq); - phys_addr_t (*get_msi_addr)(struct pcie_port *pp); - u32 (*get_msi_data)(struct pcie_port *pp, int pos); void (*scan_bus)(struct pcie_port *pp); void (*set_num_vectors)(struct pcie_port *pp); int (*msi_host_init)(struct pcie_port *pp); - void (*msi_irq_ack)(int irq, struct pcie_port *pp); }; struct pcie_port { @@ -179,8 +186,11 @@ struct pcie_port { struct irq_domain *irq_domain; struct irq_domain *msi_domain; dma_addr_t msi_data; + struct page *msi_page; + struct irq_chip *msi_irq_chip; u32 num_vectors; u32 irq_mask[MAX_MSI_CTRLS]; + struct pci_bus *root_bus; raw_spinlock_t lock; DECLARE_BITMAP(msi_irq_in_use, MAX_MSI_IRQS); }; @@ -200,7 +210,7 @@ struct dw_pcie_ep_ops { struct dw_pcie_ep { struct pci_epc *epc; - struct dw_pcie_ep_ops *ops; + const struct dw_pcie_ep_ops *ops; phys_addr_t phys_base; size_t addr_size; size_t page_size; @@ -222,6 +232,10 @@ struct dw_pcie_ops { size_t size); void (*write_dbi)(struct dw_pcie *pcie, void __iomem *base, u32 reg, size_t size, u32 val); + u32 (*read_dbi2)(struct dw_pcie *pcie, void __iomem *base, u32 reg, + size_t size); + void (*write_dbi2)(struct dw_pcie *pcie, void __iomem *base, u32 reg, + size_t size, u32 val); int (*link_up)(struct dw_pcie *pcie); int (*start_link)(struct dw_pcie *pcie); void (*stop_link)(struct dw_pcie *pcie); @@ -238,6 +252,7 @@ struct dw_pcie { struct pcie_port pp; struct dw_pcie_ep ep; const struct dw_pcie_ops *ops; + unsigned int version; }; #define to_dw_pcie_from_pp(port) container_of((port), struct dw_pcie, pp) @@ -245,13 +260,18 @@ struct dw_pcie { #define to_dw_pcie_from_ep(endpoint) \ container_of((endpoint), struct dw_pcie, ep) +u8 dw_pcie_find_capability(struct dw_pcie *pci, u8 cap); +u16 dw_pcie_find_ext_capability(struct dw_pcie *pci, u8 cap); + int dw_pcie_read(void __iomem *addr, int size, u32 *val); int dw_pcie_write(void __iomem *addr, int size, u32 val); -u32 __dw_pcie_read_dbi(struct dw_pcie *pci, void __iomem *base, u32 reg, - size_t size); -void __dw_pcie_write_dbi(struct dw_pcie *pci, void __iomem *base, u32 reg, - size_t size, u32 val); +u32 dw_pcie_read_dbi(struct dw_pcie *pci, u32 reg, size_t size); +void dw_pcie_write_dbi(struct dw_pcie *pci, u32 reg, size_t size, u32 val); +u32 dw_pcie_read_dbi2(struct dw_pcie *pci, u32 reg, size_t size); +void dw_pcie_write_dbi2(struct dw_pcie *pci, u32 reg, size_t size, u32 val); +u32 dw_pcie_read_atu(struct dw_pcie *pci, u32 reg, size_t size); +void dw_pcie_write_atu(struct dw_pcie *pci, u32 reg, size_t size, u32 val); int dw_pcie_link_up(struct dw_pcie *pci); int dw_pcie_wait_for_link(struct dw_pcie *pci); void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, @@ -265,52 +285,52 @@ void dw_pcie_setup(struct dw_pcie *pci); static inline void dw_pcie_writel_dbi(struct dw_pcie *pci, u32 reg, u32 val) { - __dw_pcie_write_dbi(pci, pci->dbi_base, reg, 0x4, val); + dw_pcie_write_dbi(pci, reg, 0x4, val); } static inline u32 dw_pcie_readl_dbi(struct dw_pcie *pci, u32 reg) { - return __dw_pcie_read_dbi(pci, pci->dbi_base, reg, 0x4); + return dw_pcie_read_dbi(pci, reg, 0x4); } static inline void dw_pcie_writew_dbi(struct dw_pcie *pci, u32 reg, u16 val) { - __dw_pcie_write_dbi(pci, pci->dbi_base, reg, 0x2, val); + dw_pcie_write_dbi(pci, reg, 0x2, val); } static inline u16 dw_pcie_readw_dbi(struct dw_pcie *pci, u32 reg) { - return __dw_pcie_read_dbi(pci, pci->dbi_base, reg, 0x2); + return dw_pcie_read_dbi(pci, reg, 0x2); } static inline void dw_pcie_writeb_dbi(struct dw_pcie *pci, u32 reg, u8 val) { - __dw_pcie_write_dbi(pci, pci->dbi_base, reg, 0x1, val); + dw_pcie_write_dbi(pci, reg, 0x1, val); } static inline u8 dw_pcie_readb_dbi(struct dw_pcie *pci, u32 reg) { - return __dw_pcie_read_dbi(pci, pci->dbi_base, reg, 0x1); + return dw_pcie_read_dbi(pci, reg, 0x1); } static inline void dw_pcie_writel_dbi2(struct dw_pcie *pci, u32 reg, u32 val) { - __dw_pcie_write_dbi(pci, pci->dbi_base2, reg, 0x4, val); + dw_pcie_write_dbi2(pci, reg, 0x4, val); } static inline u32 dw_pcie_readl_dbi2(struct dw_pcie *pci, u32 reg) { - return __dw_pcie_read_dbi(pci, pci->dbi_base2, reg, 0x4); + return dw_pcie_read_dbi2(pci, reg, 0x4); } static inline void dw_pcie_writel_atu(struct dw_pcie *pci, u32 reg, u32 val) { - __dw_pcie_write_dbi(pci, pci->atu_base, reg, 0x4, val); + dw_pcie_write_atu(pci, reg, 0x4, val); } static inline u32 dw_pcie_readl_atu(struct dw_pcie *pci, u32 reg) { - return __dw_pcie_read_dbi(pci, pci->atu_base, reg, 0x4); + return dw_pcie_read_atu(pci, reg, 0x4); } static inline void dw_pcie_dbi_ro_wr_en(struct dw_pcie *pci) @@ -341,6 +361,7 @@ void dw_pcie_msi_init(struct pcie_port *pp); void dw_pcie_free_msi(struct pcie_port *pp); void dw_pcie_setup_rc(struct pcie_port *pp); int dw_pcie_host_init(struct pcie_port *pp); +void dw_pcie_host_deinit(struct pcie_port *pp); int dw_pcie_allocate_domains(struct pcie_port *pp); #else static inline irqreturn_t dw_handle_msi_irq(struct pcie_port *pp) @@ -365,6 +386,10 @@ static inline int dw_pcie_host_init(struct pcie_port *pp) return 0; } +static inline void dw_pcie_host_deinit(struct pcie_port *pp) +{ +} + static inline int dw_pcie_allocate_domains(struct pcie_port *pp) { return 0; diff --git a/drivers/pci/controller/dwc/pcie-histb.c b/drivers/pci/controller/dwc/pcie-histb.c index 954bc2b74bbc..811b5c6d62ea 100644 --- a/drivers/pci/controller/dwc/pcie-histb.c +++ b/drivers/pci/controller/dwc/pcie-histb.c @@ -340,8 +340,8 @@ static int histb_pcie_probe(struct platform_device *pdev) hipcie->vpcie = devm_regulator_get_optional(dev, "vpcie"); if (IS_ERR(hipcie->vpcie)) { - if (PTR_ERR(hipcie->vpcie) == -EPROBE_DEFER) - return -EPROBE_DEFER; + if (PTR_ERR(hipcie->vpcie) != -ENODEV) + return PTR_ERR(hipcie->vpcie); hipcie->vpcie = NULL; } diff --git a/drivers/pci/controller/dwc/pcie-kirin.c b/drivers/pci/controller/dwc/pcie-kirin.c index 9b599296205d..c19617a912bd 100644 --- a/drivers/pci/controller/dwc/pcie-kirin.c +++ b/drivers/pci/controller/dwc/pcie-kirin.c @@ -2,7 +2,7 @@ /* * PCIe host controller driver for Kirin Phone SoCs * - * Copyright (C) 2017 Hilisicon Electronics Co., Ltd. + * Copyright (C) 2017 HiSilicon Electronics Co., Ltd. * http://www.huawei.com * * Author: Xiaowei Song <songxiaowei@huawei.com> @@ -436,7 +436,7 @@ static int kirin_pcie_host_init(struct pcie_port *pp) return 0; } -static struct dw_pcie_ops kirin_dw_pcie_ops = { +static const struct dw_pcie_ops kirin_dw_pcie_ops = { .read_dbi = kirin_pcie_read_dbi, .write_dbi = kirin_pcie_write_dbi, .link_up = kirin_pcie_link_up, diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index a7f703556790..7e581748ee9f 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -112,10 +112,10 @@ struct qcom_pcie_resources_2_3_2 { struct regulator_bulk_data supplies[QCOM_PCIE_2_3_2_MAX_SUPPLY]; }; +#define QCOM_PCIE_2_4_0_MAX_CLOCKS 4 struct qcom_pcie_resources_2_4_0 { - struct clk *aux_clk; - struct clk *master_clk; - struct clk *slave_clk; + struct clk_bulk_data clks[QCOM_PCIE_2_4_0_MAX_CLOCKS]; + int num_clks; struct reset_control *axi_m_reset; struct reset_control *axi_s_reset; struct reset_control *pipe_reset; @@ -178,6 +178,8 @@ static void qcom_ep_reset_assert(struct qcom_pcie *pcie) static void qcom_ep_reset_deassert(struct qcom_pcie *pcie) { + /* Ensure that PERST has been asserted for at least 100 ms */ + msleep(100); gpiod_set_value_cansleep(pcie->reset, 0); usleep_range(PERST_DELAY_US, PERST_DELAY_US + 500); } @@ -638,18 +640,20 @@ static int qcom_pcie_get_resources_2_4_0(struct qcom_pcie *pcie) struct qcom_pcie_resources_2_4_0 *res = &pcie->res.v2_4_0; struct dw_pcie *pci = pcie->pci; struct device *dev = pci->dev; + bool is_ipq = of_device_is_compatible(dev->of_node, "qcom,pcie-ipq4019"); + int ret; - res->aux_clk = devm_clk_get(dev, "aux"); - if (IS_ERR(res->aux_clk)) - return PTR_ERR(res->aux_clk); + res->clks[0].id = "aux"; + res->clks[1].id = "master_bus"; + res->clks[2].id = "slave_bus"; + res->clks[3].id = "iface"; - res->master_clk = devm_clk_get(dev, "master_bus"); - if (IS_ERR(res->master_clk)) - return PTR_ERR(res->master_clk); + /* qcom,pcie-ipq4019 is defined without "iface" */ + res->num_clks = is_ipq ? 3 : 4; - res->slave_clk = devm_clk_get(dev, "slave_bus"); - if (IS_ERR(res->slave_clk)) - return PTR_ERR(res->slave_clk); + ret = devm_clk_bulk_get(dev, res->num_clks, res->clks); + if (ret < 0) + return ret; res->axi_m_reset = devm_reset_control_get_exclusive(dev, "axi_m"); if (IS_ERR(res->axi_m_reset)) @@ -659,27 +663,33 @@ static int qcom_pcie_get_resources_2_4_0(struct qcom_pcie *pcie) if (IS_ERR(res->axi_s_reset)) return PTR_ERR(res->axi_s_reset); - res->pipe_reset = devm_reset_control_get_exclusive(dev, "pipe"); - if (IS_ERR(res->pipe_reset)) - return PTR_ERR(res->pipe_reset); - - res->axi_m_vmid_reset = devm_reset_control_get_exclusive(dev, - "axi_m_vmid"); - if (IS_ERR(res->axi_m_vmid_reset)) - return PTR_ERR(res->axi_m_vmid_reset); - - res->axi_s_xpu_reset = devm_reset_control_get_exclusive(dev, - "axi_s_xpu"); - if (IS_ERR(res->axi_s_xpu_reset)) - return PTR_ERR(res->axi_s_xpu_reset); - - res->parf_reset = devm_reset_control_get_exclusive(dev, "parf"); - if (IS_ERR(res->parf_reset)) - return PTR_ERR(res->parf_reset); - - res->phy_reset = devm_reset_control_get_exclusive(dev, "phy"); - if (IS_ERR(res->phy_reset)) - return PTR_ERR(res->phy_reset); + if (is_ipq) { + /* + * These resources relates to the PHY or are secure clocks, but + * are controlled here for IPQ4019 + */ + res->pipe_reset = devm_reset_control_get_exclusive(dev, "pipe"); + if (IS_ERR(res->pipe_reset)) + return PTR_ERR(res->pipe_reset); + + res->axi_m_vmid_reset = devm_reset_control_get_exclusive(dev, + "axi_m_vmid"); + if (IS_ERR(res->axi_m_vmid_reset)) + return PTR_ERR(res->axi_m_vmid_reset); + + res->axi_s_xpu_reset = devm_reset_control_get_exclusive(dev, + "axi_s_xpu"); + if (IS_ERR(res->axi_s_xpu_reset)) + return PTR_ERR(res->axi_s_xpu_reset); + + res->parf_reset = devm_reset_control_get_exclusive(dev, "parf"); + if (IS_ERR(res->parf_reset)) + return PTR_ERR(res->parf_reset); + + res->phy_reset = devm_reset_control_get_exclusive(dev, "phy"); + if (IS_ERR(res->phy_reset)) + return PTR_ERR(res->phy_reset); + } res->axi_m_sticky_reset = devm_reset_control_get_exclusive(dev, "axi_m_sticky"); @@ -699,9 +709,11 @@ static int qcom_pcie_get_resources_2_4_0(struct qcom_pcie *pcie) if (IS_ERR(res->ahb_reset)) return PTR_ERR(res->ahb_reset); - res->phy_ahb_reset = devm_reset_control_get_exclusive(dev, "phy_ahb"); - if (IS_ERR(res->phy_ahb_reset)) - return PTR_ERR(res->phy_ahb_reset); + if (is_ipq) { + res->phy_ahb_reset = devm_reset_control_get_exclusive(dev, "phy_ahb"); + if (IS_ERR(res->phy_ahb_reset)) + return PTR_ERR(res->phy_ahb_reset); + } return 0; } @@ -719,9 +731,7 @@ static void qcom_pcie_deinit_2_4_0(struct qcom_pcie *pcie) reset_control_assert(res->axi_m_sticky_reset); reset_control_assert(res->pwr_reset); reset_control_assert(res->ahb_reset); - clk_disable_unprepare(res->aux_clk); - clk_disable_unprepare(res->master_clk); - clk_disable_unprepare(res->slave_clk); + clk_bulk_disable_unprepare(res->num_clks, res->clks); } static int qcom_pcie_init_2_4_0(struct qcom_pcie *pcie) @@ -850,23 +860,9 @@ static int qcom_pcie_init_2_4_0(struct qcom_pcie *pcie) usleep_range(10000, 12000); - ret = clk_prepare_enable(res->aux_clk); - if (ret) { - dev_err(dev, "cannot prepare/enable iface clock\n"); - goto err_clk_aux; - } - - ret = clk_prepare_enable(res->master_clk); - if (ret) { - dev_err(dev, "cannot prepare/enable core clock\n"); - goto err_clk_axi_m; - } - - ret = clk_prepare_enable(res->slave_clk); - if (ret) { - dev_err(dev, "cannot prepare/enable phy clock\n"); - goto err_clk_axi_s; - } + ret = clk_bulk_prepare_enable(res->num_clks, res->clks); + if (ret) + goto err_clks; /* enable PCIe clocks and resets */ val = readl(pcie->parf + PCIE20_PARF_PHY_CTRL); @@ -891,11 +887,7 @@ static int qcom_pcie_init_2_4_0(struct qcom_pcie *pcie) return 0; -err_clk_axi_s: - clk_disable_unprepare(res->master_clk); -err_clk_axi_m: - clk_disable_unprepare(res->aux_clk); -err_clk_aux: +err_clks: reset_control_assert(res->ahb_reset); err_rst_ahb: reset_control_assert(res->pwr_reset); @@ -1129,25 +1121,8 @@ err_deinit: return ret; } -static int qcom_pcie_rd_own_conf(struct pcie_port *pp, int where, int size, - u32 *val) -{ - struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - - /* the device class is not reported correctly from the register */ - if (where == PCI_CLASS_REVISION && size == 4) { - *val = readl(pci->dbi_base + PCI_CLASS_REVISION); - *val &= 0xff; /* keep revision id */ - *val |= PCI_CLASS_BRIDGE_PCI << 16; - return PCIBIOS_SUCCESSFUL; - } - - return dw_pcie_read(pci->dbi_base + where, size, val); -} - static const struct dw_pcie_host_ops qcom_pcie_dw_ops = { .host_init = qcom_pcie_host_init, - .rd_own_conf = qcom_pcie_rd_own_conf, }; /* Qcom IP rev.: 2.1.0 Synopsys IP rev.: 4.01a */ @@ -1306,9 +1281,16 @@ static const struct of_device_id qcom_pcie_match[] = { { .compatible = "qcom,pcie-msm8996", .data = &ops_2_3_2 }, { .compatible = "qcom,pcie-ipq8074", .data = &ops_2_3_3 }, { .compatible = "qcom,pcie-ipq4019", .data = &ops_2_4_0 }, + { .compatible = "qcom,pcie-qcs404", .data = &ops_2_4_0 }, { } }; +static void qcom_fixup_class(struct pci_dev *dev) +{ + dev->class = PCI_CLASS_BRIDGE_PCI << 8; +} +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, PCI_ANY_ID, qcom_fixup_class); + static struct platform_driver qcom_pcie_driver = { .probe = qcom_pcie_probe, .driver = { diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c new file mode 100644 index 000000000000..f89f5acee72d --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-tegra194.c @@ -0,0 +1,1732 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * PCIe host controller driver for Tegra194 SoC + * + * Copyright (C) 2019 NVIDIA Corporation. + * + * Author: Vidya Sagar <vidyas@nvidia.com> + */ + +#include <linux/clk.h> +#include <linux/debugfs.h> +#include <linux/delay.h> +#include <linux/gpio.h> +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/of_gpio.h> +#include <linux/of_irq.h> +#include <linux/of_pci.h> +#include <linux/pci.h> +#include <linux/phy/phy.h> +#include <linux/pinctrl/consumer.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/random.h> +#include <linux/reset.h> +#include <linux/resource.h> +#include <linux/types.h> +#include "pcie-designware.h" +#include <soc/tegra/bpmp.h> +#include <soc/tegra/bpmp-abi.h> +#include "../../pci.h" + +#define APPL_PINMUX 0x0 +#define APPL_PINMUX_PEX_RST BIT(0) +#define APPL_PINMUX_CLKREQ_OVERRIDE_EN BIT(2) +#define APPL_PINMUX_CLKREQ_OVERRIDE BIT(3) +#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN BIT(4) +#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE BIT(5) +#define APPL_PINMUX_CLKREQ_OUT_OVRD_EN BIT(9) +#define APPL_PINMUX_CLKREQ_OUT_OVRD BIT(10) + +#define APPL_CTRL 0x4 +#define APPL_CTRL_SYS_PRE_DET_STATE BIT(6) +#define APPL_CTRL_LTSSM_EN BIT(7) +#define APPL_CTRL_HW_HOT_RST_EN BIT(20) +#define APPL_CTRL_HW_HOT_RST_MODE_MASK GENMASK(1, 0) +#define APPL_CTRL_HW_HOT_RST_MODE_SHIFT 22 +#define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST 0x1 + +#define APPL_INTR_EN_L0_0 0x8 +#define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN BIT(0) +#define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN BIT(4) +#define APPL_INTR_EN_L0_0_INT_INT_EN BIT(8) +#define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN BIT(19) +#define APPL_INTR_EN_L0_0_SYS_INTR_EN BIT(30) +#define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN BIT(31) + +#define APPL_INTR_STATUS_L0 0xC +#define APPL_INTR_STATUS_L0_LINK_STATE_INT BIT(0) +#define APPL_INTR_STATUS_L0_INT_INT BIT(8) +#define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT BIT(18) + +#define APPL_INTR_EN_L1_0_0 0x1C +#define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN BIT(1) + +#define APPL_INTR_STATUS_L1_0_0 0x20 +#define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED BIT(1) + +#define APPL_INTR_STATUS_L1_1 0x2C +#define APPL_INTR_STATUS_L1_2 0x30 +#define APPL_INTR_STATUS_L1_3 0x34 +#define APPL_INTR_STATUS_L1_6 0x3C +#define APPL_INTR_STATUS_L1_7 0x40 + +#define APPL_INTR_EN_L1_8_0 0x44 +#define APPL_INTR_EN_L1_8_BW_MGT_INT_EN BIT(2) +#define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN BIT(3) +#define APPL_INTR_EN_L1_8_INTX_EN BIT(11) +#define APPL_INTR_EN_L1_8_AER_INT_EN BIT(15) + +#define APPL_INTR_STATUS_L1_8_0 0x4C +#define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK GENMASK(11, 6) +#define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS BIT(2) +#define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS BIT(3) + +#define APPL_INTR_STATUS_L1_9 0x54 +#define APPL_INTR_STATUS_L1_10 0x58 +#define APPL_INTR_STATUS_L1_11 0x64 +#define APPL_INTR_STATUS_L1_13 0x74 +#define APPL_INTR_STATUS_L1_14 0x78 +#define APPL_INTR_STATUS_L1_15 0x7C +#define APPL_INTR_STATUS_L1_17 0x88 + +#define APPL_INTR_EN_L1_18 0x90 +#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT BIT(2) +#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR BIT(1) +#define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0) + +#define APPL_INTR_STATUS_L1_18 0x94 +#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT BIT(2) +#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR BIT(1) +#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0) + +#define APPL_MSI_CTRL_2 0xB0 + +#define APPL_LTR_MSG_1 0xC4 +#define LTR_MSG_REQ BIT(15) +#define LTR_MST_NO_SNOOP_SHIFT 16 + +#define APPL_LTR_MSG_2 0xC8 +#define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE BIT(3) + +#define APPL_LINK_STATUS 0xCC +#define APPL_LINK_STATUS_RDLH_LINK_UP BIT(0) + +#define APPL_DEBUG 0xD0 +#define APPL_DEBUG_PM_LINKST_IN_L2_LAT BIT(21) +#define APPL_DEBUG_PM_LINKST_IN_L0 0x11 +#define APPL_DEBUG_LTSSM_STATE_MASK GENMASK(8, 3) +#define APPL_DEBUG_LTSSM_STATE_SHIFT 3 +#define LTSSM_STATE_PRE_DETECT 5 + +#define APPL_RADM_STATUS 0xE4 +#define APPL_PM_XMT_TURNOFF_STATE BIT(0) + +#define APPL_DM_TYPE 0x100 +#define APPL_DM_TYPE_MASK GENMASK(3, 0) +#define APPL_DM_TYPE_RP 0x4 +#define APPL_DM_TYPE_EP 0x0 + +#define APPL_CFG_BASE_ADDR 0x104 +#define APPL_CFG_BASE_ADDR_MASK GENMASK(31, 12) + +#define APPL_CFG_IATU_DMA_BASE_ADDR 0x108 +#define APPL_CFG_IATU_DMA_BASE_ADDR_MASK GENMASK(31, 18) + +#define APPL_CFG_MISC 0x110 +#define APPL_CFG_MISC_SLV_EP_MODE BIT(14) +#define APPL_CFG_MISC_ARCACHE_MASK GENMASK(13, 10) +#define APPL_CFG_MISC_ARCACHE_SHIFT 10 +#define APPL_CFG_MISC_ARCACHE_VAL 3 + +#define APPL_CFG_SLCG_OVERRIDE 0x114 +#define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER BIT(0) + +#define APPL_CAR_RESET_OVRD 0x12C +#define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N BIT(0) + +#define IO_BASE_IO_DECODE BIT(0) +#define IO_BASE_IO_DECODE_BIT8 BIT(8) + +#define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE BIT(0) +#define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE BIT(16) + +#define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF 0x718 +#define CFG_TIMER_CTRL_ACK_NAK_SHIFT (19) + +#define EVENT_COUNTER_ALL_CLEAR 0x3 +#define EVENT_COUNTER_ENABLE_ALL 0x7 +#define EVENT_COUNTER_ENABLE_SHIFT 2 +#define EVENT_COUNTER_EVENT_SEL_MASK GENMASK(7, 0) +#define EVENT_COUNTER_EVENT_SEL_SHIFT 16 +#define EVENT_COUNTER_EVENT_Tx_L0S 0x2 +#define EVENT_COUNTER_EVENT_Rx_L0S 0x3 +#define EVENT_COUNTER_EVENT_L1 0x5 +#define EVENT_COUNTER_EVENT_L1_1 0x7 +#define EVENT_COUNTER_EVENT_L1_2 0x8 +#define EVENT_COUNTER_GROUP_SEL_SHIFT 24 +#define EVENT_COUNTER_GROUP_5 0x5 + +#define PORT_LOGIC_ACK_F_ASPM_CTRL 0x70C +#define ENTER_ASPM BIT(30) +#define L0S_ENTRANCE_LAT_SHIFT 24 +#define L0S_ENTRANCE_LAT_MASK GENMASK(26, 24) +#define L1_ENTRANCE_LAT_SHIFT 27 +#define L1_ENTRANCE_LAT_MASK GENMASK(29, 27) +#define N_FTS_SHIFT 8 +#define N_FTS_MASK GENMASK(7, 0) +#define N_FTS_VAL 52 + +#define PORT_LOGIC_GEN2_CTRL 0x80C +#define PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE BIT(17) +#define FTS_MASK GENMASK(7, 0) +#define FTS_VAL 52 + +#define PORT_LOGIC_MSI_CTRL_INT_0_EN 0x828 + +#define GEN3_EQ_CONTROL_OFF 0x8a8 +#define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT 8 +#define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK GENMASK(23, 8) +#define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK GENMASK(3, 0) + +#define GEN3_RELATED_OFF 0x890 +#define GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL BIT(0) +#define GEN3_RELATED_OFF_GEN3_EQ_DISABLE BIT(16) +#define GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT 24 +#define GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK GENMASK(25, 24) + +#define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0 +#define AMBA_ERROR_RESPONSE_CRS_SHIFT 3 +#define AMBA_ERROR_RESPONSE_CRS_MASK GENMASK(1, 0) +#define AMBA_ERROR_RESPONSE_CRS_OKAY 0 +#define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF 1 +#define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 2 + +#define PORT_LOGIC_MSIX_DOORBELL 0x948 + +#define CAP_SPCIE_CAP_OFF 0x154 +#define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK GENMASK(3, 0) +#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK GENMASK(11, 8) +#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT 8 + +#define PME_ACK_TIMEOUT 10000 + +#define LTSSM_TIMEOUT 50000 /* 50ms */ + +#define GEN3_GEN4_EQ_PRESET_INIT 5 + +#define GEN1_CORE_CLK_FREQ 62500000 +#define GEN2_CORE_CLK_FREQ 125000000 +#define GEN3_CORE_CLK_FREQ 250000000 +#define GEN4_CORE_CLK_FREQ 500000000 + +static const unsigned int pcie_gen_freq[] = { + GEN1_CORE_CLK_FREQ, + GEN2_CORE_CLK_FREQ, + GEN3_CORE_CLK_FREQ, + GEN4_CORE_CLK_FREQ +}; + +static const u32 event_cntr_ctrl_offset[] = { + 0x1d8, + 0x1a8, + 0x1a8, + 0x1a8, + 0x1c4, + 0x1d8 +}; + +static const u32 event_cntr_data_offset[] = { + 0x1dc, + 0x1ac, + 0x1ac, + 0x1ac, + 0x1c8, + 0x1dc +}; + +struct tegra_pcie_dw { + struct device *dev; + struct resource *appl_res; + struct resource *dbi_res; + struct resource *atu_dma_res; + void __iomem *appl_base; + struct clk *core_clk; + struct reset_control *core_apb_rst; + struct reset_control *core_rst; + struct dw_pcie pci; + struct tegra_bpmp *bpmp; + + bool supports_clkreq; + bool enable_cdm_check; + bool link_state; + bool update_fc_fixup; + u8 init_link_width; + u32 msi_ctrl_int; + u32 num_lanes; + u32 max_speed; + u32 cid; + u32 cfg_link_cap_l1sub; + u32 pcie_cap_base; + u32 aspm_cmrt; + u32 aspm_pwr_on_t; + u32 aspm_l0s_enter_lat; + + struct regulator *pex_ctl_supply; + struct regulator *slot_ctl_3v3; + struct regulator *slot_ctl_12v; + + unsigned int phy_count; + struct phy **phys; + + struct dentry *debugfs; +}; + +static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci) +{ + return container_of(pci, struct tegra_pcie_dw, pci); +} + +static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value, + const u32 reg) +{ + writel_relaxed(value, pcie->appl_base + reg); +} + +static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg) +{ + return readl_relaxed(pcie->appl_base + reg); +} + +struct tegra_pcie_soc { + enum dw_pcie_device_mode mode; +}; + +static void apply_bad_link_workaround(struct pcie_port *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + u32 current_link_width; + u16 val; + + /* + * NOTE:- Since this scenario is uncommon and link as such is not + * stable anyway, not waiting to confirm if link is really + * transitioning to Gen-2 speed + */ + val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); + if (val & PCI_EXP_LNKSTA_LBMS) { + current_link_width = (val & PCI_EXP_LNKSTA_NLW) >> + PCI_EXP_LNKSTA_NLW_SHIFT; + if (pcie->init_link_width > current_link_width) { + dev_warn(pci->dev, "PCIe link is bad, width reduced\n"); + val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + + PCI_EXP_LNKCTL2); + val &= ~PCI_EXP_LNKCTL2_TLS; + val |= PCI_EXP_LNKCTL2_TLS_2_5GT; + dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + + PCI_EXP_LNKCTL2, val); + + val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + + PCI_EXP_LNKCTL); + val |= PCI_EXP_LNKCTL_RL; + dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + + PCI_EXP_LNKCTL, val); + } + } +} + +static irqreturn_t tegra_pcie_rp_irq_handler(struct tegra_pcie_dw *pcie) +{ + struct dw_pcie *pci = &pcie->pci; + struct pcie_port *pp = &pci->pp; + u32 val, tmp; + u16 val_w; + + val = appl_readl(pcie, APPL_INTR_STATUS_L0); + if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) { + val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0); + if (val & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) { + appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0); + + /* SBR & Surprise Link Down WAR */ + val = appl_readl(pcie, APPL_CAR_RESET_OVRD); + val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N; + appl_writel(pcie, val, APPL_CAR_RESET_OVRD); + udelay(1); + val = appl_readl(pcie, APPL_CAR_RESET_OVRD); + val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N; + appl_writel(pcie, val, APPL_CAR_RESET_OVRD); + + val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL); + val |= PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE; + dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val); + } + } + + if (val & APPL_INTR_STATUS_L0_INT_INT) { + val = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0); + if (val & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) { + appl_writel(pcie, + APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS, + APPL_INTR_STATUS_L1_8_0); + apply_bad_link_workaround(pp); + } + if (val & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) { + appl_writel(pcie, + APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS, + APPL_INTR_STATUS_L1_8_0); + + val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + + PCI_EXP_LNKSTA); + dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w & + PCI_EXP_LNKSTA_CLS); + } + } + + val = appl_readl(pcie, APPL_INTR_STATUS_L0); + if (val & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) { + val = appl_readl(pcie, APPL_INTR_STATUS_L1_18); + tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS); + if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) { + dev_info(pci->dev, "CDM check complete\n"); + tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE; + } + if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) { + dev_err(pci->dev, "CDM comparison mismatch\n"); + tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR; + } + if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) { + dev_err(pci->dev, "CDM Logic error\n"); + tmp |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR; + } + dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, tmp); + tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR); + dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", tmp); + } + + return IRQ_HANDLED; +} + +static irqreturn_t tegra_pcie_irq_handler(int irq, void *arg) +{ + struct tegra_pcie_dw *pcie = arg; + + return tegra_pcie_rp_irq_handler(pcie); +} + +static int tegra_pcie_dw_rd_own_conf(struct pcie_port *pp, int where, int size, + u32 *val) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + + /* + * This is an endpoint mode specific register happen to appear even + * when controller is operating in root port mode and system hangs + * when it is accessed with link being in ASPM-L1 state. + * So skip accessing it altogether + */ + if (where == PORT_LOGIC_MSIX_DOORBELL) { + *val = 0x00000000; + return PCIBIOS_SUCCESSFUL; + } + + return dw_pcie_read(pci->dbi_base + where, size, val); +} + +static int tegra_pcie_dw_wr_own_conf(struct pcie_port *pp, int where, int size, + u32 val) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + + /* + * This is an endpoint mode specific register happen to appear even + * when controller is operating in root port mode and system hangs + * when it is accessed with link being in ASPM-L1 state. + * So skip accessing it altogether + */ + if (where == PORT_LOGIC_MSIX_DOORBELL) + return PCIBIOS_SUCCESSFUL; + + return dw_pcie_write(pci->dbi_base + where, size, val); +} + +#if defined(CONFIG_PCIEASPM) +static void disable_aspm_l11(struct tegra_pcie_dw *pcie) +{ + u32 val; + + val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub); + val &= ~PCI_L1SS_CAP_ASPM_L1_1; + dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val); +} + +static void disable_aspm_l12(struct tegra_pcie_dw *pcie) +{ + u32 val; + + val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub); + val &= ~PCI_L1SS_CAP_ASPM_L1_2; + dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val); +} + +static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event) +{ + u32 val; + + val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid]); + val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT); + val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; + val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT; + val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; + dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val); + val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_data_offset[pcie->cid]); + + return val; +} + +static int aspm_state_cnt(struct seq_file *s, void *data) +{ + struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *) + dev_get_drvdata(s->private); + u32 val; + + seq_printf(s, "Tx L0s entry count : %u\n", + event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S)); + + seq_printf(s, "Rx L0s entry count : %u\n", + event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S)); + + seq_printf(s, "Link L1 entry count : %u\n", + event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1)); + + seq_printf(s, "Link L1.1 entry count : %u\n", + event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1)); + + seq_printf(s, "Link L1.2 entry count : %u\n", + event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2)); + + /* Clear all counters */ + dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], + EVENT_COUNTER_ALL_CLEAR); + + /* Re-enable counting */ + val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; + val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; + dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val); + + return 0; +} + +static void init_host_aspm(struct tegra_pcie_dw *pcie) +{ + struct dw_pcie *pci = &pcie->pci; + u32 val; + + val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS); + pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP; + + /* Enable ASPM counters */ + val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; + val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; + dw_pcie_writel_dbi(pci, event_cntr_ctrl_offset[pcie->cid], val); + + /* Program T_cmrt and T_pwr_on values */ + val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub); + val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE); + val |= (pcie->aspm_cmrt << 8); + val |= (pcie->aspm_pwr_on_t << 19); + dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val); + + /* Program L0s and L1 entrance latencies */ + val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL); + val &= ~L0S_ENTRANCE_LAT_MASK; + val |= (pcie->aspm_l0s_enter_lat << L0S_ENTRANCE_LAT_SHIFT); + val |= ENTER_ASPM; + dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val); +} + +static int init_debugfs(struct tegra_pcie_dw *pcie) +{ + struct dentry *d; + + d = debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt", + pcie->debugfs, aspm_state_cnt); + if (IS_ERR_OR_NULL(d)) + dev_err(pcie->dev, + "Failed to create debugfs file \"aspm_state_cnt\"\n"); + + return 0; +} +#else +static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; } +static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; } +static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; } +static inline int init_debugfs(struct tegra_pcie_dw *pcie) { return 0; } +#endif + +static void tegra_pcie_enable_system_interrupts(struct pcie_port *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + u32 val; + u16 val_w; + + val = appl_readl(pcie, APPL_INTR_EN_L0_0); + val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN; + appl_writel(pcie, val, APPL_INTR_EN_L0_0); + + val = appl_readl(pcie, APPL_INTR_EN_L1_0_0); + val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN; + appl_writel(pcie, val, APPL_INTR_EN_L1_0_0); + + if (pcie->enable_cdm_check) { + val = appl_readl(pcie, APPL_INTR_EN_L0_0); + val |= APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN; + appl_writel(pcie, val, APPL_INTR_EN_L0_0); + + val = appl_readl(pcie, APPL_INTR_EN_L1_18); + val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR; + val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR; + appl_writel(pcie, val, APPL_INTR_EN_L1_18); + } + + val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base + + PCI_EXP_LNKSTA); + pcie->init_link_width = (val_w & PCI_EXP_LNKSTA_NLW) >> + PCI_EXP_LNKSTA_NLW_SHIFT; + + val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base + + PCI_EXP_LNKCTL); + val_w |= PCI_EXP_LNKCTL_LBMIE; + dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL, + val_w); +} + +static void tegra_pcie_enable_legacy_interrupts(struct pcie_port *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + u32 val; + + /* Enable legacy interrupt generation */ + val = appl_readl(pcie, APPL_INTR_EN_L0_0); + val |= APPL_INTR_EN_L0_0_SYS_INTR_EN; + val |= APPL_INTR_EN_L0_0_INT_INT_EN; + appl_writel(pcie, val, APPL_INTR_EN_L0_0); + + val = appl_readl(pcie, APPL_INTR_EN_L1_8_0); + val |= APPL_INTR_EN_L1_8_INTX_EN; + val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN; + val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN; + if (IS_ENABLED(CONFIG_PCIEAER)) + val |= APPL_INTR_EN_L1_8_AER_INT_EN; + appl_writel(pcie, val, APPL_INTR_EN_L1_8_0); +} + +static void tegra_pcie_enable_msi_interrupts(struct pcie_port *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + u32 val; + + dw_pcie_msi_init(pp); + + /* Enable MSI interrupt generation */ + val = appl_readl(pcie, APPL_INTR_EN_L0_0); + val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN; + val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN; + appl_writel(pcie, val, APPL_INTR_EN_L0_0); +} + +static void tegra_pcie_enable_interrupts(struct pcie_port *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + + /* Clear interrupt statuses before enabling interrupts */ + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); + + tegra_pcie_enable_system_interrupts(pp); + tegra_pcie_enable_legacy_interrupts(pp); + if (IS_ENABLED(CONFIG_PCI_MSI)) + tegra_pcie_enable_msi_interrupts(pp); +} + +static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie) +{ + struct dw_pcie *pci = &pcie->pci; + u32 val, offset, i; + + /* Program init preset */ + for (i = 0; i < pcie->num_lanes; i++) { + dw_pcie_read(pci->dbi_base + CAP_SPCIE_CAP_OFF + + (i * 2), 2, &val); + val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK; + val |= GEN3_GEN4_EQ_PRESET_INIT; + val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK; + val |= (GEN3_GEN4_EQ_PRESET_INIT << + CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT); + dw_pcie_write(pci->dbi_base + CAP_SPCIE_CAP_OFF + + (i * 2), 2, val); + + offset = dw_pcie_find_ext_capability(pci, + PCI_EXT_CAP_ID_PL_16GT) + + PCI_PL_16GT_LE_CTRL; + dw_pcie_read(pci->dbi_base + offset + i, 1, &val); + val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK; + val |= GEN3_GEN4_EQ_PRESET_INIT; + val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK; + val |= (GEN3_GEN4_EQ_PRESET_INIT << + PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT); + dw_pcie_write(pci->dbi_base + offset + i, 1, val); + } + + val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); + val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; + dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); + + val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); + val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK; + val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT); + val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK; + dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val); + + val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); + val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; + val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT); + dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); + + val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); + val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK; + val |= (0x360 << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT); + val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK; + dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val); + + val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); + val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; + dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); +} + +static void tegra_pcie_prepare_host(struct pcie_port *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + u32 val; + + val = dw_pcie_readl_dbi(pci, PCI_IO_BASE); + val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8); + dw_pcie_writel_dbi(pci, PCI_IO_BASE, val); + + val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE); + val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE; + val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE; + dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val); + + dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0); + + /* Configure FTS */ + val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL); + val &= ~(N_FTS_MASK << N_FTS_SHIFT); + val |= N_FTS_VAL << N_FTS_SHIFT; + dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val); + + val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL); + val &= ~FTS_MASK; + val |= FTS_VAL; + dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val); + + /* Enable as 0xFFFF0001 response for CRS */ + val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT); + val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT); + val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 << + AMBA_ERROR_RESPONSE_CRS_SHIFT); + dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val); + + /* Configure Max Speed from DT */ + if (pcie->max_speed && pcie->max_speed != -EINVAL) { + val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + + PCI_EXP_LNKCAP); + val &= ~PCI_EXP_LNKCAP_SLS; + val |= pcie->max_speed; + dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, + val); + } + + /* Configure Max lane width from DT */ + val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP); + val &= ~PCI_EXP_LNKCAP_MLW; + val |= (pcie->num_lanes << PCI_EXP_LNKSTA_NLW_SHIFT); + dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, val); + + config_gen3_gen4_eq_presets(pcie); + + init_host_aspm(pcie); + + val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); + val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; + dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); + + if (pcie->update_fc_fixup) { + val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF); + val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT; + dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val); + } + + dw_pcie_setup_rc(pp); + + clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ); + + /* Assert RST */ + val = appl_readl(pcie, APPL_PINMUX); + val &= ~APPL_PINMUX_PEX_RST; + appl_writel(pcie, val, APPL_PINMUX); + + usleep_range(100, 200); + + /* Enable LTSSM */ + val = appl_readl(pcie, APPL_CTRL); + val |= APPL_CTRL_LTSSM_EN; + appl_writel(pcie, val, APPL_CTRL); + + /* De-assert RST */ + val = appl_readl(pcie, APPL_PINMUX); + val |= APPL_PINMUX_PEX_RST; + appl_writel(pcie, val, APPL_PINMUX); + + msleep(100); +} + +static int tegra_pcie_dw_host_init(struct pcie_port *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + u32 val, tmp, offset, speed; + + tegra_pcie_prepare_host(pp); + + if (dw_pcie_wait_for_link(pci)) { + /* + * There are some endpoints which can't get the link up if + * root port has Data Link Feature (DLF) enabled. + * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info + * on Scaled Flow Control and DLF. + * So, need to confirm that is indeed the case here and attempt + * link up once again with DLF disabled. + */ + val = appl_readl(pcie, APPL_DEBUG); + val &= APPL_DEBUG_LTSSM_STATE_MASK; + val >>= APPL_DEBUG_LTSSM_STATE_SHIFT; + tmp = appl_readl(pcie, APPL_LINK_STATUS); + tmp &= APPL_LINK_STATUS_RDLH_LINK_UP; + if (!(val == 0x11 && !tmp)) { + /* Link is down for all good reasons */ + return 0; + } + + dev_info(pci->dev, "Link is down in DLL"); + dev_info(pci->dev, "Trying again with DLFE disabled\n"); + /* Disable LTSSM */ + val = appl_readl(pcie, APPL_CTRL); + val &= ~APPL_CTRL_LTSSM_EN; + appl_writel(pcie, val, APPL_CTRL); + + reset_control_assert(pcie->core_rst); + reset_control_deassert(pcie->core_rst); + + offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF); + val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP); + val &= ~PCI_DLF_EXCHANGE_ENABLE; + dw_pcie_writel_dbi(pci, offset, val); + + tegra_pcie_prepare_host(pp); + + if (dw_pcie_wait_for_link(pci)) + return 0; + } + + speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) & + PCI_EXP_LNKSTA_CLS; + clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]); + + tegra_pcie_enable_interrupts(pp); + + return 0; +} + +static int tegra_pcie_dw_link_up(struct dw_pcie *pci) +{ + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); + + return !!(val & PCI_EXP_LNKSTA_DLLLA); +} + +static void tegra_pcie_set_msi_vec_num(struct pcie_port *pp) +{ + pp->num_vectors = MAX_MSI_IRQS; +} + +static const struct dw_pcie_ops tegra_dw_pcie_ops = { + .link_up = tegra_pcie_dw_link_up, +}; + +static struct dw_pcie_host_ops tegra_pcie_dw_host_ops = { + .rd_own_conf = tegra_pcie_dw_rd_own_conf, + .wr_own_conf = tegra_pcie_dw_wr_own_conf, + .host_init = tegra_pcie_dw_host_init, + .set_num_vectors = tegra_pcie_set_msi_vec_num, +}; + +static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie) +{ + unsigned int phy_count = pcie->phy_count; + + while (phy_count--) { + phy_power_off(pcie->phys[phy_count]); + phy_exit(pcie->phys[phy_count]); + } +} + +static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie) +{ + unsigned int i; + int ret; + + for (i = 0; i < pcie->phy_count; i++) { + ret = phy_init(pcie->phys[i]); + if (ret < 0) + goto phy_power_off; + + ret = phy_power_on(pcie->phys[i]); + if (ret < 0) + goto phy_exit; + } + + return 0; + +phy_power_off: + while (i--) { + phy_power_off(pcie->phys[i]); +phy_exit: + phy_exit(pcie->phys[i]); + } + + return ret; +} + +static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie) +{ + struct device_node *np = pcie->dev->of_node; + int ret; + + ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt); + if (ret < 0) { + dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret); + return ret; + } + + ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us", + &pcie->aspm_pwr_on_t); + if (ret < 0) + dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n", + ret); + + ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us", + &pcie->aspm_l0s_enter_lat); + if (ret < 0) + dev_info(pcie->dev, + "Failed to read ASPM L0s Entrance latency: %d\n", ret); + + ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes); + if (ret < 0) { + dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret); + return ret; + } + + pcie->max_speed = of_pci_get_max_link_speed(np); + + ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid); + if (ret) { + dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret); + return ret; + } + + ret = of_property_count_strings(np, "phy-names"); + if (ret < 0) { + dev_err(pcie->dev, "Failed to find PHY entries: %d\n", + ret); + return ret; + } + pcie->phy_count = ret; + + if (of_property_read_bool(np, "nvidia,update-fc-fixup")) + pcie->update_fc_fixup = true; + + pcie->supports_clkreq = + of_property_read_bool(pcie->dev->of_node, "supports-clkreq"); + + pcie->enable_cdm_check = + of_property_read_bool(np, "snps,enable-cdm-check"); + + return 0; +} + +static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie, + bool enable) +{ + struct mrq_uphy_response resp; + struct tegra_bpmp_message msg; + struct mrq_uphy_request req; + + /* Controller-5 doesn't need to have its state set by BPMP-FW */ + if (pcie->cid == 5) + return 0; + + memset(&req, 0, sizeof(req)); + memset(&resp, 0, sizeof(resp)); + + req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE; + req.controller_state.pcie_controller = pcie->cid; + req.controller_state.enable = enable; + + memset(&msg, 0, sizeof(msg)); + msg.mrq = MRQ_UPHY; + msg.tx.data = &req; + msg.tx.size = sizeof(req); + msg.rx.data = &resp; + msg.rx.size = sizeof(resp); + + return tegra_bpmp_transfer(pcie->bpmp, &msg); +} + +static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie) +{ + struct pcie_port *pp = &pcie->pci.pp; + struct pci_bus *child, *root_bus = NULL; + struct pci_dev *pdev; + + /* + * link doesn't go into L2 state with some of the endpoints with Tegra + * if they are not in D0 state. So, need to make sure that immediate + * downstream devices are in D0 state before sending PME_TurnOff to put + * link into L2 state. + * This is as per PCI Express Base r4.0 v1.0 September 27-2017, + * 5.2 Link State Power Management (Page #428). + */ + + list_for_each_entry(child, &pp->root_bus->children, node) { + /* Bring downstream devices to D0 if they are not already in */ + if (child->parent == pp->root_bus) { + root_bus = child; + break; + } + } + + if (!root_bus) { + dev_err(pcie->dev, "Failed to find downstream devices\n"); + return; + } + + list_for_each_entry(pdev, &root_bus->devices, bus_list) { + if (PCI_SLOT(pdev->devfn) == 0) { + if (pci_set_power_state(pdev, PCI_D0)) + dev_err(pcie->dev, + "Failed to transition %s to D0 state\n", + dev_name(&pdev->dev)); + } + } +} + +static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie) +{ + pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3"); + if (IS_ERR(pcie->slot_ctl_3v3)) { + if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV) + return PTR_ERR(pcie->slot_ctl_3v3); + + pcie->slot_ctl_3v3 = NULL; + } + + pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v"); + if (IS_ERR(pcie->slot_ctl_12v)) { + if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV) + return PTR_ERR(pcie->slot_ctl_12v); + + pcie->slot_ctl_12v = NULL; + } + + return 0; +} + +static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie) +{ + int ret; + + if (pcie->slot_ctl_3v3) { + ret = regulator_enable(pcie->slot_ctl_3v3); + if (ret < 0) { + dev_err(pcie->dev, + "Failed to enable 3.3V slot supply: %d\n", ret); + return ret; + } + } + + if (pcie->slot_ctl_12v) { + ret = regulator_enable(pcie->slot_ctl_12v); + if (ret < 0) { + dev_err(pcie->dev, + "Failed to enable 12V slot supply: %d\n", ret); + goto fail_12v_enable; + } + } + + /* + * According to PCI Express Card Electromechanical Specification + * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive) + * should be a minimum of 100ms. + */ + if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v) + msleep(100); + + return 0; + +fail_12v_enable: + if (pcie->slot_ctl_3v3) + regulator_disable(pcie->slot_ctl_3v3); + return ret; +} + +static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie) +{ + if (pcie->slot_ctl_12v) + regulator_disable(pcie->slot_ctl_12v); + if (pcie->slot_ctl_3v3) + regulator_disable(pcie->slot_ctl_3v3); +} + +static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie, + bool en_hw_hot_rst) +{ + int ret; + u32 val; + + ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true); + if (ret) { + dev_err(pcie->dev, + "Failed to enable controller %u: %d\n", pcie->cid, ret); + return ret; + } + + ret = tegra_pcie_enable_slot_regulators(pcie); + if (ret < 0) + goto fail_slot_reg_en; + + ret = regulator_enable(pcie->pex_ctl_supply); + if (ret < 0) { + dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret); + goto fail_reg_en; + } + + ret = clk_prepare_enable(pcie->core_clk); + if (ret) { + dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret); + goto fail_core_clk; + } + + ret = reset_control_deassert(pcie->core_apb_rst); + if (ret) { + dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n", + ret); + goto fail_core_apb_rst; + } + + if (en_hw_hot_rst) { + /* Enable HW_HOT_RST mode */ + val = appl_readl(pcie, APPL_CTRL); + val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << + APPL_CTRL_HW_HOT_RST_MODE_SHIFT); + val |= APPL_CTRL_HW_HOT_RST_EN; + appl_writel(pcie, val, APPL_CTRL); + } + + ret = tegra_pcie_enable_phy(pcie); + if (ret) { + dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret); + goto fail_phy; + } + + /* Update CFG base address */ + appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK, + APPL_CFG_BASE_ADDR); + + /* Configure this core for RP mode operation */ + appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE); + + appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE); + + val = appl_readl(pcie, APPL_CTRL); + appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL); + + val = appl_readl(pcie, APPL_CFG_MISC); + val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT); + appl_writel(pcie, val, APPL_CFG_MISC); + + if (!pcie->supports_clkreq) { + val = appl_readl(pcie, APPL_PINMUX); + val |= APPL_PINMUX_CLKREQ_OUT_OVRD_EN; + val |= APPL_PINMUX_CLKREQ_OUT_OVRD; + appl_writel(pcie, val, APPL_PINMUX); + } + + /* Update iATU_DMA base address */ + appl_writel(pcie, + pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK, + APPL_CFG_IATU_DMA_BASE_ADDR); + + reset_control_deassert(pcie->core_rst); + + pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci, + PCI_CAP_ID_EXP); + + /* Disable ASPM-L1SS advertisement as there is no CLKREQ routing */ + if (!pcie->supports_clkreq) { + disable_aspm_l11(pcie); + disable_aspm_l12(pcie); + } + + return ret; + +fail_phy: + reset_control_assert(pcie->core_apb_rst); +fail_core_apb_rst: + clk_disable_unprepare(pcie->core_clk); +fail_core_clk: + regulator_disable(pcie->pex_ctl_supply); +fail_reg_en: + tegra_pcie_disable_slot_regulators(pcie); +fail_slot_reg_en: + tegra_pcie_bpmp_set_ctrl_state(pcie, false); + + return ret; +} + +static int __deinit_controller(struct tegra_pcie_dw *pcie) +{ + int ret; + + ret = reset_control_assert(pcie->core_rst); + if (ret) { + dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", + ret); + return ret; + } + + tegra_pcie_disable_phy(pcie); + + ret = reset_control_assert(pcie->core_apb_rst); + if (ret) { + dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret); + return ret; + } + + clk_disable_unprepare(pcie->core_clk); + + ret = regulator_disable(pcie->pex_ctl_supply); + if (ret) { + dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret); + return ret; + } + + tegra_pcie_disable_slot_regulators(pcie); + + ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false); + if (ret) { + dev_err(pcie->dev, "Failed to disable controller %d: %d\n", + pcie->cid, ret); + return ret; + } + + return ret; +} + +static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie) +{ + struct dw_pcie *pci = &pcie->pci; + struct pcie_port *pp = &pci->pp; + int ret; + + ret = tegra_pcie_config_controller(pcie, false); + if (ret < 0) + return ret; + + pp->ops = &tegra_pcie_dw_host_ops; + + ret = dw_pcie_host_init(pp); + if (ret < 0) { + dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret); + goto fail_host_init; + } + + return 0; + +fail_host_init: + return __deinit_controller(pcie); +} + +static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie) +{ + u32 val; + + if (!tegra_pcie_dw_link_up(&pcie->pci)) + return 0; + + val = appl_readl(pcie, APPL_RADM_STATUS); + val |= APPL_PM_XMT_TURNOFF_STATE; + appl_writel(pcie, val, APPL_RADM_STATUS); + + return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val, + val & APPL_DEBUG_PM_LINKST_IN_L2_LAT, + 1, PME_ACK_TIMEOUT); +} + +static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie) +{ + u32 data; + int err; + + if (!tegra_pcie_dw_link_up(&pcie->pci)) { + dev_dbg(pcie->dev, "PCIe link is not up...!\n"); + return; + } + + if (tegra_pcie_try_link_l2(pcie)) { + dev_info(pcie->dev, "Link didn't transition to L2 state\n"); + /* + * TX lane clock freq will reset to Gen1 only if link is in L2 + * or detect state. + * So apply pex_rst to end point to force RP to go into detect + * state + */ + data = appl_readl(pcie, APPL_PINMUX); + data &= ~APPL_PINMUX_PEX_RST; + appl_writel(pcie, data, APPL_PINMUX); + + err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, + data, + ((data & + APPL_DEBUG_LTSSM_STATE_MASK) >> + APPL_DEBUG_LTSSM_STATE_SHIFT) == + LTSSM_STATE_PRE_DETECT, + 1, LTSSM_TIMEOUT); + if (err) { + dev_info(pcie->dev, "Link didn't go to detect state\n"); + } else { + /* Disable LTSSM after link is in detect state */ + data = appl_readl(pcie, APPL_CTRL); + data &= ~APPL_CTRL_LTSSM_EN; + appl_writel(pcie, data, APPL_CTRL); + } + } + /* + * DBI registers may not be accessible after this as PLL-E would be + * down depending on how CLKREQ is pulled by end point + */ + data = appl_readl(pcie, APPL_PINMUX); + data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE); + /* Cut REFCLK to slot */ + data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN; + data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE; + appl_writel(pcie, data, APPL_PINMUX); +} + +static int tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie) +{ + tegra_pcie_downstream_dev_to_D0(pcie); + dw_pcie_host_deinit(&pcie->pci.pp); + tegra_pcie_dw_pme_turnoff(pcie); + + return __deinit_controller(pcie); +} + +static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie) +{ + struct pcie_port *pp = &pcie->pci.pp; + struct device *dev = pcie->dev; + char *name; + int ret; + + if (IS_ENABLED(CONFIG_PCI_MSI)) { + pp->msi_irq = of_irq_get_byname(dev->of_node, "msi"); + if (!pp->msi_irq) { + dev_err(dev, "Failed to get MSI interrupt\n"); + return -ENODEV; + } + } + + pm_runtime_enable(dev); + + ret = pm_runtime_get_sync(dev); + if (ret < 0) { + dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n", + ret); + goto fail_pm_get_sync; + } + + ret = pinctrl_pm_select_default_state(dev); + if (ret < 0) { + dev_err(dev, "Failed to configure sideband pins: %d\n", ret); + goto fail_pinctrl; + } + + tegra_pcie_init_controller(pcie); + + pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci); + if (!pcie->link_state) { + ret = -ENOMEDIUM; + goto fail_host_init; + } + + name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node); + if (!name) { + ret = -ENOMEM; + goto fail_host_init; + } + + pcie->debugfs = debugfs_create_dir(name, NULL); + if (!pcie->debugfs) + dev_err(dev, "Failed to create debugfs\n"); + else + init_debugfs(pcie); + + return ret; + +fail_host_init: + tegra_pcie_deinit_controller(pcie); +fail_pinctrl: + pm_runtime_put_sync(dev); +fail_pm_get_sync: + pm_runtime_disable(dev); + return ret; +} + +static int tegra_pcie_dw_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct resource *atu_dma_res; + struct tegra_pcie_dw *pcie; + struct resource *dbi_res; + struct pcie_port *pp; + struct dw_pcie *pci; + struct phy **phys; + char *name; + int ret; + u32 i; + + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + pci = &pcie->pci; + pci->dev = &pdev->dev; + pci->ops = &tegra_dw_pcie_ops; + pp = &pci->pp; + pcie->dev = &pdev->dev; + + ret = tegra_pcie_dw_parse_dt(pcie); + if (ret < 0) { + dev_err(dev, "Failed to parse device tree: %d\n", ret); + return ret; + } + + ret = tegra_pcie_get_slot_regulators(pcie); + if (ret < 0) { + dev_err(dev, "Failed to get slot regulators: %d\n", ret); + return ret; + } + + pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl"); + if (IS_ERR(pcie->pex_ctl_supply)) { + ret = PTR_ERR(pcie->pex_ctl_supply); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to get regulator: %ld\n", + PTR_ERR(pcie->pex_ctl_supply)); + return ret; + } + + pcie->core_clk = devm_clk_get(dev, "core"); + if (IS_ERR(pcie->core_clk)) { + dev_err(dev, "Failed to get core clock: %ld\n", + PTR_ERR(pcie->core_clk)); + return PTR_ERR(pcie->core_clk); + } + + pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "appl"); + if (!pcie->appl_res) { + dev_err(dev, "Failed to find \"appl\" region\n"); + return -ENODEV; + } + + pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res); + if (IS_ERR(pcie->appl_base)) + return PTR_ERR(pcie->appl_base); + + pcie->core_apb_rst = devm_reset_control_get(dev, "apb"); + if (IS_ERR(pcie->core_apb_rst)) { + dev_err(dev, "Failed to get APB reset: %ld\n", + PTR_ERR(pcie->core_apb_rst)); + return PTR_ERR(pcie->core_apb_rst); + } + + phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL); + if (!phys) + return -ENOMEM; + + for (i = 0; i < pcie->phy_count; i++) { + name = kasprintf(GFP_KERNEL, "p2u-%u", i); + if (!name) { + dev_err(dev, "Failed to create P2U string\n"); + return -ENOMEM; + } + phys[i] = devm_phy_get(dev, name); + kfree(name); + if (IS_ERR(phys[i])) { + ret = PTR_ERR(phys[i]); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to get PHY: %d\n", ret); + return ret; + } + } + + pcie->phys = phys; + + dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi"); + if (!dbi_res) { + dev_err(dev, "Failed to find \"dbi\" region\n"); + return -ENODEV; + } + pcie->dbi_res = dbi_res; + + pci->dbi_base = devm_ioremap_resource(dev, dbi_res); + if (IS_ERR(pci->dbi_base)) + return PTR_ERR(pci->dbi_base); + + /* Tegra HW locates DBI2 at a fixed offset from DBI */ + pci->dbi_base2 = pci->dbi_base + 0x1000; + + atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "atu_dma"); + if (!atu_dma_res) { + dev_err(dev, "Failed to find \"atu_dma\" region\n"); + return -ENODEV; + } + pcie->atu_dma_res = atu_dma_res; + + pci->atu_base = devm_ioremap_resource(dev, atu_dma_res); + if (IS_ERR(pci->atu_base)) + return PTR_ERR(pci->atu_base); + + pcie->core_rst = devm_reset_control_get(dev, "core"); + if (IS_ERR(pcie->core_rst)) { + dev_err(dev, "Failed to get core reset: %ld\n", + PTR_ERR(pcie->core_rst)); + return PTR_ERR(pcie->core_rst); + } + + pp->irq = platform_get_irq_byname(pdev, "intr"); + if (!pp->irq) { + dev_err(dev, "Failed to get \"intr\" interrupt\n"); + return -ENODEV; + } + + ret = devm_request_irq(dev, pp->irq, tegra_pcie_irq_handler, + IRQF_SHARED, "tegra-pcie-intr", pcie); + if (ret) { + dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq, ret); + return ret; + } + + pcie->bpmp = tegra_bpmp_get(dev); + if (IS_ERR(pcie->bpmp)) + return PTR_ERR(pcie->bpmp); + + platform_set_drvdata(pdev, pcie); + + ret = tegra_pcie_config_rp(pcie); + if (ret && ret != -ENOMEDIUM) + goto fail; + else + return 0; + +fail: + tegra_bpmp_put(pcie->bpmp); + return ret; +} + +static int tegra_pcie_dw_remove(struct platform_device *pdev) +{ + struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev); + + if (!pcie->link_state) + return 0; + + debugfs_remove_recursive(pcie->debugfs); + tegra_pcie_deinit_controller(pcie); + pm_runtime_put_sync(pcie->dev); + pm_runtime_disable(pcie->dev); + tegra_bpmp_put(pcie->bpmp); + + return 0; +} + +static int tegra_pcie_dw_suspend_late(struct device *dev) +{ + struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); + u32 val; + + if (!pcie->link_state) + return 0; + + /* Enable HW_HOT_RST mode */ + val = appl_readl(pcie, APPL_CTRL); + val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << + APPL_CTRL_HW_HOT_RST_MODE_SHIFT); + val |= APPL_CTRL_HW_HOT_RST_EN; + appl_writel(pcie, val, APPL_CTRL); + + return 0; +} + +static int tegra_pcie_dw_suspend_noirq(struct device *dev) +{ + struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); + + if (!pcie->link_state) + return 0; + + /* Save MSI interrupt vector */ + pcie->msi_ctrl_int = dw_pcie_readl_dbi(&pcie->pci, + PORT_LOGIC_MSI_CTRL_INT_0_EN); + tegra_pcie_downstream_dev_to_D0(pcie); + tegra_pcie_dw_pme_turnoff(pcie); + + return __deinit_controller(pcie); +} + +static int tegra_pcie_dw_resume_noirq(struct device *dev) +{ + struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); + int ret; + + if (!pcie->link_state) + return 0; + + ret = tegra_pcie_config_controller(pcie, true); + if (ret < 0) + return ret; + + ret = tegra_pcie_dw_host_init(&pcie->pci.pp); + if (ret < 0) { + dev_err(dev, "Failed to init host: %d\n", ret); + goto fail_host_init; + } + + /* Restore MSI interrupt vector */ + dw_pcie_writel_dbi(&pcie->pci, PORT_LOGIC_MSI_CTRL_INT_0_EN, + pcie->msi_ctrl_int); + + return 0; + +fail_host_init: + return __deinit_controller(pcie); +} + +static int tegra_pcie_dw_resume_early(struct device *dev) +{ + struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); + u32 val; + + if (!pcie->link_state) + return 0; + + /* Disable HW_HOT_RST mode */ + val = appl_readl(pcie, APPL_CTRL); + val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << + APPL_CTRL_HW_HOT_RST_MODE_SHIFT); + val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST << + APPL_CTRL_HW_HOT_RST_MODE_SHIFT; + val &= ~APPL_CTRL_HW_HOT_RST_EN; + appl_writel(pcie, val, APPL_CTRL); + + return 0; +} + +static void tegra_pcie_dw_shutdown(struct platform_device *pdev) +{ + struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev); + + if (!pcie->link_state) + return; + + debugfs_remove_recursive(pcie->debugfs); + tegra_pcie_downstream_dev_to_D0(pcie); + + disable_irq(pcie->pci.pp.irq); + if (IS_ENABLED(CONFIG_PCI_MSI)) + disable_irq(pcie->pci.pp.msi_irq); + + tegra_pcie_dw_pme_turnoff(pcie); + __deinit_controller(pcie); +} + +static const struct of_device_id tegra_pcie_dw_of_match[] = { + { + .compatible = "nvidia,tegra194-pcie", + }, + {}, +}; + +static const struct dev_pm_ops tegra_pcie_dw_pm_ops = { + .suspend_late = tegra_pcie_dw_suspend_late, + .suspend_noirq = tegra_pcie_dw_suspend_noirq, + .resume_noirq = tegra_pcie_dw_resume_noirq, + .resume_early = tegra_pcie_dw_resume_early, +}; + +static struct platform_driver tegra_pcie_dw_driver = { + .probe = tegra_pcie_dw_probe, + .remove = tegra_pcie_dw_remove, + .shutdown = tegra_pcie_dw_shutdown, + .driver = { + .name = "tegra194-pcie", + .pm = &tegra_pcie_dw_pm_ops, + .of_match_table = tegra_pcie_dw_of_match, + }, +}; +module_platform_driver(tegra_pcie_dw_driver); + +MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match); + +MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>"); +MODULE_DESCRIPTION("NVIDIA PCIe host controller driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/dwc/pcie-uniphier.c b/drivers/pci/controller/dwc/pcie-uniphier.c index d5dc40289cce..3f30ee4a00b3 100644 --- a/drivers/pci/controller/dwc/pcie-uniphier.c +++ b/drivers/pci/controller/dwc/pcie-uniphier.c @@ -270,6 +270,7 @@ static int uniphier_pcie_config_legacy_irq(struct pcie_port *pp) struct uniphier_pcie_priv *priv = to_uniphier_pcie(pci); struct device_node *np = pci->dev->of_node; struct device_node *np_intc; + int ret = 0; np_intc = of_get_child_by_name(np, "legacy-interrupt-controller"); if (!np_intc) { @@ -280,20 +281,24 @@ static int uniphier_pcie_config_legacy_irq(struct pcie_port *pp) pp->irq = irq_of_parse_and_map(np_intc, 0); if (!pp->irq) { dev_err(pci->dev, "Failed to get an IRQ entry in legacy-interrupt-controller\n"); - return -EINVAL; + ret = -EINVAL; + goto out_put_node; } priv->legacy_irq_domain = irq_domain_add_linear(np_intc, PCI_NUM_INTX, &uniphier_intx_domain_ops, pp); if (!priv->legacy_irq_domain) { dev_err(pci->dev, "Failed to get INTx domain\n"); - return -ENODEV; + ret = -ENODEV; + goto out_put_node; } irq_set_chained_handler_and_data(pp->irq, uniphier_pcie_irq_handler, pp); - return 0; +out_put_node: + of_node_put(np_intc); + return ret; } static int uniphier_pcie_host_init(struct pcie_port *pp) diff --git a/drivers/pci/controller/pci-aardvark.c b/drivers/pci/controller/pci-aardvark.c index eb58dfdaba1b..fc0fe4d4de49 100644 --- a/drivers/pci/controller/pci-aardvark.c +++ b/drivers/pci/controller/pci-aardvark.c @@ -308,7 +308,7 @@ static void advk_pcie_setup_hw(struct advk_pcie *pcie) advk_writel(pcie, PCIE_ISR1_ALL_MASK, PCIE_ISR1_MASK_REG); - /* Unmask all MSI's */ + /* Unmask all MSIs */ advk_writel(pcie, 0, PCIE_MSI_MASK_REG); /* Enable summary interrupt for GIC SPI source */ @@ -794,6 +794,7 @@ static int advk_pcie_init_irq_domain(struct advk_pcie *pcie) struct device_node *node = dev->of_node; struct device_node *pcie_intc_node; struct irq_chip *irq_chip; + int ret = 0; pcie_intc_node = of_get_next_child(node, NULL); if (!pcie_intc_node) { @@ -806,8 +807,8 @@ static int advk_pcie_init_irq_domain(struct advk_pcie *pcie) irq_chip->name = devm_kasprintf(dev, GFP_KERNEL, "%s-irq", dev_name(dev)); if (!irq_chip->name) { - of_node_put(pcie_intc_node); - return -ENOMEM; + ret = -ENOMEM; + goto out_put_node; } irq_chip->irq_mask = advk_pcie_irq_mask; @@ -819,11 +820,13 @@ static int advk_pcie_init_irq_domain(struct advk_pcie *pcie) &advk_pcie_irq_domain_ops, pcie); if (!pcie->irq_domain) { dev_err(dev, "Failed to get a INTx IRQ domain\n"); - of_node_put(pcie_intc_node); - return -ENOMEM; + ret = -ENOMEM; + goto out_put_node; } - return 0; +out_put_node: + of_node_put(pcie_intc_node); + return ret; } static void advk_pcie_remove_irq_domain(struct advk_pcie *pcie) diff --git a/drivers/pci/controller/pci-host-common.c b/drivers/pci/controller/pci-host-common.c index c742881b5061..c8cb9c5188a4 100644 --- a/drivers/pci/controller/pci-host-common.c +++ b/drivers/pci/controller/pci-host-common.c @@ -43,9 +43,8 @@ static struct pci_config_window *gen_pci_init(struct device *dev, goto err_out; } - err = devm_add_action(dev, gen_pci_unmap_cfg, cfg); + err = devm_add_action_or_reset(dev, gen_pci_unmap_cfg, cfg); if (err) { - gen_pci_unmap_cfg(cfg); goto err_out; } return cfg; diff --git a/drivers/pci/controller/pci-host-generic.c b/drivers/pci/controller/pci-host-generic.c index dea3ec7592a2..75a2fb930d4b 100644 --- a/drivers/pci/controller/pci-host-generic.c +++ b/drivers/pci/controller/pci-host-generic.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Simple, generic PCI host controller driver targetting firmware-initialised + * Simple, generic PCI host controller driver targeting firmware-initialised * systems and virtual machines (e.g. the PCI emulation provided by kvmtool). * * Copyright (C) 2014 ARM Limited diff --git a/drivers/pci/controller/pci-hyperv-intf.c b/drivers/pci/controller/pci-hyperv-intf.c new file mode 100644 index 000000000000..cc96be450360 --- /dev/null +++ b/drivers/pci/controller/pci-hyperv-intf.c @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) Microsoft Corporation. + * + * Author: + * Haiyang Zhang <haiyangz@microsoft.com> + * + * This small module is a helper driver allows other drivers to + * have a common interface with the Hyper-V PCI frontend driver. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/hyperv.h> + +struct hyperv_pci_block_ops hvpci_block_ops; +EXPORT_SYMBOL_GPL(hvpci_block_ops); + +int hyperv_read_cfg_blk(struct pci_dev *dev, void *buf, unsigned int buf_len, + unsigned int block_id, unsigned int *bytes_returned) +{ + if (!hvpci_block_ops.read_block) + return -EOPNOTSUPP; + + return hvpci_block_ops.read_block(dev, buf, buf_len, block_id, + bytes_returned); +} +EXPORT_SYMBOL_GPL(hyperv_read_cfg_blk); + +int hyperv_write_cfg_blk(struct pci_dev *dev, void *buf, unsigned int len, + unsigned int block_id) +{ + if (!hvpci_block_ops.write_block) + return -EOPNOTSUPP; + + return hvpci_block_ops.write_block(dev, buf, len, block_id); +} +EXPORT_SYMBOL_GPL(hyperv_write_cfg_blk); + +int hyperv_reg_block_invalidate(struct pci_dev *dev, void *context, + void (*block_invalidate)(void *context, + u64 block_mask)) +{ + if (!hvpci_block_ops.reg_blk_invalidate) + return -EOPNOTSUPP; + + return hvpci_block_ops.reg_blk_invalidate(dev, context, + block_invalidate); +} +EXPORT_SYMBOL_GPL(hyperv_reg_block_invalidate); + +static void __exit exit_hv_pci_intf(void) +{ +} + +static int __init init_hv_pci_intf(void) +{ + return 0; +} + +module_init(init_hv_pci_intf); +module_exit(exit_hv_pci_intf); + +MODULE_DESCRIPTION("Hyper-V PCI Interface"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c index 95441a35eceb..f1f300218fab 100644 --- a/drivers/pci/controller/pci-hyperv.c +++ b/drivers/pci/controller/pci-hyperv.c @@ -365,6 +365,39 @@ struct pci_delete_interrupt { struct tran_int_desc int_desc; } __packed; +/* + * Note: the VM must pass a valid block id, wslot and bytes_requested. + */ +struct pci_read_block { + struct pci_message message_type; + u32 block_id; + union win_slot_encoding wslot; + u32 bytes_requested; +} __packed; + +struct pci_read_block_response { + struct vmpacket_descriptor hdr; + u32 status; + u8 bytes[HV_CONFIG_BLOCK_SIZE_MAX]; +} __packed; + +/* + * Note: the VM must pass a valid block id, wslot and byte_count. + */ +struct pci_write_block { + struct pci_message message_type; + u32 block_id; + union win_slot_encoding wslot; + u32 byte_count; + u8 bytes[HV_CONFIG_BLOCK_SIZE_MAX]; +} __packed; + +struct pci_dev_inval_block { + struct pci_incoming_message incoming; + union win_slot_encoding wslot; + u64 block_mask; +} __packed; + struct pci_dev_incoming { struct pci_incoming_message incoming; union win_slot_encoding wslot; @@ -499,6 +532,9 @@ struct hv_pci_dev { struct hv_pcibus_device *hbus; struct work_struct wrk; + void (*block_invalidate)(void *context, u64 block_mask); + void *invalidate_context; + /* * What would be observed if one wrote 0xFFFFFFFF to a BAR and then * read it back, for each of the BAR offsets within config space. @@ -817,6 +853,253 @@ static struct pci_ops hv_pcifront_ops = { .write = hv_pcifront_write_config, }; +/* + * Paravirtual backchannel + * + * Hyper-V SR-IOV provides a backchannel mechanism in software for + * communication between a VF driver and a PF driver. These + * "configuration blocks" are similar in concept to PCI configuration space, + * but instead of doing reads and writes in 32-bit chunks through a very slow + * path, packets of up to 128 bytes can be sent or received asynchronously. + * + * Nearly every SR-IOV device contains just such a communications channel in + * hardware, so using this one in software is usually optional. Using the + * software channel, however, allows driver implementers to leverage software + * tools that fuzz the communications channel looking for vulnerabilities. + * + * The usage model for these packets puts the responsibility for reading or + * writing on the VF driver. The VF driver sends a read or a write packet, + * indicating which "block" is being referred to by number. + * + * If the PF driver wishes to initiate communication, it can "invalidate" one or + * more of the first 64 blocks. This invalidation is delivered via a callback + * supplied by the VF driver by this driver. + * + * No protocol is implied, except that supplied by the PF and VF drivers. + */ + +struct hv_read_config_compl { + struct hv_pci_compl comp_pkt; + void *buf; + unsigned int len; + unsigned int bytes_returned; +}; + +/** + * hv_pci_read_config_compl() - Invoked when a response packet + * for a read config block operation arrives. + * @context: Identifies the read config operation + * @resp: The response packet itself + * @resp_packet_size: Size in bytes of the response packet + */ +static void hv_pci_read_config_compl(void *context, struct pci_response *resp, + int resp_packet_size) +{ + struct hv_read_config_compl *comp = context; + struct pci_read_block_response *read_resp = + (struct pci_read_block_response *)resp; + unsigned int data_len, hdr_len; + + hdr_len = offsetof(struct pci_read_block_response, bytes); + if (resp_packet_size < hdr_len) { + comp->comp_pkt.completion_status = -1; + goto out; + } + + data_len = resp_packet_size - hdr_len; + if (data_len > 0 && read_resp->status == 0) { + comp->bytes_returned = min(comp->len, data_len); + memcpy(comp->buf, read_resp->bytes, comp->bytes_returned); + } else { + comp->bytes_returned = 0; + } + + comp->comp_pkt.completion_status = read_resp->status; +out: + complete(&comp->comp_pkt.host_event); +} + +/** + * hv_read_config_block() - Sends a read config block request to + * the back-end driver running in the Hyper-V parent partition. + * @pdev: The PCI driver's representation for this device. + * @buf: Buffer into which the config block will be copied. + * @len: Size in bytes of buf. + * @block_id: Identifies the config block which has been requested. + * @bytes_returned: Size which came back from the back-end driver. + * + * Return: 0 on success, -errno on failure + */ +int hv_read_config_block(struct pci_dev *pdev, void *buf, unsigned int len, + unsigned int block_id, unsigned int *bytes_returned) +{ + struct hv_pcibus_device *hbus = + container_of(pdev->bus->sysdata, struct hv_pcibus_device, + sysdata); + struct { + struct pci_packet pkt; + char buf[sizeof(struct pci_read_block)]; + } pkt; + struct hv_read_config_compl comp_pkt; + struct pci_read_block *read_blk; + int ret; + + if (len == 0 || len > HV_CONFIG_BLOCK_SIZE_MAX) + return -EINVAL; + + init_completion(&comp_pkt.comp_pkt.host_event); + comp_pkt.buf = buf; + comp_pkt.len = len; + + memset(&pkt, 0, sizeof(pkt)); + pkt.pkt.completion_func = hv_pci_read_config_compl; + pkt.pkt.compl_ctxt = &comp_pkt; + read_blk = (struct pci_read_block *)&pkt.pkt.message; + read_blk->message_type.type = PCI_READ_BLOCK; + read_blk->wslot.slot = devfn_to_wslot(pdev->devfn); + read_blk->block_id = block_id; + read_blk->bytes_requested = len; + + ret = vmbus_sendpacket(hbus->hdev->channel, read_blk, + sizeof(*read_blk), (unsigned long)&pkt.pkt, + VM_PKT_DATA_INBAND, + VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); + if (ret) + return ret; + + ret = wait_for_response(hbus->hdev, &comp_pkt.comp_pkt.host_event); + if (ret) + return ret; + + if (comp_pkt.comp_pkt.completion_status != 0 || + comp_pkt.bytes_returned == 0) { + dev_err(&hbus->hdev->device, + "Read Config Block failed: 0x%x, bytes_returned=%d\n", + comp_pkt.comp_pkt.completion_status, + comp_pkt.bytes_returned); + return -EIO; + } + + *bytes_returned = comp_pkt.bytes_returned; + return 0; +} + +/** + * hv_pci_write_config_compl() - Invoked when a response packet for a write + * config block operation arrives. + * @context: Identifies the write config operation + * @resp: The response packet itself + * @resp_packet_size: Size in bytes of the response packet + */ +static void hv_pci_write_config_compl(void *context, struct pci_response *resp, + int resp_packet_size) +{ + struct hv_pci_compl *comp_pkt = context; + + comp_pkt->completion_status = resp->status; + complete(&comp_pkt->host_event); +} + +/** + * hv_write_config_block() - Sends a write config block request to the + * back-end driver running in the Hyper-V parent partition. + * @pdev: The PCI driver's representation for this device. + * @buf: Buffer from which the config block will be copied. + * @len: Size in bytes of buf. + * @block_id: Identifies the config block which is being written. + * + * Return: 0 on success, -errno on failure + */ +int hv_write_config_block(struct pci_dev *pdev, void *buf, unsigned int len, + unsigned int block_id) +{ + struct hv_pcibus_device *hbus = + container_of(pdev->bus->sysdata, struct hv_pcibus_device, + sysdata); + struct { + struct pci_packet pkt; + char buf[sizeof(struct pci_write_block)]; + u32 reserved; + } pkt; + struct hv_pci_compl comp_pkt; + struct pci_write_block *write_blk; + u32 pkt_size; + int ret; + + if (len == 0 || len > HV_CONFIG_BLOCK_SIZE_MAX) + return -EINVAL; + + init_completion(&comp_pkt.host_event); + + memset(&pkt, 0, sizeof(pkt)); + pkt.pkt.completion_func = hv_pci_write_config_compl; + pkt.pkt.compl_ctxt = &comp_pkt; + write_blk = (struct pci_write_block *)&pkt.pkt.message; + write_blk->message_type.type = PCI_WRITE_BLOCK; + write_blk->wslot.slot = devfn_to_wslot(pdev->devfn); + write_blk->block_id = block_id; + write_blk->byte_count = len; + memcpy(write_blk->bytes, buf, len); + pkt_size = offsetof(struct pci_write_block, bytes) + len; + /* + * This quirk is required on some hosts shipped around 2018, because + * these hosts don't check the pkt_size correctly (new hosts have been + * fixed since early 2019). The quirk is also safe on very old hosts + * and new hosts, because, on them, what really matters is the length + * specified in write_blk->byte_count. + */ + pkt_size += sizeof(pkt.reserved); + + ret = vmbus_sendpacket(hbus->hdev->channel, write_blk, pkt_size, + (unsigned long)&pkt.pkt, VM_PKT_DATA_INBAND, + VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); + if (ret) + return ret; + + ret = wait_for_response(hbus->hdev, &comp_pkt.host_event); + if (ret) + return ret; + + if (comp_pkt.completion_status != 0) { + dev_err(&hbus->hdev->device, + "Write Config Block failed: 0x%x\n", + comp_pkt.completion_status); + return -EIO; + } + + return 0; +} + +/** + * hv_register_block_invalidate() - Invoked when a config block invalidation + * arrives from the back-end driver. + * @pdev: The PCI driver's representation for this device. + * @context: Identifies the device. + * @block_invalidate: Identifies all of the blocks being invalidated. + * + * Return: 0 on success, -errno on failure + */ +int hv_register_block_invalidate(struct pci_dev *pdev, void *context, + void (*block_invalidate)(void *context, + u64 block_mask)) +{ + struct hv_pcibus_device *hbus = + container_of(pdev->bus->sysdata, struct hv_pcibus_device, + sysdata); + struct hv_pci_dev *hpdev; + + hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(pdev->devfn)); + if (!hpdev) + return -ENODEV; + + hpdev->block_invalidate = block_invalidate; + hpdev->invalidate_context = context; + + put_pcichild(hpdev); + return 0; + +} + /* Interrupt management hooks */ static void hv_int_desc_free(struct hv_pci_dev *hpdev, struct tran_int_desc *int_desc) @@ -1486,6 +1769,21 @@ static void hv_pci_assign_slots(struct hv_pcibus_device *hbus) } } +/* + * Remove entries in sysfs pci slot directory. + */ +static void hv_pci_remove_slots(struct hv_pcibus_device *hbus) +{ + struct hv_pci_dev *hpdev; + + list_for_each_entry(hpdev, &hbus->children, list_entry) { + if (!hpdev->pci_slot) + continue; + pci_destroy_slot(hpdev->pci_slot); + hpdev->pci_slot = NULL; + } +} + /** * create_root_hv_pci_bus() - Expose a new root PCI bus * @hbus: Root PCI bus, as understood by this driver @@ -1761,6 +2059,10 @@ static void pci_devices_present_work(struct work_struct *work) hpdev = list_first_entry(&removed, struct hv_pci_dev, list_entry); list_del(&hpdev->list_entry); + + if (hpdev->pci_slot) + pci_destroy_slot(hpdev->pci_slot); + put_pcichild(hpdev); } @@ -1856,6 +2158,7 @@ static void hv_pci_devices_present(struct hv_pcibus_device *hbus, static void hv_eject_device_work(struct work_struct *work) { struct pci_eject_response *ejct_pkt; + struct hv_pcibus_device *hbus; struct hv_pci_dev *hpdev; struct pci_dev *pdev; unsigned long flags; @@ -1866,6 +2169,7 @@ static void hv_eject_device_work(struct work_struct *work) } ctxt; hpdev = container_of(work, struct hv_pci_dev, wrk); + hbus = hpdev->hbus; WARN_ON(hpdev->state != hv_pcichild_ejecting); @@ -1876,8 +2180,7 @@ static void hv_eject_device_work(struct work_struct *work) * because hbus->pci_bus may not exist yet. */ wslot = wslot_to_devfn(hpdev->desc.win_slot.slot); - pdev = pci_get_domain_bus_and_slot(hpdev->hbus->sysdata.domain, 0, - wslot); + pdev = pci_get_domain_bus_and_slot(hbus->sysdata.domain, 0, wslot); if (pdev) { pci_lock_rescan_remove(); pci_stop_and_remove_bus_device(pdev); @@ -1885,9 +2188,9 @@ static void hv_eject_device_work(struct work_struct *work) pci_unlock_rescan_remove(); } - spin_lock_irqsave(&hpdev->hbus->device_list_lock, flags); + spin_lock_irqsave(&hbus->device_list_lock, flags); list_del(&hpdev->list_entry); - spin_unlock_irqrestore(&hpdev->hbus->device_list_lock, flags); + spin_unlock_irqrestore(&hbus->device_list_lock, flags); if (hpdev->pci_slot) pci_destroy_slot(hpdev->pci_slot); @@ -1896,13 +2199,18 @@ static void hv_eject_device_work(struct work_struct *work) ejct_pkt = (struct pci_eject_response *)&ctxt.pkt.message; ejct_pkt->message_type.type = PCI_EJECTION_COMPLETE; ejct_pkt->wslot.slot = hpdev->desc.win_slot.slot; - vmbus_sendpacket(hpdev->hbus->hdev->channel, ejct_pkt, + vmbus_sendpacket(hbus->hdev->channel, ejct_pkt, sizeof(*ejct_pkt), (unsigned long)&ctxt.pkt, VM_PKT_DATA_INBAND, 0); + /* For the get_pcichild() in hv_pci_eject_device() */ + put_pcichild(hpdev); + /* For the two refs got in new_pcichild_device() */ put_pcichild(hpdev); put_pcichild(hpdev); - put_hvpcibus(hpdev->hbus); + /* hpdev has been freed. Do not use it any more. */ + + put_hvpcibus(hbus); } /** @@ -1943,6 +2251,7 @@ static void hv_pci_onchannelcallback(void *context) struct pci_response *response; struct pci_incoming_message *new_message; struct pci_bus_relations *bus_rel; + struct pci_dev_inval_block *inval; struct pci_dev_incoming *dev_message; struct hv_pci_dev *hpdev; @@ -2020,6 +2329,21 @@ static void hv_pci_onchannelcallback(void *context) } break; + case PCI_INVALIDATE_BLOCK: + + inval = (struct pci_dev_inval_block *)buffer; + hpdev = get_pcichild_wslot(hbus, + inval->wslot.slot); + if (hpdev) { + if (hpdev->block_invalidate) { + hpdev->block_invalidate( + hpdev->invalidate_context, + inval->block_mask); + } + put_pcichild(hpdev); + } + break; + default: dev_warn(&hbus->hdev->device, "Unimplemented protocol message %x\n", @@ -2485,6 +2809,48 @@ static void put_hvpcibus(struct hv_pcibus_device *hbus) complete(&hbus->remove_event); } +#define HVPCI_DOM_MAP_SIZE (64 * 1024) +static DECLARE_BITMAP(hvpci_dom_map, HVPCI_DOM_MAP_SIZE); + +/* + * PCI domain number 0 is used by emulated devices on Gen1 VMs, so define 0 + * as invalid for passthrough PCI devices of this driver. + */ +#define HVPCI_DOM_INVALID 0 + +/** + * hv_get_dom_num() - Get a valid PCI domain number + * Check if the PCI domain number is in use, and return another number if + * it is in use. + * + * @dom: Requested domain number + * + * return: domain number on success, HVPCI_DOM_INVALID on failure + */ +static u16 hv_get_dom_num(u16 dom) +{ + unsigned int i; + + if (test_and_set_bit(dom, hvpci_dom_map) == 0) + return dom; + + for_each_clear_bit(i, hvpci_dom_map, HVPCI_DOM_MAP_SIZE) { + if (test_and_set_bit(i, hvpci_dom_map) == 0) + return i; + } + + return HVPCI_DOM_INVALID; +} + +/** + * hv_put_dom_num() - Mark the PCI domain number as free + * @dom: Domain number to be freed + */ +static void hv_put_dom_num(u16 dom) +{ + clear_bit(dom, hvpci_dom_map); +} + /** * hv_pci_probe() - New VMBus channel probe, for a root PCI bus * @hdev: VMBus's tracking struct for this root PCI bus @@ -2496,6 +2862,8 @@ static int hv_pci_probe(struct hv_device *hdev, const struct hv_vmbus_device_id *dev_id) { struct hv_pcibus_device *hbus; + u16 dom_req, dom; + char *name; int ret; /* @@ -2510,19 +2878,34 @@ static int hv_pci_probe(struct hv_device *hdev, hbus->state = hv_pcibus_init; /* - * The PCI bus "domain" is what is called "segment" in ACPI and - * other specs. Pull it from the instance ID, to get something - * unique. Bytes 8 and 9 are what is used in Windows guests, so - * do the same thing for consistency. Note that, since this code - * only runs in a Hyper-V VM, Hyper-V can (and does) guarantee - * that (1) the only domain in use for something that looks like - * a physical PCI bus (which is actually emulated by the - * hypervisor) is domain 0 and (2) there will be no overlap - * between domains derived from these instance IDs in the same - * VM. + * The PCI bus "domain" is what is called "segment" in ACPI and other + * specs. Pull it from the instance ID, to get something usually + * unique. In rare cases of collision, we will find out another number + * not in use. + * + * Note that, since this code only runs in a Hyper-V VM, Hyper-V + * together with this guest driver can guarantee that (1) The only + * domain used by Gen1 VMs for something that looks like a physical + * PCI bus (which is actually emulated by the hypervisor) is domain 0. + * (2) There will be no overlap between domains (after fixing possible + * collisions) in the same VM. */ - hbus->sysdata.domain = hdev->dev_instance.b[9] | - hdev->dev_instance.b[8] << 8; + dom_req = hdev->dev_instance.b[5] << 8 | hdev->dev_instance.b[4]; + dom = hv_get_dom_num(dom_req); + + if (dom == HVPCI_DOM_INVALID) { + dev_err(&hdev->device, + "Unable to use dom# 0x%hx or other numbers", dom_req); + ret = -EINVAL; + goto free_bus; + } + + if (dom != dom_req) + dev_info(&hdev->device, + "PCI dom# 0x%hx has collision, using 0x%hx", + dom_req, dom); + + hbus->sysdata.domain = dom; hbus->hdev = hdev; refcount_set(&hbus->remove_lock, 1); @@ -2537,7 +2920,7 @@ static int hv_pci_probe(struct hv_device *hdev, hbus->sysdata.domain); if (!hbus->wq) { ret = -ENOMEM; - goto free_bus; + goto free_dom; } ret = vmbus_open(hdev->channel, pci_ring_size, pci_ring_size, NULL, 0, @@ -2564,7 +2947,14 @@ static int hv_pci_probe(struct hv_device *hdev, goto free_config; } - hbus->sysdata.fwnode = irq_domain_alloc_fwnode(hbus); + name = kasprintf(GFP_KERNEL, "%pUL", &hdev->dev_instance); + if (!name) { + ret = -ENOMEM; + goto unmap; + } + + hbus->sysdata.fwnode = irq_domain_alloc_named_fwnode(name); + kfree(name); if (!hbus->sysdata.fwnode) { ret = -ENOMEM; goto unmap; @@ -2614,6 +3004,8 @@ close: vmbus_close(hdev->channel); destroy_wq: destroy_workqueue(hbus->wq); +free_dom: + hv_put_dom_num(hbus->sysdata.domain); free_bus: free_page((unsigned long)hbus); return ret; @@ -2676,6 +3068,7 @@ static int hv_pci_remove(struct hv_device *hdev) /* Remove the bus from PCI's point of view. */ pci_lock_rescan_remove(); pci_stop_root_bus(hbus->pci_bus); + hv_pci_remove_slots(hbus); pci_remove_root_bus(hbus->pci_bus); pci_unlock_rescan_remove(); hbus->state = hv_pcibus_removed; @@ -2694,6 +3087,9 @@ static int hv_pci_remove(struct hv_device *hdev) put_hvpcibus(hbus); wait_for_completion(&hbus->remove_event); destroy_workqueue(hbus->wq); + + hv_put_dom_num(hbus->sysdata.domain); + free_page((unsigned long)hbus); return 0; } @@ -2717,10 +3113,22 @@ static struct hv_driver hv_pci_drv = { static void __exit exit_hv_pci_drv(void) { vmbus_driver_unregister(&hv_pci_drv); + + hvpci_block_ops.read_block = NULL; + hvpci_block_ops.write_block = NULL; + hvpci_block_ops.reg_blk_invalidate = NULL; } static int __init init_hv_pci_drv(void) { + /* Set the invalid domain number's bit, so it will not be used */ + set_bit(HVPCI_DOM_INVALID, hvpci_dom_map); + + /* Initialize PCI block r/w interface */ + hvpci_block_ops.read_block = hv_read_config_block; + hvpci_block_ops.write_block = hv_write_config_block; + hvpci_block_ops.reg_blk_invalidate = hv_register_block_invalidate; + return vmbus_driver_register(&hv_pci_drv); } diff --git a/drivers/pci/controller/pci-tegra.c b/drivers/pci/controller/pci-tegra.c index f4f53d092e00..673a1725ef38 100644 --- a/drivers/pci/controller/pci-tegra.c +++ b/drivers/pci/controller/pci-tegra.c @@ -17,6 +17,7 @@ #include <linux/debugfs.h> #include <linux/delay.h> #include <linux/export.h> +#include <linux/gpio/consumer.h> #include <linux/interrupt.h> #include <linux/iopoll.h> #include <linux/irq.h> @@ -30,6 +31,7 @@ #include <linux/of_platform.h> #include <linux/pci.h> #include <linux/phy/phy.h> +#include <linux/pinctrl/consumer.h> #include <linux/platform_device.h> #include <linux/reset.h> #include <linux/sizes.h> @@ -95,7 +97,8 @@ #define AFI_MSI_EN_VEC7 0xa8 #define AFI_CONFIGURATION 0xac -#define AFI_CONFIGURATION_EN_FPCI (1 << 0) +#define AFI_CONFIGURATION_EN_FPCI (1 << 0) +#define AFI_CONFIGURATION_CLKEN_OVERRIDE (1 << 31) #define AFI_FPCI_ERROR_MASKS 0xb0 @@ -159,13 +162,14 @@ #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_211 (0x1 << 20) #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_411 (0x2 << 20) #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_111 (0x2 << 20) +#define AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(x) (1 << ((x) + 29)) +#define AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO_ALL (0x7 << 29) #define AFI_FUSE 0x104 #define AFI_FUSE_PCIE_T0_GEN2_DIS (1 << 2) #define AFI_PEX0_CTRL 0x110 #define AFI_PEX1_CTRL 0x118 -#define AFI_PEX2_CTRL 0x128 #define AFI_PEX_CTRL_RST (1 << 0) #define AFI_PEX_CTRL_CLKREQ_EN (1 << 1) #define AFI_PEX_CTRL_REFCLK_EN (1 << 3) @@ -177,20 +181,74 @@ #define AFI_PEXBIAS_CTRL_0 0x168 +#define RP_PRIV_XP_DL 0x00000494 +#define RP_PRIV_XP_DL_GEN2_UPD_FC_TSHOLD (0x1ff << 1) + +#define RP_RX_HDR_LIMIT 0x00000e00 +#define RP_RX_HDR_LIMIT_PW_MASK (0xff << 8) +#define RP_RX_HDR_LIMIT_PW (0x0e << 8) + +#define RP_ECTL_2_R1 0x00000e84 +#define RP_ECTL_2_R1_RX_CTLE_1C_MASK 0xffff + +#define RP_ECTL_4_R1 0x00000e8c +#define RP_ECTL_4_R1_RX_CDR_CTRL_1C_MASK (0xffff << 16) +#define RP_ECTL_4_R1_RX_CDR_CTRL_1C_SHIFT 16 + +#define RP_ECTL_5_R1 0x00000e90 +#define RP_ECTL_5_R1_RX_EQ_CTRL_L_1C_MASK 0xffffffff + +#define RP_ECTL_6_R1 0x00000e94 +#define RP_ECTL_6_R1_RX_EQ_CTRL_H_1C_MASK 0xffffffff + +#define RP_ECTL_2_R2 0x00000ea4 +#define RP_ECTL_2_R2_RX_CTLE_1C_MASK 0xffff + +#define RP_ECTL_4_R2 0x00000eac +#define RP_ECTL_4_R2_RX_CDR_CTRL_1C_MASK (0xffff << 16) +#define RP_ECTL_4_R2_RX_CDR_CTRL_1C_SHIFT 16 + +#define RP_ECTL_5_R2 0x00000eb0 +#define RP_ECTL_5_R2_RX_EQ_CTRL_L_1C_MASK 0xffffffff + +#define RP_ECTL_6_R2 0x00000eb4 +#define RP_ECTL_6_R2_RX_EQ_CTRL_H_1C_MASK 0xffffffff + #define RP_VEND_XP 0x00000f00 -#define RP_VEND_XP_DL_UP (1 << 30) +#define RP_VEND_XP_DL_UP (1 << 30) +#define RP_VEND_XP_OPPORTUNISTIC_ACK (1 << 27) +#define RP_VEND_XP_OPPORTUNISTIC_UPDATEFC (1 << 28) +#define RP_VEND_XP_UPDATE_FC_THRESHOLD_MASK (0xff << 18) + +#define RP_VEND_CTL0 0x00000f44 +#define RP_VEND_CTL0_DSK_RST_PULSE_WIDTH_MASK (0xf << 12) +#define RP_VEND_CTL0_DSK_RST_PULSE_WIDTH (0x9 << 12) + +#define RP_VEND_CTL1 0x00000f48 +#define RP_VEND_CTL1_ERPT (1 << 13) + +#define RP_VEND_XP_BIST 0x00000f4c +#define RP_VEND_XP_BIST_GOTO_L1_L2_AFTER_DLLP_DONE (1 << 28) #define RP_VEND_CTL2 0x00000fa8 #define RP_VEND_CTL2_PCA_ENABLE (1 << 7) #define RP_PRIV_MISC 0x00000fe0 -#define RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT (0xe << 0) -#define RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT (0xf << 0) +#define RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT (0xe << 0) +#define RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT (0xf << 0) +#define RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD_MASK (0x7f << 16) +#define RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD (0xf << 16) +#define RP_PRIV_MISC_CTLR_CLK_CLAMP_ENABLE (1 << 23) +#define RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD_MASK (0x7f << 24) +#define RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD (0xf << 24) +#define RP_PRIV_MISC_TMS_CLK_CLAMP_ENABLE (1 << 31) #define RP_LINK_CONTROL_STATUS 0x00000090 #define RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE 0x20000000 #define RP_LINK_CONTROL_STATUS_LINKSTAT_MASK 0x3fff0000 +#define RP_LINK_CONTROL_STATUS_2 0x000000b0 + #define PADS_CTL_SEL 0x0000009c #define PADS_CTL 0x000000a0 @@ -226,14 +284,15 @@ #define PADS_REFCLK_CFG_DRVI_SHIFT 12 /* 15:12 */ #define PME_ACK_TIMEOUT 10000 +#define LINK_RETRAIN_TIMEOUT 100000 /* in usec */ struct tegra_msi { struct msi_controller chip; DECLARE_BITMAP(used, INT_PCI_MSI_NR); struct irq_domain *domain; - unsigned long pages; struct mutex lock; - u64 phys; + void *virt; + dma_addr_t phys; int irq; }; @@ -249,10 +308,12 @@ struct tegra_pcie_soc { unsigned int num_ports; const struct tegra_pcie_port_soc *ports; unsigned int msi_base_shift; + unsigned long afi_pex2_ctrl; u32 pads_pll_ctl; u32 tx_ref_sel; u32 pads_refclk_cfg0; u32 pads_refclk_cfg1; + u32 update_fc_threshold; bool has_pex_clkreq_en; bool has_pex_bias_ctrl; bool has_intr_prsnt_sense; @@ -260,6 +321,24 @@ struct tegra_pcie_soc { bool has_gen2; bool force_pca_enable; bool program_uphy; + bool update_clamp_threshold; + bool program_deskew_time; + bool raw_violation_fixup; + bool update_fc_timer; + bool has_cache_bars; + struct { + struct { + u32 rp_ectl_2_r1; + u32 rp_ectl_4_r1; + u32 rp_ectl_5_r1; + u32 rp_ectl_6_r1; + u32 rp_ectl_2_r2; + u32 rp_ectl_4_r2; + u32 rp_ectl_5_r2; + u32 rp_ectl_6_r2; + } regs; + bool enable; + } ectl; }; static inline struct tegra_msi *to_tegra_msi(struct msi_controller *chip) @@ -321,6 +400,8 @@ struct tegra_pcie_port { unsigned int lanes; struct phy **phys; + + struct gpio_desc *reset_gpio; }; struct tegra_pcie_bus { @@ -440,6 +521,7 @@ static struct pci_ops tegra_pcie_ops = { static unsigned long tegra_pcie_port_get_pex_ctrl(struct tegra_pcie_port *port) { + const struct tegra_pcie_soc *soc = port->pcie->soc; unsigned long ret = 0; switch (port->index) { @@ -452,7 +534,7 @@ static unsigned long tegra_pcie_port_get_pex_ctrl(struct tegra_pcie_port *port) break; case 2: - ret = AFI_PEX2_CTRL; + ret = soc->afi_pex2_ctrl; break; } @@ -465,15 +547,162 @@ static void tegra_pcie_port_reset(struct tegra_pcie_port *port) unsigned long value; /* pulse reset signal */ - value = afi_readl(port->pcie, ctrl); - value &= ~AFI_PEX_CTRL_RST; - afi_writel(port->pcie, value, ctrl); + if (port->reset_gpio) { + gpiod_set_value(port->reset_gpio, 1); + } else { + value = afi_readl(port->pcie, ctrl); + value &= ~AFI_PEX_CTRL_RST; + afi_writel(port->pcie, value, ctrl); + } usleep_range(1000, 2000); - value = afi_readl(port->pcie, ctrl); - value |= AFI_PEX_CTRL_RST; - afi_writel(port->pcie, value, ctrl); + if (port->reset_gpio) { + gpiod_set_value(port->reset_gpio, 0); + } else { + value = afi_readl(port->pcie, ctrl); + value |= AFI_PEX_CTRL_RST; + afi_writel(port->pcie, value, ctrl); + } +} + +static void tegra_pcie_enable_rp_features(struct tegra_pcie_port *port) +{ + const struct tegra_pcie_soc *soc = port->pcie->soc; + u32 value; + + /* Enable AER capability */ + value = readl(port->base + RP_VEND_CTL1); + value |= RP_VEND_CTL1_ERPT; + writel(value, port->base + RP_VEND_CTL1); + + /* Optimal settings to enhance bandwidth */ + value = readl(port->base + RP_VEND_XP); + value |= RP_VEND_XP_OPPORTUNISTIC_ACK; + value |= RP_VEND_XP_OPPORTUNISTIC_UPDATEFC; + writel(value, port->base + RP_VEND_XP); + + /* + * LTSSM will wait for DLLP to finish before entering L1 or L2, + * to avoid truncation of PM messages which results in receiver errors + */ + value = readl(port->base + RP_VEND_XP_BIST); + value |= RP_VEND_XP_BIST_GOTO_L1_L2_AFTER_DLLP_DONE; + writel(value, port->base + RP_VEND_XP_BIST); + + value = readl(port->base + RP_PRIV_MISC); + value |= RP_PRIV_MISC_CTLR_CLK_CLAMP_ENABLE; + value |= RP_PRIV_MISC_TMS_CLK_CLAMP_ENABLE; + + if (soc->update_clamp_threshold) { + value &= ~(RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD_MASK | + RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD_MASK); + value |= RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD | + RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD; + } + + writel(value, port->base + RP_PRIV_MISC); +} + +static void tegra_pcie_program_ectl_settings(struct tegra_pcie_port *port) +{ + const struct tegra_pcie_soc *soc = port->pcie->soc; + u32 value; + + value = readl(port->base + RP_ECTL_2_R1); + value &= ~RP_ECTL_2_R1_RX_CTLE_1C_MASK; + value |= soc->ectl.regs.rp_ectl_2_r1; + writel(value, port->base + RP_ECTL_2_R1); + + value = readl(port->base + RP_ECTL_4_R1); + value &= ~RP_ECTL_4_R1_RX_CDR_CTRL_1C_MASK; + value |= soc->ectl.regs.rp_ectl_4_r1 << + RP_ECTL_4_R1_RX_CDR_CTRL_1C_SHIFT; + writel(value, port->base + RP_ECTL_4_R1); + + value = readl(port->base + RP_ECTL_5_R1); + value &= ~RP_ECTL_5_R1_RX_EQ_CTRL_L_1C_MASK; + value |= soc->ectl.regs.rp_ectl_5_r1; + writel(value, port->base + RP_ECTL_5_R1); + + value = readl(port->base + RP_ECTL_6_R1); + value &= ~RP_ECTL_6_R1_RX_EQ_CTRL_H_1C_MASK; + value |= soc->ectl.regs.rp_ectl_6_r1; + writel(value, port->base + RP_ECTL_6_R1); + + value = readl(port->base + RP_ECTL_2_R2); + value &= ~RP_ECTL_2_R2_RX_CTLE_1C_MASK; + value |= soc->ectl.regs.rp_ectl_2_r2; + writel(value, port->base + RP_ECTL_2_R2); + + value = readl(port->base + RP_ECTL_4_R2); + value &= ~RP_ECTL_4_R2_RX_CDR_CTRL_1C_MASK; + value |= soc->ectl.regs.rp_ectl_4_r2 << + RP_ECTL_4_R2_RX_CDR_CTRL_1C_SHIFT; + writel(value, port->base + RP_ECTL_4_R2); + + value = readl(port->base + RP_ECTL_5_R2); + value &= ~RP_ECTL_5_R2_RX_EQ_CTRL_L_1C_MASK; + value |= soc->ectl.regs.rp_ectl_5_r2; + writel(value, port->base + RP_ECTL_5_R2); + + value = readl(port->base + RP_ECTL_6_R2); + value &= ~RP_ECTL_6_R2_RX_EQ_CTRL_H_1C_MASK; + value |= soc->ectl.regs.rp_ectl_6_r2; + writel(value, port->base + RP_ECTL_6_R2); +} + +static void tegra_pcie_apply_sw_fixup(struct tegra_pcie_port *port) +{ + const struct tegra_pcie_soc *soc = port->pcie->soc; + u32 value; + + /* + * Sometimes link speed change from Gen2 to Gen1 fails due to + * instability in deskew logic on lane-0. Increase the deskew + * retry time to resolve this issue. + */ + if (soc->program_deskew_time) { + value = readl(port->base + RP_VEND_CTL0); + value &= ~RP_VEND_CTL0_DSK_RST_PULSE_WIDTH_MASK; + value |= RP_VEND_CTL0_DSK_RST_PULSE_WIDTH; + writel(value, port->base + RP_VEND_CTL0); + } + + /* Fixup for read after write violation. */ + if (soc->raw_violation_fixup) { + value = readl(port->base + RP_RX_HDR_LIMIT); + value &= ~RP_RX_HDR_LIMIT_PW_MASK; + value |= RP_RX_HDR_LIMIT_PW; + writel(value, port->base + RP_RX_HDR_LIMIT); + + value = readl(port->base + RP_PRIV_XP_DL); + value |= RP_PRIV_XP_DL_GEN2_UPD_FC_TSHOLD; + writel(value, port->base + RP_PRIV_XP_DL); + + value = readl(port->base + RP_VEND_XP); + value &= ~RP_VEND_XP_UPDATE_FC_THRESHOLD_MASK; + value |= soc->update_fc_threshold; + writel(value, port->base + RP_VEND_XP); + } + + if (soc->update_fc_timer) { + value = readl(port->base + RP_VEND_XP); + value &= ~RP_VEND_XP_UPDATE_FC_THRESHOLD_MASK; + value |= soc->update_fc_threshold; + writel(value, port->base + RP_VEND_XP); + } + + /* + * PCIe link doesn't come up with few legacy PCIe endpoints if + * root port advertises both Gen-1 and Gen-2 speeds in Tegra. + * Hence, the strategy followed here is to initially advertise + * only Gen-1 and after link is up, retrain link to Gen-2 speed + */ + value = readl(port->base + RP_LINK_CONTROL_STATUS_2); + value &= ~PCI_EXP_LNKSTA_CLS; + value |= PCI_EXP_LNKSTA_CLS_2_5GB; + writel(value, port->base + RP_LINK_CONTROL_STATUS_2); } static void tegra_pcie_port_enable(struct tegra_pcie_port *port) @@ -500,6 +729,13 @@ static void tegra_pcie_port_enable(struct tegra_pcie_port *port) value |= RP_VEND_CTL2_PCA_ENABLE; writel(value, port->base + RP_VEND_CTL2); } + + tegra_pcie_enable_rp_features(port); + + if (soc->ectl.enable) + tegra_pcie_program_ectl_settings(port); + + tegra_pcie_apply_sw_fixup(port); } static void tegra_pcie_port_disable(struct tegra_pcie_port *port) @@ -521,6 +757,12 @@ static void tegra_pcie_port_disable(struct tegra_pcie_port *port) value &= ~AFI_PEX_CTRL_REFCLK_EN; afi_writel(port->pcie, value, ctrl); + + /* disable PCIe port and set CLKREQ# as GPIO to allow PLLE power down */ + value = afi_readl(port->pcie, AFI_PCIE_CONFIG); + value |= AFI_PCIE_CONFIG_PCIE_DISABLE(port->index); + value |= AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(port->index); + afi_writel(port->pcie, value, AFI_PCIE_CONFIG); } static void tegra_pcie_port_free(struct tegra_pcie_port *port) @@ -545,12 +787,15 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0bf1, tegra_pcie_fixup_class); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0e1c, tegra_pcie_fixup_class); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0e1d, tegra_pcie_fixup_class); -/* Tegra PCIE requires relaxed ordering */ +/* Tegra20 and Tegra30 PCIE requires relaxed ordering */ static void tegra_pcie_relax_enable(struct pci_dev *dev) { pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN); } -DECLARE_PCI_FIXUP_FINAL(PCI_ANY_ID, PCI_ANY_ID, tegra_pcie_relax_enable); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0bf0, tegra_pcie_relax_enable); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0bf1, tegra_pcie_relax_enable); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0e1c, tegra_pcie_relax_enable); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0e1d, tegra_pcie_relax_enable); static int tegra_pcie_request_resources(struct tegra_pcie *pcie) { @@ -635,7 +880,7 @@ static irqreturn_t tegra_pcie_isr(int irq, void *arg) * do not pollute kernel log with master abort reports since they * happen a lot during enumeration */ - if (code == AFI_INTR_MASTER_ABORT) + if (code == AFI_INTR_MASTER_ABORT || code == AFI_INTR_PE_PRSNT_SENSE) dev_dbg(dev, "%s, signature: %08x\n", err_msg[code], signature); else dev_err(dev, "%s, signature: %08x\n", err_msg[code], signature); @@ -704,11 +949,13 @@ static void tegra_pcie_setup_translations(struct tegra_pcie *pcie) afi_writel(pcie, 0, AFI_AXI_BAR5_SZ); afi_writel(pcie, 0, AFI_FPCI_BAR5); - /* map all upstream transactions as uncached */ - afi_writel(pcie, 0, AFI_CACHE_BAR0_ST); - afi_writel(pcie, 0, AFI_CACHE_BAR0_SZ); - afi_writel(pcie, 0, AFI_CACHE_BAR1_ST); - afi_writel(pcie, 0, AFI_CACHE_BAR1_SZ); + if (pcie->soc->has_cache_bars) { + /* map all upstream transactions as uncached */ + afi_writel(pcie, 0, AFI_CACHE_BAR0_ST); + afi_writel(pcie, 0, AFI_CACHE_BAR0_SZ); + afi_writel(pcie, 0, AFI_CACHE_BAR1_ST); + afi_writel(pcie, 0, AFI_CACHE_BAR1_SZ); + } /* MSI translations are setup only when needed */ afi_writel(pcie, 0, AFI_MSI_FPCI_BAR_ST); @@ -852,7 +1099,6 @@ static int tegra_pcie_port_phy_power_off(struct tegra_pcie_port *port) static int tegra_pcie_phy_power_on(struct tegra_pcie *pcie) { struct device *dev = pcie->dev; - const struct tegra_pcie_soc *soc = pcie->soc; struct tegra_pcie_port *port; int err; @@ -878,12 +1124,6 @@ static int tegra_pcie_phy_power_on(struct tegra_pcie *pcie) } } - /* Configure the reference clock driver */ - pads_writel(pcie, soc->pads_refclk_cfg0, PADS_REFCLK_CFG0); - - if (soc->num_ports > 2) - pads_writel(pcie, soc->pads_refclk_cfg1, PADS_REFCLK_CFG1); - return 0; } @@ -918,13 +1158,11 @@ static int tegra_pcie_phy_power_off(struct tegra_pcie *pcie) return 0; } -static int tegra_pcie_enable_controller(struct tegra_pcie *pcie) +static void tegra_pcie_enable_controller(struct tegra_pcie *pcie) { - struct device *dev = pcie->dev; const struct tegra_pcie_soc *soc = pcie->soc; struct tegra_pcie_port *port; unsigned long value; - int err; /* enable PLL power down */ if (pcie->phy) { @@ -942,9 +1180,12 @@ static int tegra_pcie_enable_controller(struct tegra_pcie *pcie) value = afi_readl(pcie, AFI_PCIE_CONFIG); value &= ~AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK; value |= AFI_PCIE_CONFIG_PCIE_DISABLE_ALL | pcie->xbar_config; + value |= AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO_ALL; - list_for_each_entry(port, &pcie->ports, list) + list_for_each_entry(port, &pcie->ports, list) { value &= ~AFI_PCIE_CONFIG_PCIE_DISABLE(port->index); + value &= ~AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(port->index); + } afi_writel(pcie, value, AFI_PCIE_CONFIG); @@ -958,20 +1199,10 @@ static int tegra_pcie_enable_controller(struct tegra_pcie *pcie) afi_writel(pcie, value, AFI_FUSE); } - if (soc->program_uphy) { - err = tegra_pcie_phy_power_on(pcie); - if (err < 0) { - dev_err(dev, "failed to power on PHY(s): %d\n", err); - return err; - } - } - - /* take the PCIe interface module out of reset */ - reset_control_deassert(pcie->pcie_xrst); - - /* finally enable PCIe */ + /* Disable AFI dynamic clock gating and enable PCIe */ value = afi_readl(pcie, AFI_CONFIGURATION); value |= AFI_CONFIGURATION_EN_FPCI; + value |= AFI_CONFIGURATION_CLKEN_OVERRIDE; afi_writel(pcie, value, AFI_CONFIGURATION); value = AFI_INTR_EN_INI_SLVERR | AFI_INTR_EN_INI_DECERR | @@ -989,22 +1220,6 @@ static int tegra_pcie_enable_controller(struct tegra_pcie *pcie) /* disable all exceptions */ afi_writel(pcie, 0, AFI_FPCI_ERROR_MASKS); - - return 0; -} - -static void tegra_pcie_disable_controller(struct tegra_pcie *pcie) -{ - int err; - - reset_control_assert(pcie->pcie_xrst); - - if (pcie->soc->program_uphy) { - err = tegra_pcie_phy_power_off(pcie); - if (err < 0) - dev_err(pcie->dev, "failed to power off PHY(s): %d\n", - err); - } } static void tegra_pcie_power_off(struct tegra_pcie *pcie) @@ -1014,13 +1229,11 @@ static void tegra_pcie_power_off(struct tegra_pcie *pcie) int err; reset_control_assert(pcie->afi_rst); - reset_control_assert(pcie->pex_rst); clk_disable_unprepare(pcie->pll_e); if (soc->has_cml_clk) clk_disable_unprepare(pcie->cml_clk); clk_disable_unprepare(pcie->afi_clk); - clk_disable_unprepare(pcie->pex_clk); if (!dev->pm_domain) tegra_powergate_power_off(TEGRA_POWERGATE_PCIE); @@ -1048,46 +1261,66 @@ static int tegra_pcie_power_on(struct tegra_pcie *pcie) if (err < 0) dev_err(dev, "failed to enable regulators: %d\n", err); - if (dev->pm_domain) { - err = clk_prepare_enable(pcie->pex_clk); + if (!dev->pm_domain) { + err = tegra_powergate_power_on(TEGRA_POWERGATE_PCIE); if (err) { - dev_err(dev, "failed to enable PEX clock: %d\n", err); - return err; + dev_err(dev, "failed to power ungate: %d\n", err); + goto regulator_disable; } - reset_control_deassert(pcie->pex_rst); - } else { - err = tegra_powergate_sequence_power_up(TEGRA_POWERGATE_PCIE, - pcie->pex_clk, - pcie->pex_rst); + err = tegra_powergate_remove_clamping(TEGRA_POWERGATE_PCIE); if (err) { - dev_err(dev, "powerup sequence failed: %d\n", err); - return err; + dev_err(dev, "failed to remove clamp: %d\n", err); + goto powergate; } } - reset_control_deassert(pcie->afi_rst); - err = clk_prepare_enable(pcie->afi_clk); if (err < 0) { dev_err(dev, "failed to enable AFI clock: %d\n", err); - return err; + goto powergate; } if (soc->has_cml_clk) { err = clk_prepare_enable(pcie->cml_clk); if (err < 0) { dev_err(dev, "failed to enable CML clock: %d\n", err); - return err; + goto disable_afi_clk; } } err = clk_prepare_enable(pcie->pll_e); if (err < 0) { dev_err(dev, "failed to enable PLLE clock: %d\n", err); - return err; + goto disable_cml_clk; } + reset_control_deassert(pcie->afi_rst); + return 0; + +disable_cml_clk: + if (soc->has_cml_clk) + clk_disable_unprepare(pcie->cml_clk); +disable_afi_clk: + clk_disable_unprepare(pcie->afi_clk); +powergate: + if (!dev->pm_domain) + tegra_powergate_power_off(TEGRA_POWERGATE_PCIE); +regulator_disable: + regulator_bulk_disable(pcie->num_supplies, pcie->supplies); + + return err; +} + +static void tegra_pcie_apply_pad_settings(struct tegra_pcie *pcie) +{ + const struct tegra_pcie_soc *soc = pcie->soc; + + /* Configure the reference clock driver */ + pads_writel(pcie, soc->pads_refclk_cfg0, PADS_REFCLK_CFG0); + + if (soc->num_ports > 2) + pads_writel(pcie, soc->pads_refclk_cfg1, PADS_REFCLK_CFG1); } static int tegra_pcie_clocks_get(struct tegra_pcie *pcie) @@ -1536,7 +1769,7 @@ static int tegra_pcie_msi_setup(struct tegra_pcie *pcie) err = platform_get_irq_byname(pdev, "msi"); if (err < 0) { dev_err(dev, "failed to get IRQ: %d\n", err); - goto err; + goto free_irq_domain; } msi->irq = err; @@ -1545,17 +1778,35 @@ static int tegra_pcie_msi_setup(struct tegra_pcie *pcie) tegra_msi_irq_chip.name, pcie); if (err < 0) { dev_err(dev, "failed to request IRQ: %d\n", err); - goto err; + goto free_irq_domain; + } + + /* Though the PCIe controller can address >32-bit address space, to + * facilitate endpoints that support only 32-bit MSI target address, + * the mask is set to 32-bit to make sure that MSI target address is + * always a 32-bit address + */ + err = dma_set_coherent_mask(dev, DMA_BIT_MASK(32)); + if (err < 0) { + dev_err(dev, "failed to set DMA coherent mask: %d\n", err); + goto free_irq; + } + + msi->virt = dma_alloc_attrs(dev, PAGE_SIZE, &msi->phys, GFP_KERNEL, + DMA_ATTR_NO_KERNEL_MAPPING); + if (!msi->virt) { + dev_err(dev, "failed to allocate DMA memory for MSI\n"); + err = -ENOMEM; + goto free_irq; } - /* setup AFI/FPCI range */ - msi->pages = __get_free_pages(GFP_KERNEL, 0); - msi->phys = virt_to_phys((void *)msi->pages); host->msi = &msi->chip; return 0; -err: +free_irq: + free_irq(msi->irq, pcie); +free_irq_domain: irq_domain_remove(msi->domain); return err; } @@ -1592,7 +1843,8 @@ static void tegra_pcie_msi_teardown(struct tegra_pcie *pcie) struct tegra_msi *msi = &pcie->msi; unsigned int i, irq; - free_pages(msi->pages, 0); + dma_free_attrs(pcie->dev, PAGE_SIZE, msi->virt, msi->phys, + DMA_ATTR_NO_KERNEL_MAPPING); if (msi->irq > 0) free_irq(msi->irq, pcie); @@ -1628,6 +1880,15 @@ static int tegra_pcie_disable_msi(struct tegra_pcie *pcie) return 0; } +static void tegra_pcie_disable_interrupts(struct tegra_pcie *pcie) +{ + u32 value; + + value = afi_readl(pcie, AFI_INTR_MASK); + value &= ~AFI_INTR_MASK_INT_MASK; + afi_writel(pcie, value, AFI_INTR_MASK); +} + static int tegra_pcie_get_xbar_config(struct tegra_pcie *pcie, u32 lanes, u32 *xbar) { @@ -1971,18 +2232,20 @@ static int tegra_pcie_parse_dt(struct tegra_pcie *pcie) struct tegra_pcie_port *rp; unsigned int index; u32 value; + char *label; err = of_pci_get_devfn(port); if (err < 0) { dev_err(dev, "failed to parse address: %d\n", err); - return err; + goto err_node_put; } index = PCI_SLOT(err); if (index < 1 || index > soc->num_ports) { dev_err(dev, "invalid port number: %d\n", index); - return -EINVAL; + err = -EINVAL; + goto err_node_put; } index--; @@ -1991,12 +2254,13 @@ static int tegra_pcie_parse_dt(struct tegra_pcie *pcie) if (err < 0) { dev_err(dev, "failed to parse # of lanes: %d\n", err); - return err; + goto err_node_put; } if (value > 16) { dev_err(dev, "invalid # of lanes: %u\n", value); - return -EINVAL; + err = -EINVAL; + goto err_node_put; } lanes |= value << (index << 3); @@ -2010,13 +2274,15 @@ static int tegra_pcie_parse_dt(struct tegra_pcie *pcie) lane += value; rp = devm_kzalloc(dev, sizeof(*rp), GFP_KERNEL); - if (!rp) - return -ENOMEM; + if (!rp) { + err = -ENOMEM; + goto err_node_put; + } err = of_address_to_resource(port, 0, &rp->regs); if (err < 0) { dev_err(dev, "failed to parse address: %d\n", err); - return err; + goto err_node_put; } INIT_LIST_HEAD(&rp->list); @@ -2029,6 +2295,31 @@ static int tegra_pcie_parse_dt(struct tegra_pcie *pcie) if (IS_ERR(rp->base)) return PTR_ERR(rp->base); + label = devm_kasprintf(dev, GFP_KERNEL, "pex-reset-%u", index); + if (!label) { + dev_err(dev, "failed to create reset GPIO label\n"); + return -ENOMEM; + } + + /* + * Returns -ENOENT if reset-gpios property is not populated + * and in this case fall back to using AFI per port register + * to toggle PERST# SFIO line. + */ + rp->reset_gpio = devm_gpiod_get_from_of_node(dev, port, + "reset-gpios", 0, + GPIOD_OUT_LOW, + label); + if (IS_ERR(rp->reset_gpio)) { + if (PTR_ERR(rp->reset_gpio) == -ENOENT) { + rp->reset_gpio = NULL; + } else { + dev_err(dev, "failed to get reset GPIO: %d\n", + err); + return PTR_ERR(rp->reset_gpio); + } + } + list_add_tail(&rp->list, &pcie->ports); } @@ -2043,6 +2334,10 @@ static int tegra_pcie_parse_dt(struct tegra_pcie *pcie) return err; return 0; + +err_node_put: + of_node_put(port); + return err; } /* @@ -2076,7 +2371,7 @@ static bool tegra_pcie_port_check_link(struct tegra_pcie_port *port) } while (--timeout); if (!timeout) { - dev_err(dev, "link %u down, retrying\n", port->index); + dev_dbg(dev, "link %u down, retrying\n", port->index); goto retry; } @@ -2098,6 +2393,64 @@ retry: return false; } +static void tegra_pcie_change_link_speed(struct tegra_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct tegra_pcie_port *port; + ktime_t deadline; + u32 value; + + list_for_each_entry(port, &pcie->ports, list) { + /* + * "Supported Link Speeds Vector" in "Link Capabilities 2" + * is not supported by Tegra. tegra_pcie_change_link_speed() + * is called only for Tegra chips which support Gen2. + * So there no harm if supported link speed is not verified. + */ + value = readl(port->base + RP_LINK_CONTROL_STATUS_2); + value &= ~PCI_EXP_LNKSTA_CLS; + value |= PCI_EXP_LNKSTA_CLS_5_0GB; + writel(value, port->base + RP_LINK_CONTROL_STATUS_2); + + /* + * Poll until link comes back from recovery to avoid race + * condition. + */ + deadline = ktime_add_us(ktime_get(), LINK_RETRAIN_TIMEOUT); + + while (ktime_before(ktime_get(), deadline)) { + value = readl(port->base + RP_LINK_CONTROL_STATUS); + if ((value & PCI_EXP_LNKSTA_LT) == 0) + break; + + usleep_range(2000, 3000); + } + + if (value & PCI_EXP_LNKSTA_LT) + dev_warn(dev, "PCIe port %u link is in recovery\n", + port->index); + + /* Retrain the link */ + value = readl(port->base + RP_LINK_CONTROL_STATUS); + value |= PCI_EXP_LNKCTL_RL; + writel(value, port->base + RP_LINK_CONTROL_STATUS); + + deadline = ktime_add_us(ktime_get(), LINK_RETRAIN_TIMEOUT); + + while (ktime_before(ktime_get(), deadline)) { + value = readl(port->base + RP_LINK_CONTROL_STATUS); + if ((value & PCI_EXP_LNKSTA_LT) == 0) + break; + + usleep_range(2000, 3000); + } + + if (value & PCI_EXP_LNKSTA_LT) + dev_err(dev, "failed to retrain link of port %u\n", + port->index); + } +} + static void tegra_pcie_enable_ports(struct tegra_pcie *pcie) { struct device *dev = pcie->dev; @@ -2108,7 +2461,12 @@ static void tegra_pcie_enable_ports(struct tegra_pcie *pcie) port->index, port->lanes); tegra_pcie_port_enable(port); + } + + /* Start LTSSM from Tegra side */ + reset_control_deassert(pcie->pcie_xrst); + list_for_each_entry_safe(port, tmp, &pcie->ports, list) { if (tegra_pcie_port_check_link(port)) continue; @@ -2117,12 +2475,17 @@ static void tegra_pcie_enable_ports(struct tegra_pcie *pcie) tegra_pcie_port_disable(port); tegra_pcie_port_free(port); } + + if (pcie->soc->has_gen2) + tegra_pcie_change_link_speed(pcie); } static void tegra_pcie_disable_ports(struct tegra_pcie *pcie) { struct tegra_pcie_port *port, *tmp; + reset_control_assert(pcie->pcie_xrst); + list_for_each_entry_safe(port, tmp, &pcie->ports, list) tegra_pcie_port_disable(port); } @@ -2136,6 +2499,7 @@ static const struct tegra_pcie_soc tegra20_pcie = { .num_ports = 2, .ports = tegra20_pcie_ports, .msi_base_shift = 0, + .afi_pex2_ctrl = 0x128, .pads_pll_ctl = PADS_PLL_CTL_TEGRA20, .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_DIV10, .pads_refclk_cfg0 = 0xfa5cfa5c, @@ -2146,6 +2510,12 @@ static const struct tegra_pcie_soc tegra20_pcie = { .has_gen2 = false, .force_pca_enable = false, .program_uphy = true, + .update_clamp_threshold = false, + .program_deskew_time = false, + .raw_violation_fixup = false, + .update_fc_timer = false, + .has_cache_bars = true, + .ectl.enable = false, }; static const struct tegra_pcie_port_soc tegra30_pcie_ports[] = { @@ -2169,6 +2539,12 @@ static const struct tegra_pcie_soc tegra30_pcie = { .has_gen2 = false, .force_pca_enable = false, .program_uphy = true, + .update_clamp_threshold = false, + .program_deskew_time = false, + .raw_violation_fixup = false, + .update_fc_timer = false, + .has_cache_bars = false, + .ectl.enable = false, }; static const struct tegra_pcie_soc tegra124_pcie = { @@ -2178,6 +2554,8 @@ static const struct tegra_pcie_soc tegra124_pcie = { .pads_pll_ctl = PADS_PLL_CTL_TEGRA30, .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN, .pads_refclk_cfg0 = 0x44ac44ac, + /* FC threshold is bit[25:18] */ + .update_fc_threshold = 0x03fc0000, .has_pex_clkreq_en = true, .has_pex_bias_ctrl = true, .has_intr_prsnt_sense = true, @@ -2185,6 +2563,12 @@ static const struct tegra_pcie_soc tegra124_pcie = { .has_gen2 = true, .force_pca_enable = false, .program_uphy = true, + .update_clamp_threshold = true, + .program_deskew_time = false, + .raw_violation_fixup = true, + .update_fc_timer = false, + .has_cache_bars = false, + .ectl.enable = false, }; static const struct tegra_pcie_soc tegra210_pcie = { @@ -2194,6 +2578,8 @@ static const struct tegra_pcie_soc tegra210_pcie = { .pads_pll_ctl = PADS_PLL_CTL_TEGRA30, .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN, .pads_refclk_cfg0 = 0x90b890b8, + /* FC threshold is bit[25:18] */ + .update_fc_threshold = 0x01800000, .has_pex_clkreq_en = true, .has_pex_bias_ctrl = true, .has_intr_prsnt_sense = true, @@ -2201,6 +2587,24 @@ static const struct tegra_pcie_soc tegra210_pcie = { .has_gen2 = true, .force_pca_enable = true, .program_uphy = true, + .update_clamp_threshold = true, + .program_deskew_time = true, + .raw_violation_fixup = false, + .update_fc_timer = true, + .has_cache_bars = false, + .ectl = { + .regs = { + .rp_ectl_2_r1 = 0x0000000f, + .rp_ectl_4_r1 = 0x00000067, + .rp_ectl_5_r1 = 0x55010000, + .rp_ectl_6_r1 = 0x00000001, + .rp_ectl_2_r2 = 0x0000008f, + .rp_ectl_4_r2 = 0x000000c7, + .rp_ectl_5_r2 = 0x55010000, + .rp_ectl_6_r2 = 0x00000001, + }, + .enable = true, + }, }; static const struct tegra_pcie_port_soc tegra186_pcie_ports[] = { @@ -2213,6 +2617,7 @@ static const struct tegra_pcie_soc tegra186_pcie = { .num_ports = 3, .ports = tegra186_pcie_ports, .msi_base_shift = 8, + .afi_pex2_ctrl = 0x19c, .pads_pll_ctl = PADS_PLL_CTL_TEGRA30, .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN, .pads_refclk_cfg0 = 0x80b880b8, @@ -2224,6 +2629,12 @@ static const struct tegra_pcie_soc tegra186_pcie = { .has_gen2 = true, .force_pca_enable = false, .program_uphy = false, + .update_clamp_threshold = false, + .program_deskew_time = false, + .raw_violation_fixup = false, + .update_fc_timer = false, + .has_cache_bars = false, + .ectl.enable = false, }; static const struct of_device_id tegra_pcie_of_match[] = { @@ -2466,16 +2877,32 @@ static int __maybe_unused tegra_pcie_pm_suspend(struct device *dev) { struct tegra_pcie *pcie = dev_get_drvdata(dev); struct tegra_pcie_port *port; + int err; list_for_each_entry(port, &pcie->ports, list) tegra_pcie_pme_turnoff(port); tegra_pcie_disable_ports(pcie); + /* + * AFI_INTR is unmasked in tegra_pcie_enable_controller(), mask it to + * avoid unwanted interrupts raised by AFI after pex_rst is asserted. + */ + tegra_pcie_disable_interrupts(pcie); + + if (pcie->soc->program_uphy) { + err = tegra_pcie_phy_power_off(pcie); + if (err < 0) + dev_err(dev, "failed to power off PHY(s): %d\n", err); + } + + reset_control_assert(pcie->pex_rst); + clk_disable_unprepare(pcie->pex_clk); + if (IS_ENABLED(CONFIG_PCI_MSI)) tegra_pcie_disable_msi(pcie); - tegra_pcie_disable_controller(pcie); + pinctrl_pm_select_idle_state(dev); tegra_pcie_power_off(pcie); return 0; @@ -2491,20 +2918,45 @@ static int __maybe_unused tegra_pcie_pm_resume(struct device *dev) dev_err(dev, "tegra pcie power on fail: %d\n", err); return err; } - err = tegra_pcie_enable_controller(pcie); - if (err) { - dev_err(dev, "tegra pcie controller enable fail: %d\n", err); + + err = pinctrl_pm_select_default_state(dev); + if (err < 0) { + dev_err(dev, "failed to disable PCIe IO DPD: %d\n", err); goto poweroff; } + + tegra_pcie_enable_controller(pcie); tegra_pcie_setup_translations(pcie); if (IS_ENABLED(CONFIG_PCI_MSI)) tegra_pcie_enable_msi(pcie); + err = clk_prepare_enable(pcie->pex_clk); + if (err) { + dev_err(dev, "failed to enable PEX clock: %d\n", err); + goto pex_dpd_enable; + } + + reset_control_deassert(pcie->pex_rst); + + if (pcie->soc->program_uphy) { + err = tegra_pcie_phy_power_on(pcie); + if (err < 0) { + dev_err(dev, "failed to power on PHY(s): %d\n", err); + goto disable_pex_clk; + } + } + + tegra_pcie_apply_pad_settings(pcie); tegra_pcie_enable_ports(pcie); return 0; +disable_pex_clk: + reset_control_assert(pcie->pex_rst); + clk_disable_unprepare(pcie->pex_clk); +pex_dpd_enable: + pinctrl_pm_select_idle_state(dev); poweroff: tegra_pcie_power_off(pcie); diff --git a/drivers/pci/controller/pcie-altera-msi.c b/drivers/pci/controller/pcie-altera-msi.c index 025ef7d9a046..16d938920ca5 100644 --- a/drivers/pci/controller/pcie-altera-msi.c +++ b/drivers/pci/controller/pcie-altera-msi.c @@ -10,6 +10,7 @@ #include <linux/interrupt.h> #include <linux/irqchip/chained_irq.h> #include <linux/init.h> +#include <linux/module.h> #include <linux/msi.h> #include <linux/of_address.h> #include <linux/of_irq.h> @@ -288,4 +289,13 @@ static int __init altera_msi_init(void) { return platform_driver_register(&altera_msi_driver); } + +static void __exit altera_msi_exit(void) +{ + platform_driver_unregister(&altera_msi_driver); +} + subsys_initcall(altera_msi_init); +MODULE_DEVICE_TABLE(of, altera_msi_of_match); +module_exit(altera_msi_exit); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pcie-altera.c b/drivers/pci/controller/pcie-altera.c index 27edcebd1726..d2497ca43828 100644 --- a/drivers/pci/controller/pcie-altera.c +++ b/drivers/pci/controller/pcie-altera.c @@ -10,6 +10,7 @@ #include <linux/interrupt.h> #include <linux/irqchip/chained_irq.h> #include <linux/init.h> +#include <linux/module.h> #include <linux/of_address.h> #include <linux/of_device.h> #include <linux/of_irq.h> @@ -43,6 +44,8 @@ #define S10_RP_RXCPL_STATUS 0x200C #define S10_RP_CFG_ADDR(pcie, reg) \ (((pcie)->hip_base) + (reg) + (1 << 20)) +#define S10_RP_SECONDARY(pcie) \ + readb(S10_RP_CFG_ADDR(pcie, PCI_SECONDARY_BUS)) /* TLP configuration type 0 and 1 */ #define TLP_FMTTYPE_CFGRD0 0x04 /* Configuration Read Type 0 */ @@ -54,14 +57,9 @@ #define TLP_WRITE_TAG 0x10 #define RP_DEVFN 0 #define TLP_REQ_ID(bus, devfn) (((bus) << 8) | (devfn)) -#define TLP_CFGRD_DW0(pcie, bus) \ - ((((bus == pcie->root_bus_nr) ? pcie->pcie_data->cfgrd0 \ - : pcie->pcie_data->cfgrd1) << 24) | \ - TLP_PAYLOAD_SIZE) -#define TLP_CFGWR_DW0(pcie, bus) \ - ((((bus == pcie->root_bus_nr) ? pcie->pcie_data->cfgwr0 \ - : pcie->pcie_data->cfgwr1) << 24) | \ - TLP_PAYLOAD_SIZE) +#define TLP_CFG_DW0(pcie, cfg) \ + (((cfg) << 24) | \ + TLP_PAYLOAD_SIZE) #define TLP_CFG_DW1(pcie, tag, be) \ (((TLP_REQ_ID(pcie->root_bus_nr, RP_DEVFN)) << 16) | (tag << 8) | (be)) #define TLP_CFG_DW2(bus, devfn, offset) \ @@ -321,14 +319,31 @@ static void s10_tlp_write_packet(struct altera_pcie *pcie, u32 *headers, s10_tlp_write_tx(pcie, data, RP_TX_EOP); } +static void get_tlp_header(struct altera_pcie *pcie, u8 bus, u32 devfn, + int where, u8 byte_en, bool read, u32 *headers) +{ + u8 cfg; + u8 cfg0 = read ? pcie->pcie_data->cfgrd0 : pcie->pcie_data->cfgwr0; + u8 cfg1 = read ? pcie->pcie_data->cfgrd1 : pcie->pcie_data->cfgwr1; + u8 tag = read ? TLP_READ_TAG : TLP_WRITE_TAG; + + if (pcie->pcie_data->version == ALTERA_PCIE_V1) + cfg = (bus == pcie->root_bus_nr) ? cfg0 : cfg1; + else + cfg = (bus > S10_RP_SECONDARY(pcie)) ? cfg0 : cfg1; + + headers[0] = TLP_CFG_DW0(pcie, cfg); + headers[1] = TLP_CFG_DW1(pcie, tag, byte_en); + headers[2] = TLP_CFG_DW2(bus, devfn, where); +} + static int tlp_cfg_dword_read(struct altera_pcie *pcie, u8 bus, u32 devfn, int where, u8 byte_en, u32 *value) { u32 headers[TLP_HDR_SIZE]; - headers[0] = TLP_CFGRD_DW0(pcie, bus); - headers[1] = TLP_CFG_DW1(pcie, TLP_READ_TAG, byte_en); - headers[2] = TLP_CFG_DW2(bus, devfn, where); + get_tlp_header(pcie, bus, devfn, where, byte_en, true, + headers); pcie->pcie_data->ops->tlp_write_pkt(pcie, headers, 0, false); @@ -341,9 +356,8 @@ static int tlp_cfg_dword_write(struct altera_pcie *pcie, u8 bus, u32 devfn, u32 headers[TLP_HDR_SIZE]; int ret; - headers[0] = TLP_CFGWR_DW0(pcie, bus); - headers[1] = TLP_CFG_DW1(pcie, TLP_WRITE_TAG, byte_en); - headers[2] = TLP_CFG_DW2(bus, devfn, where); + get_tlp_header(pcie, bus, devfn, where, byte_en, false, + headers); /* check alignment to Qword */ if ((where & 0x7) == 0) @@ -705,6 +719,13 @@ static int altera_pcie_init_irq_domain(struct altera_pcie *pcie) return 0; } +static void altera_pcie_irq_teardown(struct altera_pcie *pcie) +{ + irq_set_chained_handler_and_data(pcie->irq, NULL, NULL); + irq_domain_remove(pcie->irq_domain); + irq_dispose_mapping(pcie->irq); +} + static int altera_pcie_parse_dt(struct altera_pcie *pcie) { struct device *dev = &pcie->pdev->dev; @@ -798,6 +819,7 @@ static int altera_pcie_probe(struct platform_device *pdev) pcie = pci_host_bridge_priv(bridge); pcie->pdev = pdev; + platform_set_drvdata(pdev, pcie); match = of_match_device(altera_pcie_of_match, &pdev->dev); if (!match) @@ -855,13 +877,28 @@ static int altera_pcie_probe(struct platform_device *pdev) return ret; } +static int altera_pcie_remove(struct platform_device *pdev) +{ + struct altera_pcie *pcie = platform_get_drvdata(pdev); + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + + pci_stop_root_bus(bridge->bus); + pci_remove_root_bus(bridge->bus); + pci_free_resource_list(&pcie->resources); + altera_pcie_irq_teardown(pcie); + + return 0; +} + static struct platform_driver altera_pcie_driver = { .probe = altera_pcie_probe, + .remove = altera_pcie_remove, .driver = { .name = "altera-pcie", .of_match_table = altera_pcie_of_match, - .suppress_bind_attrs = true, }, }; -builtin_platform_driver(altera_pcie_driver); +MODULE_DEVICE_TABLE(of, altera_pcie_of_match); +module_platform_driver(altera_pcie_driver); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pcie-iproc-msi.c b/drivers/pci/controller/pcie-iproc-msi.c index cb3401a931f8..0a3f61be5625 100644 --- a/drivers/pci/controller/pcie-iproc-msi.c +++ b/drivers/pci/controller/pcie-iproc-msi.c @@ -367,7 +367,7 @@ static void iproc_msi_handler(struct irq_desc *desc) /* * Now go read the tail pointer again to see if there are new - * oustanding events that came in during the above window. + * outstanding events that came in during the above window. */ } while (true); diff --git a/drivers/pci/controller/pcie-iproc-platform.c b/drivers/pci/controller/pcie-iproc-platform.c index f30f5f3fb5c1..9ee6200a66f4 100644 --- a/drivers/pci/controller/pcie-iproc-platform.c +++ b/drivers/pci/controller/pcie-iproc-platform.c @@ -87,18 +87,15 @@ static int iproc_pcie_pltfm_probe(struct platform_device *pdev) /* * DT nodes are not used by all platforms that use the iProc PCIe - * core driver. For platforms that require explict inbound mapping + * core driver. For platforms that require explicit inbound mapping * configuration, "dma-ranges" would have been present in DT */ pcie->need_ib_cfg = of_property_read_bool(np, "dma-ranges"); /* PHY use is optional */ - pcie->phy = devm_phy_get(dev, "pcie-phy"); - if (IS_ERR(pcie->phy)) { - if (PTR_ERR(pcie->phy) == -EPROBE_DEFER) - return -EPROBE_DEFER; - pcie->phy = NULL; - } + pcie->phy = devm_phy_optional_get(dev, "pcie-phy"); + if (IS_ERR(pcie->phy)) + return PTR_ERR(pcie->phy); ret = devm_of_pci_get_host_bridge_resources(dev, 0, 0xff, &resources, &iobase); diff --git a/drivers/pci/controller/pcie-iproc.c b/drivers/pci/controller/pcie-iproc.c index c20fd6bd68fd..2d457bfdaf66 100644 --- a/drivers/pci/controller/pcie-iproc.c +++ b/drivers/pci/controller/pcie-iproc.c @@ -60,6 +60,10 @@ #define APB_ERR_EN_SHIFT 0 #define APB_ERR_EN BIT(APB_ERR_EN_SHIFT) +#define CFG_RD_SUCCESS 0 +#define CFG_RD_UR 1 +#define CFG_RD_CRS 2 +#define CFG_RD_CA 3 #define CFG_RETRY_STATUS 0xffff0001 #define CFG_RETRY_STATUS_TIMEOUT_US 500000 /* 500 milliseconds */ @@ -159,7 +163,7 @@ enum iproc_pcie_ib_map_type { * @size_unit: inbound mapping region size unit, could be SZ_1K, SZ_1M, or * SZ_1G * @region_sizes: list of supported inbound mapping region sizes in KB, MB, or - * GB, depedning on the size unit + * GB, depending on the size unit * @nr_sizes: number of supported inbound mapping region sizes * @nr_windows: number of supported inbound mapping windows for the region * @imap_addr_offset: register offset between the upper and lower 32-bit @@ -289,6 +293,9 @@ enum iproc_pcie_reg { IPROC_PCIE_IARR4, IPROC_PCIE_IMAP4, + /* config read status */ + IPROC_PCIE_CFG_RD_STATUS, + /* link status */ IPROC_PCIE_LINK_STATUS, @@ -350,6 +357,7 @@ static const u16 iproc_pcie_reg_paxb_v2[] = { [IPROC_PCIE_IMAP3] = 0xe08, [IPROC_PCIE_IARR4] = 0xe68, [IPROC_PCIE_IMAP4] = 0xe70, + [IPROC_PCIE_CFG_RD_STATUS] = 0xee0, [IPROC_PCIE_LINK_STATUS] = 0xf0c, [IPROC_PCIE_APB_ERR_EN] = 0xf40, }; @@ -474,10 +482,12 @@ static void __iomem *iproc_pcie_map_ep_cfg_reg(struct iproc_pcie *pcie, return (pcie->base + offset); } -static unsigned int iproc_pcie_cfg_retry(void __iomem *cfg_data_p) +static unsigned int iproc_pcie_cfg_retry(struct iproc_pcie *pcie, + void __iomem *cfg_data_p) { int timeout = CFG_RETRY_STATUS_TIMEOUT_US; unsigned int data; + u32 status; /* * As per PCIe spec r3.1, sec 2.3.2, CRS Software Visibility only @@ -498,6 +508,15 @@ static unsigned int iproc_pcie_cfg_retry(void __iomem *cfg_data_p) */ data = readl(cfg_data_p); while (data == CFG_RETRY_STATUS && timeout--) { + /* + * CRS state is set in CFG_RD status register + * This will handle the case where CFG_RETRY_STATUS is + * valid config data. + */ + status = iproc_pcie_read_reg(pcie, IPROC_PCIE_CFG_RD_STATUS); + if (status != CFG_RD_CRS) + return data; + udelay(1); data = readl(cfg_data_p); } @@ -576,7 +595,7 @@ static int iproc_pcie_config_read(struct pci_bus *bus, unsigned int devfn, if (!cfg_data_p) return PCIBIOS_DEVICE_NOT_FOUND; - data = iproc_pcie_cfg_retry(cfg_data_p); + data = iproc_pcie_cfg_retry(pcie, cfg_data_p); *val = data; if (size <= 2) @@ -936,8 +955,25 @@ static int iproc_pcie_setup_ob(struct iproc_pcie *pcie, u64 axi_addr, resource_size_t window_size = ob_map->window_sizes[size_idx] * SZ_1M; - if (size < window_size) - continue; + /* + * Keep iterating until we reach the last window and + * with the minimal window size at index zero. In this + * case, we take a compromise by mapping it using the + * minimum window size that can be supported + */ + if (size < window_size) { + if (size_idx > 0 || window_idx > 0) + continue; + + /* + * For the corner case of reaching the minimal + * window size that can be supported on the + * last window + */ + axi_addr = ALIGN_DOWN(axi_addr, window_size); + pci_addr = ALIGN_DOWN(pci_addr, window_size); + size = window_size; + } if (!IS_ALIGNED(axi_addr, window_size) || !IS_ALIGNED(pci_addr, window_size)) { @@ -1146,11 +1182,43 @@ err_ib: return ret; } +static int iproc_pcie_add_dma_range(struct device *dev, + struct list_head *resources, + struct of_pci_range *range) +{ + struct resource *res; + struct resource_entry *entry, *tmp; + struct list_head *head = resources; + + res = devm_kzalloc(dev, sizeof(struct resource), GFP_KERNEL); + if (!res) + return -ENOMEM; + + resource_list_for_each_entry(tmp, resources) { + if (tmp->res->start < range->cpu_addr) + head = &tmp->node; + } + + res->start = range->cpu_addr; + res->end = res->start + range->size - 1; + + entry = resource_list_create_entry(res, 0); + if (!entry) + return -ENOMEM; + + entry->offset = res->start - range->cpu_addr; + resource_list_add(entry, head); + + return 0; +} + static int iproc_pcie_map_dma_ranges(struct iproc_pcie *pcie) { + struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); struct of_pci_range range; struct of_pci_range_parser parser; int ret; + LIST_HEAD(resources); /* Get the dma-ranges from DT */ ret = of_pci_dma_range_parser_init(&parser, pcie->dev->of_node); @@ -1158,13 +1226,23 @@ static int iproc_pcie_map_dma_ranges(struct iproc_pcie *pcie) return ret; for_each_of_pci_range(&parser, &range) { + ret = iproc_pcie_add_dma_range(pcie->dev, + &resources, + &range); + if (ret) + goto out; /* Each range entry corresponds to an inbound mapping region */ ret = iproc_pcie_setup_ib(pcie, &range, IPROC_PCIE_IB_MAP_MEM); if (ret) - return ret; + goto out; } + list_splice_init(&resources, &host->dma_ranges); + return 0; +out: + pci_free_resource_list(&resources); + return ret; } static int iproce_pcie_get_msi(struct iproc_pcie *pcie, @@ -1320,14 +1398,18 @@ static int iproc_pcie_msi_enable(struct iproc_pcie *pcie) if (pcie->need_msi_steer) { ret = iproc_pcie_msi_steer(pcie, msi_node); if (ret) - return ret; + goto out_put_node; } /* * If another MSI controller is being used, the call below should fail * but that is okay */ - return iproc_msi_init(pcie, msi_node); + ret = iproc_msi_init(pcie, msi_node); + +out_put_node: + of_node_put(msi_node); + return ret; } static void iproc_pcie_msi_disable(struct iproc_pcie *pcie) @@ -1347,7 +1429,6 @@ static int iproc_pcie_rev_init(struct iproc_pcie *pcie) break; case IPROC_PCIE_PAXB: regs = iproc_pcie_reg_paxb; - pcie->iproc_cfg_read = true; pcie->has_apb_err_disable = true; if (pcie->need_ob_cfg) { pcie->ob_map = paxb_ob_map; @@ -1356,6 +1437,7 @@ static int iproc_pcie_rev_init(struct iproc_pcie *pcie) break; case IPROC_PCIE_PAXB_V2: regs = iproc_pcie_reg_paxb_v2; + pcie->iproc_cfg_read = true; pcie->has_apb_err_disable = true; if (pcie->need_ob_cfg) { pcie->ob_map = paxb_v2_ob_map; diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c index 0b6c72804e03..626a7c352dfd 100644 --- a/drivers/pci/controller/pcie-mediatek.c +++ b/drivers/pci/controller/pcie-mediatek.c @@ -73,6 +73,7 @@ #define PCIE_MSI_VECTOR 0x0c0 #define PCIE_CONF_VEND_ID 0x100 +#define PCIE_CONF_DEVICE_ID 0x102 #define PCIE_CONF_CLASS_ID 0x106 #define PCIE_INT_MASK 0x420 @@ -141,12 +142,16 @@ struct mtk_pcie_port; /** * struct mtk_pcie_soc - differentiate between host generations * @need_fix_class_id: whether this host's class ID needed to be fixed or not + * @need_fix_device_id: whether this host's device ID needed to be fixed or not + * @device_id: device ID which this host need to be fixed * @ops: pointer to configuration access functions * @startup: pointer to controller setting functions * @setup_irq: pointer to initialize IRQ functions */ struct mtk_pcie_soc { bool need_fix_class_id; + bool need_fix_device_id; + unsigned int device_id; struct pci_ops *ops; int (*startup)(struct mtk_pcie_port *port); int (*setup_irq)(struct mtk_pcie_port *port, struct device_node *node); @@ -578,6 +583,7 @@ static int mtk_pcie_init_irq_domain(struct mtk_pcie_port *port, port->irq_domain = irq_domain_add_linear(pcie_intc_node, PCI_NUM_INTX, &intx_domain_ops, port); + of_node_put(pcie_intc_node); if (!port->irq_domain) { dev_err(dev, "failed to get INTx IRQ domain\n"); return -ENODEV; @@ -629,8 +635,6 @@ static void mtk_pcie_intr_handler(struct irq_desc *desc) } chained_irq_exit(irqchip, desc); - - return; } static int mtk_pcie_setup_irq(struct mtk_pcie_port *port, @@ -695,6 +699,9 @@ static int mtk_pcie_startup_port_v2(struct mtk_pcie_port *port) writew(val, port->base + PCIE_CONF_CLASS_ID); } + if (soc->need_fix_device_id) + writew(soc->device_id, port->base + PCIE_CONF_DEVICE_ID); + /* 100ms timeout value should be enough for Gen1/2 training */ err = readl_poll_timeout(port->base + PCIE_LINK_STATUS_V2, val, !!(val & PCIE_PORT_LINKUP_V2), 20, @@ -915,49 +922,29 @@ static int mtk_pcie_parse_port(struct mtk_pcie *pcie, /* sys_ck might be divided into the following parts in some chips */ snprintf(name, sizeof(name), "ahb_ck%d", slot); - port->ahb_ck = devm_clk_get(dev, name); - if (IS_ERR(port->ahb_ck)) { - if (PTR_ERR(port->ahb_ck) == -EPROBE_DEFER) - return -EPROBE_DEFER; - - port->ahb_ck = NULL; - } + port->ahb_ck = devm_clk_get_optional(dev, name); + if (IS_ERR(port->ahb_ck)) + return PTR_ERR(port->ahb_ck); snprintf(name, sizeof(name), "axi_ck%d", slot); - port->axi_ck = devm_clk_get(dev, name); - if (IS_ERR(port->axi_ck)) { - if (PTR_ERR(port->axi_ck) == -EPROBE_DEFER) - return -EPROBE_DEFER; - - port->axi_ck = NULL; - } + port->axi_ck = devm_clk_get_optional(dev, name); + if (IS_ERR(port->axi_ck)) + return PTR_ERR(port->axi_ck); snprintf(name, sizeof(name), "aux_ck%d", slot); - port->aux_ck = devm_clk_get(dev, name); - if (IS_ERR(port->aux_ck)) { - if (PTR_ERR(port->aux_ck) == -EPROBE_DEFER) - return -EPROBE_DEFER; - - port->aux_ck = NULL; - } + port->aux_ck = devm_clk_get_optional(dev, name); + if (IS_ERR(port->aux_ck)) + return PTR_ERR(port->aux_ck); snprintf(name, sizeof(name), "obff_ck%d", slot); - port->obff_ck = devm_clk_get(dev, name); - if (IS_ERR(port->obff_ck)) { - if (PTR_ERR(port->obff_ck) == -EPROBE_DEFER) - return -EPROBE_DEFER; - - port->obff_ck = NULL; - } + port->obff_ck = devm_clk_get_optional(dev, name); + if (IS_ERR(port->obff_ck)) + return PTR_ERR(port->obff_ck); snprintf(name, sizeof(name), "pipe_ck%d", slot); - port->pipe_ck = devm_clk_get(dev, name); - if (IS_ERR(port->pipe_ck)) { - if (PTR_ERR(port->pipe_ck) == -EPROBE_DEFER) - return -EPROBE_DEFER; - - port->pipe_ck = NULL; - } + port->pipe_ck = devm_clk_get_optional(dev, name); + if (IS_ERR(port->pipe_ck)) + return PTR_ERR(port->pipe_ck); snprintf(name, sizeof(name), "pcie-rst%d", slot); port->reset = devm_reset_control_get_optional_exclusive(dev, name); @@ -1235,11 +1222,21 @@ static const struct mtk_pcie_soc mtk_pcie_soc_mt7622 = { .setup_irq = mtk_pcie_setup_irq, }; +static const struct mtk_pcie_soc mtk_pcie_soc_mt7629 = { + .need_fix_class_id = true, + .need_fix_device_id = true, + .device_id = PCI_DEVICE_ID_MEDIATEK_7629, + .ops = &mtk_pcie_ops_v2, + .startup = mtk_pcie_startup_port_v2, + .setup_irq = mtk_pcie_setup_irq, +}; + static const struct of_device_id mtk_pcie_ids[] = { { .compatible = "mediatek,mt2701-pcie", .data = &mtk_pcie_soc_v1 }, { .compatible = "mediatek,mt7623-pcie", .data = &mtk_pcie_soc_v1 }, { .compatible = "mediatek,mt2712-pcie", .data = &mtk_pcie_soc_mt2712 }, { .compatible = "mediatek,mt7622-pcie", .data = &mtk_pcie_soc_mt7622 }, + { .compatible = "mediatek,mt7629-pcie", .data = &mtk_pcie_soc_mt7629 }, {}, }; diff --git a/drivers/pci/controller/pcie-mobiveil.c b/drivers/pci/controller/pcie-mobiveil.c index 77052a0712d0..a45a6447b01d 100644 --- a/drivers/pci/controller/pcie-mobiveil.c +++ b/drivers/pci/controller/pcie-mobiveil.c @@ -31,91 +31,103 @@ * translation tables are grouped into windows, each window registers are * grouped into blocks of 4 or 16 registers each */ -#define PAB_REG_BLOCK_SIZE 16 -#define PAB_EXT_REG_BLOCK_SIZE 4 +#define PAB_REG_BLOCK_SIZE 16 +#define PAB_EXT_REG_BLOCK_SIZE 4 -#define PAB_REG_ADDR(offset, win) (offset + (win * PAB_REG_BLOCK_SIZE)) -#define PAB_EXT_REG_ADDR(offset, win) (offset + (win * PAB_EXT_REG_BLOCK_SIZE)) +#define PAB_REG_ADDR(offset, win) \ + (offset + (win * PAB_REG_BLOCK_SIZE)) +#define PAB_EXT_REG_ADDR(offset, win) \ + (offset + (win * PAB_EXT_REG_BLOCK_SIZE)) -#define LTSSM_STATUS 0x0404 -#define LTSSM_STATUS_L0_MASK 0x3f -#define LTSSM_STATUS_L0 0x2d +#define LTSSM_STATUS 0x0404 +#define LTSSM_STATUS_L0_MASK 0x3f +#define LTSSM_STATUS_L0 0x2d -#define PAB_CTRL 0x0808 -#define AMBA_PIO_ENABLE_SHIFT 0 -#define PEX_PIO_ENABLE_SHIFT 1 -#define PAGE_SEL_SHIFT 13 -#define PAGE_SEL_MASK 0x3f -#define PAGE_LO_MASK 0x3ff -#define PAGE_SEL_EN 0xc00 -#define PAGE_SEL_OFFSET_SHIFT 10 +#define PAB_CTRL 0x0808 +#define AMBA_PIO_ENABLE_SHIFT 0 +#define PEX_PIO_ENABLE_SHIFT 1 +#define PAGE_SEL_SHIFT 13 +#define PAGE_SEL_MASK 0x3f +#define PAGE_LO_MASK 0x3ff +#define PAGE_SEL_OFFSET_SHIFT 10 -#define PAB_AXI_PIO_CTRL 0x0840 -#define APIO_EN_MASK 0xf +#define PAB_AXI_PIO_CTRL 0x0840 +#define APIO_EN_MASK 0xf -#define PAB_PEX_PIO_CTRL 0x08c0 -#define PIO_ENABLE_SHIFT 0 +#define PAB_PEX_PIO_CTRL 0x08c0 +#define PIO_ENABLE_SHIFT 0 #define PAB_INTP_AMBA_MISC_ENB 0x0b0c -#define PAB_INTP_AMBA_MISC_STAT 0x0b1c +#define PAB_INTP_AMBA_MISC_STAT 0x0b1c #define PAB_INTP_INTX_MASK 0x01e0 #define PAB_INTP_MSI_MASK 0x8 -#define PAB_AXI_AMAP_CTRL(win) PAB_REG_ADDR(0x0ba0, win) -#define WIN_ENABLE_SHIFT 0 -#define WIN_TYPE_SHIFT 1 +#define PAB_AXI_AMAP_CTRL(win) PAB_REG_ADDR(0x0ba0, win) +#define WIN_ENABLE_SHIFT 0 +#define WIN_TYPE_SHIFT 1 +#define WIN_TYPE_MASK 0x3 +#define WIN_SIZE_MASK 0xfffffc00 #define PAB_EXT_AXI_AMAP_SIZE(win) PAB_EXT_REG_ADDR(0xbaf0, win) +#define PAB_EXT_AXI_AMAP_AXI_WIN(win) PAB_EXT_REG_ADDR(0x80a0, win) #define PAB_AXI_AMAP_AXI_WIN(win) PAB_REG_ADDR(0x0ba4, win) #define AXI_WINDOW_ALIGN_MASK 3 #define PAB_AXI_AMAP_PEX_WIN_L(win) PAB_REG_ADDR(0x0ba8, win) -#define PAB_BUS_SHIFT 24 -#define PAB_DEVICE_SHIFT 19 -#define PAB_FUNCTION_SHIFT 16 +#define PAB_BUS_SHIFT 24 +#define PAB_DEVICE_SHIFT 19 +#define PAB_FUNCTION_SHIFT 16 #define PAB_AXI_AMAP_PEX_WIN_H(win) PAB_REG_ADDR(0x0bac, win) #define PAB_INTP_AXI_PIO_CLASS 0x474 -#define PAB_PEX_AMAP_CTRL(win) PAB_REG_ADDR(0x4ba0, win) -#define AMAP_CTRL_EN_SHIFT 0 -#define AMAP_CTRL_TYPE_SHIFT 1 +#define PAB_PEX_AMAP_CTRL(win) PAB_REG_ADDR(0x4ba0, win) +#define AMAP_CTRL_EN_SHIFT 0 +#define AMAP_CTRL_TYPE_SHIFT 1 +#define AMAP_CTRL_TYPE_MASK 3 #define PAB_EXT_PEX_AMAP_SIZEN(win) PAB_EXT_REG_ADDR(0xbef0, win) +#define PAB_EXT_PEX_AMAP_AXI_WIN(win) PAB_EXT_REG_ADDR(0xb4a0, win) #define PAB_PEX_AMAP_AXI_WIN(win) PAB_REG_ADDR(0x4ba4, win) #define PAB_PEX_AMAP_PEX_WIN_L(win) PAB_REG_ADDR(0x4ba8, win) #define PAB_PEX_AMAP_PEX_WIN_H(win) PAB_REG_ADDR(0x4bac, win) /* starting offset of INTX bits in status register */ -#define PAB_INTX_START 5 +#define PAB_INTX_START 5 /* supported number of MSI interrupts */ -#define PCI_NUM_MSI 16 +#define PCI_NUM_MSI 16 /* MSI registers */ -#define MSI_BASE_LO_OFFSET 0x04 -#define MSI_BASE_HI_OFFSET 0x08 -#define MSI_SIZE_OFFSET 0x0c -#define MSI_ENABLE_OFFSET 0x14 -#define MSI_STATUS_OFFSET 0x18 -#define MSI_DATA_OFFSET 0x20 -#define MSI_ADDR_L_OFFSET 0x24 -#define MSI_ADDR_H_OFFSET 0x28 +#define MSI_BASE_LO_OFFSET 0x04 +#define MSI_BASE_HI_OFFSET 0x08 +#define MSI_SIZE_OFFSET 0x0c +#define MSI_ENABLE_OFFSET 0x14 +#define MSI_STATUS_OFFSET 0x18 +#define MSI_DATA_OFFSET 0x20 +#define MSI_ADDR_L_OFFSET 0x24 +#define MSI_ADDR_H_OFFSET 0x28 /* outbound and inbound window definitions */ -#define WIN_NUM_0 0 -#define WIN_NUM_1 1 -#define CFG_WINDOW_TYPE 0 -#define IO_WINDOW_TYPE 1 -#define MEM_WINDOW_TYPE 2 -#define IB_WIN_SIZE ((u64)256 * 1024 * 1024 * 1024) -#define MAX_PIO_WINDOWS 8 +#define WIN_NUM_0 0 +#define WIN_NUM_1 1 +#define CFG_WINDOW_TYPE 0 +#define IO_WINDOW_TYPE 1 +#define MEM_WINDOW_TYPE 2 +#define IB_WIN_SIZE ((u64)256 * 1024 * 1024 * 1024) +#define MAX_PIO_WINDOWS 8 /* Parameters for the waiting for link up routine */ -#define LINK_WAIT_MAX_RETRIES 10 -#define LINK_WAIT_MIN 90000 -#define LINK_WAIT_MAX 100000 +#define LINK_WAIT_MAX_RETRIES 10 +#define LINK_WAIT_MIN 90000 +#define LINK_WAIT_MAX 100000 + +#define PAGED_ADDR_BNDRY 0xc00 +#define OFFSET_TO_PAGE_ADDR(off) \ + ((off & PAGE_LO_MASK) | PAGED_ADDR_BNDRY) +#define OFFSET_TO_PAGE_IDX(off) \ + ((off >> PAGE_SEL_OFFSET_SHIFT) & PAGE_SEL_MASK) struct mobiveil_msi { /* MSI information */ struct mutex lock; /* protect bitmap variable */ @@ -145,15 +157,119 @@ struct mobiveil_pcie { struct mobiveil_msi msi; }; -static inline void csr_writel(struct mobiveil_pcie *pcie, const u32 value, - const u32 reg) +/* + * mobiveil_pcie_sel_page - routine to access paged register + * + * Registers whose address greater than PAGED_ADDR_BNDRY (0xc00) are paged, + * for this scheme to work extracted higher 6 bits of the offset will be + * written to pg_sel field of PAB_CTRL register and rest of the lower 10 + * bits enabled with PAGED_ADDR_BNDRY are used as offset of the register. + */ +static void mobiveil_pcie_sel_page(struct mobiveil_pcie *pcie, u8 pg_idx) +{ + u32 val; + + val = readl(pcie->csr_axi_slave_base + PAB_CTRL); + val &= ~(PAGE_SEL_MASK << PAGE_SEL_SHIFT); + val |= (pg_idx & PAGE_SEL_MASK) << PAGE_SEL_SHIFT; + + writel(val, pcie->csr_axi_slave_base + PAB_CTRL); +} + +static void *mobiveil_pcie_comp_addr(struct mobiveil_pcie *pcie, u32 off) +{ + if (off < PAGED_ADDR_BNDRY) { + /* For directly accessed registers, clear the pg_sel field */ + mobiveil_pcie_sel_page(pcie, 0); + return pcie->csr_axi_slave_base + off; + } + + mobiveil_pcie_sel_page(pcie, OFFSET_TO_PAGE_IDX(off)); + return pcie->csr_axi_slave_base + OFFSET_TO_PAGE_ADDR(off); +} + +static int mobiveil_pcie_read(void __iomem *addr, int size, u32 *val) { - writel_relaxed(value, pcie->csr_axi_slave_base + reg); + if ((uintptr_t)addr & (size - 1)) { + *val = 0; + return PCIBIOS_BAD_REGISTER_NUMBER; + } + + switch (size) { + case 4: + *val = readl(addr); + break; + case 2: + *val = readw(addr); + break; + case 1: + *val = readb(addr); + break; + default: + *val = 0; + return PCIBIOS_BAD_REGISTER_NUMBER; + } + + return PCIBIOS_SUCCESSFUL; } -static inline u32 csr_readl(struct mobiveil_pcie *pcie, const u32 reg) +static int mobiveil_pcie_write(void __iomem *addr, int size, u32 val) { - return readl_relaxed(pcie->csr_axi_slave_base + reg); + if ((uintptr_t)addr & (size - 1)) + return PCIBIOS_BAD_REGISTER_NUMBER; + + switch (size) { + case 4: + writel(val, addr); + break; + case 2: + writew(val, addr); + break; + case 1: + writeb(val, addr); + break; + default: + return PCIBIOS_BAD_REGISTER_NUMBER; + } + + return PCIBIOS_SUCCESSFUL; +} + +static u32 csr_read(struct mobiveil_pcie *pcie, u32 off, size_t size) +{ + void *addr; + u32 val; + int ret; + + addr = mobiveil_pcie_comp_addr(pcie, off); + + ret = mobiveil_pcie_read(addr, size, &val); + if (ret) + dev_err(&pcie->pdev->dev, "read CSR address failed\n"); + + return val; +} + +static void csr_write(struct mobiveil_pcie *pcie, u32 val, u32 off, size_t size) +{ + void *addr; + int ret; + + addr = mobiveil_pcie_comp_addr(pcie, off); + + ret = mobiveil_pcie_write(addr, size, val); + if (ret) + dev_err(&pcie->pdev->dev, "write CSR address failed\n"); +} + +static u32 csr_readl(struct mobiveil_pcie *pcie, u32 off) +{ + return csr_read(pcie, off, 0x4); +} + +static void csr_writel(struct mobiveil_pcie *pcie, u32 val, u32 off) +{ + csr_write(pcie, val, off, 0x4); } static bool mobiveil_pcie_link_up(struct mobiveil_pcie *pcie) @@ -174,7 +290,7 @@ static bool mobiveil_pcie_valid_device(struct pci_bus *bus, unsigned int devfn) * Do not read more than one device on the bus directly * attached to RC */ - if ((bus->primary == pcie->root_bus_nr) && (devfn > 0)) + if ((bus->primary == pcie->root_bus_nr) && (PCI_SLOT(devfn) > 0)) return false; return true; @@ -185,17 +301,17 @@ static bool mobiveil_pcie_valid_device(struct pci_bus *bus, unsigned int devfn) * root port or endpoint */ static void __iomem *mobiveil_pcie_map_bus(struct pci_bus *bus, - unsigned int devfn, int where) + unsigned int devfn, int where) { struct mobiveil_pcie *pcie = bus->sysdata; + u32 value; if (!mobiveil_pcie_valid_device(bus, devfn)) return NULL; - if (bus->number == pcie->root_bus_nr) { - /* RC config access */ + /* RC config access */ + if (bus->number == pcie->root_bus_nr) return pcie->csr_axi_slave_base + where; - } /* * EP config access (in Config/APIO space) @@ -203,10 +319,12 @@ static void __iomem *mobiveil_pcie_map_bus(struct pci_bus *bus, * (BDF) in PAB_AXI_AMAP_PEX_WIN_L0 Register. * Relies on pci_lock serialization */ - csr_writel(pcie, bus->number << PAB_BUS_SHIFT | - PCI_SLOT(devfn) << PAB_DEVICE_SHIFT | - PCI_FUNC(devfn) << PAB_FUNCTION_SHIFT, - PAB_AXI_AMAP_PEX_WIN_L(WIN_NUM_0)); + value = bus->number << PAB_BUS_SHIFT | + PCI_SLOT(devfn) << PAB_DEVICE_SHIFT | + PCI_FUNC(devfn) << PAB_FUNCTION_SHIFT; + + csr_writel(pcie, value, PAB_AXI_AMAP_PEX_WIN_L(WIN_NUM_0)); + return pcie->config_axi_slave_base + where; } @@ -241,24 +359,29 @@ static void mobiveil_pcie_isr(struct irq_desc *desc) /* Handle INTx */ if (intr_status & PAB_INTP_INTX_MASK) { - shifted_status = csr_readl(pcie, PAB_INTP_AMBA_MISC_STAT) >> - PAB_INTX_START; + shifted_status = csr_readl(pcie, PAB_INTP_AMBA_MISC_STAT); + shifted_status &= PAB_INTP_INTX_MASK; + shifted_status >>= PAB_INTX_START; do { for_each_set_bit(bit, &shifted_status, PCI_NUM_INTX) { virq = irq_find_mapping(pcie->intx_domain, - bit + 1); + bit + 1); if (virq) generic_handle_irq(virq); else - dev_err_ratelimited(dev, - "unexpected IRQ, INT%d\n", bit); + dev_err_ratelimited(dev, "unexpected IRQ, INT%d\n", + bit); - /* clear interrupt */ - csr_writel(pcie, - shifted_status << PAB_INTX_START, - PAB_INTP_AMBA_MISC_STAT); + /* clear interrupt handled */ + csr_writel(pcie, 1 << (PAB_INTX_START + bit), + PAB_INTP_AMBA_MISC_STAT); } - } while ((shifted_status >> PAB_INTX_START) != 0); + + shifted_status = csr_readl(pcie, + PAB_INTP_AMBA_MISC_STAT); + shifted_status &= PAB_INTP_INTX_MASK; + shifted_status >>= PAB_INTX_START; + } while (shifted_status != 0); } /* read extra MSI status register */ @@ -266,8 +389,7 @@ static void mobiveil_pcie_isr(struct irq_desc *desc) /* handle MSI interrupts */ while (msi_status & 1) { - msi_data = readl_relaxed(pcie->apb_csr_base - + MSI_DATA_OFFSET); + msi_data = readl_relaxed(pcie->apb_csr_base + MSI_DATA_OFFSET); /* * MSI_STATUS_OFFSET register gets updated to zero @@ -276,18 +398,18 @@ static void mobiveil_pcie_isr(struct irq_desc *desc) * two dummy reads. */ msi_addr_lo = readl_relaxed(pcie->apb_csr_base + - MSI_ADDR_L_OFFSET); + MSI_ADDR_L_OFFSET); msi_addr_hi = readl_relaxed(pcie->apb_csr_base + - MSI_ADDR_H_OFFSET); + MSI_ADDR_H_OFFSET); dev_dbg(dev, "MSI registers, data: %08x, addr: %08x:%08x\n", - msi_data, msi_addr_hi, msi_addr_lo); + msi_data, msi_addr_hi, msi_addr_lo); virq = irq_find_mapping(msi->dev_domain, msi_data); if (virq) generic_handle_irq(virq); msi_status = readl_relaxed(pcie->apb_csr_base + - MSI_STATUS_OFFSET); + MSI_STATUS_OFFSET); } /* Clear the interrupt status */ @@ -304,7 +426,7 @@ static int mobiveil_pcie_parse_dt(struct mobiveil_pcie *pcie) /* map config resource */ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, - "config_axi_slave"); + "config_axi_slave"); pcie->config_axi_slave_base = devm_pci_remap_cfg_resource(dev, res); if (IS_ERR(pcie->config_axi_slave_base)) return PTR_ERR(pcie->config_axi_slave_base); @@ -312,7 +434,7 @@ static int mobiveil_pcie_parse_dt(struct mobiveil_pcie *pcie) /* map csr resource */ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, - "csr_axi_slave"); + "csr_axi_slave"); pcie->csr_axi_slave_base = devm_pci_remap_cfg_resource(dev, res); if (IS_ERR(pcie->csr_axi_slave_base)) return PTR_ERR(pcie->csr_axi_slave_base); @@ -337,92 +459,53 @@ static int mobiveil_pcie_parse_dt(struct mobiveil_pcie *pcie) return -ENODEV; } - irq_set_chained_handler_and_data(pcie->irq, mobiveil_pcie_isr, pcie); - return 0; } -/* - * select_paged_register - routine to access paged register of root complex - * - * registers of RC are paged, for this scheme to work - * extracted higher 6 bits of the offset will be written to pg_sel - * field of PAB_CTRL register and rest of the lower 10 bits enabled with - * PAGE_SEL_EN are used as offset of the register. - */ -static void select_paged_register(struct mobiveil_pcie *pcie, u32 offset) -{ - int pab_ctrl_dw, pg_sel; - - /* clear pg_sel field */ - pab_ctrl_dw = csr_readl(pcie, PAB_CTRL); - pab_ctrl_dw = (pab_ctrl_dw & ~(PAGE_SEL_MASK << PAGE_SEL_SHIFT)); - - /* set pg_sel field */ - pg_sel = (offset >> PAGE_SEL_OFFSET_SHIFT) & PAGE_SEL_MASK; - pab_ctrl_dw |= ((pg_sel << PAGE_SEL_SHIFT)); - csr_writel(pcie, pab_ctrl_dw, PAB_CTRL); -} - -static void write_paged_register(struct mobiveil_pcie *pcie, - u32 val, u32 offset) -{ - u32 off = (offset & PAGE_LO_MASK) | PAGE_SEL_EN; - - select_paged_register(pcie, offset); - csr_writel(pcie, val, off); -} - -static u32 read_paged_register(struct mobiveil_pcie *pcie, u32 offset) -{ - u32 off = (offset & PAGE_LO_MASK) | PAGE_SEL_EN; - - select_paged_register(pcie, offset); - return csr_readl(pcie, off); -} - static void program_ib_windows(struct mobiveil_pcie *pcie, int win_num, - int pci_addr, u32 type, u64 size) + u64 cpu_addr, u64 pci_addr, u32 type, u64 size) { - int pio_ctrl_val; - int amap_ctrl_dw; + u32 value; u64 size64 = ~(size - 1); - if ((pcie->ib_wins_configured + 1) > pcie->ppio_wins) { + if (win_num >= pcie->ppio_wins) { dev_err(&pcie->pdev->dev, "ERROR: max inbound windows reached !\n"); return; } - pio_ctrl_val = csr_readl(pcie, PAB_PEX_PIO_CTRL); - csr_writel(pcie, - pio_ctrl_val | (1 << PIO_ENABLE_SHIFT), PAB_PEX_PIO_CTRL); - amap_ctrl_dw = read_paged_register(pcie, PAB_PEX_AMAP_CTRL(win_num)); - amap_ctrl_dw = (amap_ctrl_dw | (type << AMAP_CTRL_TYPE_SHIFT)); - amap_ctrl_dw = (amap_ctrl_dw | (1 << AMAP_CTRL_EN_SHIFT)); + value = csr_readl(pcie, PAB_PEX_AMAP_CTRL(win_num)); + value &= ~(AMAP_CTRL_TYPE_MASK << AMAP_CTRL_TYPE_SHIFT | WIN_SIZE_MASK); + value |= type << AMAP_CTRL_TYPE_SHIFT | 1 << AMAP_CTRL_EN_SHIFT | + (lower_32_bits(size64) & WIN_SIZE_MASK); + csr_writel(pcie, value, PAB_PEX_AMAP_CTRL(win_num)); + + csr_writel(pcie, upper_32_bits(size64), + PAB_EXT_PEX_AMAP_SIZEN(win_num)); - write_paged_register(pcie, amap_ctrl_dw | lower_32_bits(size64), - PAB_PEX_AMAP_CTRL(win_num)); + csr_writel(pcie, lower_32_bits(cpu_addr), + PAB_PEX_AMAP_AXI_WIN(win_num)); + csr_writel(pcie, upper_32_bits(cpu_addr), + PAB_EXT_PEX_AMAP_AXI_WIN(win_num)); - write_paged_register(pcie, upper_32_bits(size64), - PAB_EXT_PEX_AMAP_SIZEN(win_num)); + csr_writel(pcie, lower_32_bits(pci_addr), + PAB_PEX_AMAP_PEX_WIN_L(win_num)); + csr_writel(pcie, upper_32_bits(pci_addr), + PAB_PEX_AMAP_PEX_WIN_H(win_num)); - write_paged_register(pcie, pci_addr, PAB_PEX_AMAP_AXI_WIN(win_num)); - write_paged_register(pcie, pci_addr, PAB_PEX_AMAP_PEX_WIN_L(win_num)); - write_paged_register(pcie, 0, PAB_PEX_AMAP_PEX_WIN_H(win_num)); + pcie->ib_wins_configured++; } /* * routine to program the outbound windows */ static void program_ob_windows(struct mobiveil_pcie *pcie, int win_num, - u64 cpu_addr, u64 pci_addr, u32 config_io_bit, u64 size) + u64 cpu_addr, u64 pci_addr, u32 type, u64 size) { - - u32 value, type; + u32 value; u64 size64 = ~(size - 1); - if ((pcie->ob_wins_configured + 1) > pcie->apio_wins) { + if (win_num >= pcie->apio_wins) { dev_err(&pcie->pdev->dev, "ERROR: max outbound windows reached !\n"); return; @@ -432,28 +515,27 @@ static void program_ob_windows(struct mobiveil_pcie *pcie, int win_num, * program Enable Bit to 1, Type Bit to (00) base 2, AXI Window Size Bit * to 4 KB in PAB_AXI_AMAP_CTRL register */ - type = config_io_bit; value = csr_readl(pcie, PAB_AXI_AMAP_CTRL(win_num)); - csr_writel(pcie, 1 << WIN_ENABLE_SHIFT | type << WIN_TYPE_SHIFT | - lower_32_bits(size64), PAB_AXI_AMAP_CTRL(win_num)); + value &= ~(WIN_TYPE_MASK << WIN_TYPE_SHIFT | WIN_SIZE_MASK); + value |= 1 << WIN_ENABLE_SHIFT | type << WIN_TYPE_SHIFT | + (lower_32_bits(size64) & WIN_SIZE_MASK); + csr_writel(pcie, value, PAB_AXI_AMAP_CTRL(win_num)); - write_paged_register(pcie, upper_32_bits(size64), - PAB_EXT_AXI_AMAP_SIZE(win_num)); + csr_writel(pcie, upper_32_bits(size64), PAB_EXT_AXI_AMAP_SIZE(win_num)); /* * program AXI window base with appropriate value in * PAB_AXI_AMAP_AXI_WIN0 register */ - value = csr_readl(pcie, PAB_AXI_AMAP_AXI_WIN(win_num)); - csr_writel(pcie, cpu_addr & (~AXI_WINDOW_ALIGN_MASK), - PAB_AXI_AMAP_AXI_WIN(win_num)); - - value = csr_readl(pcie, PAB_AXI_AMAP_PEX_WIN_H(win_num)); + csr_writel(pcie, lower_32_bits(cpu_addr) & (~AXI_WINDOW_ALIGN_MASK), + PAB_AXI_AMAP_AXI_WIN(win_num)); + csr_writel(pcie, upper_32_bits(cpu_addr), + PAB_EXT_AXI_AMAP_AXI_WIN(win_num)); csr_writel(pcie, lower_32_bits(pci_addr), - PAB_AXI_AMAP_PEX_WIN_L(win_num)); + PAB_AXI_AMAP_PEX_WIN_L(win_num)); csr_writel(pcie, upper_32_bits(pci_addr), - PAB_AXI_AMAP_PEX_WIN_H(win_num)); + PAB_AXI_AMAP_PEX_WIN_H(win_num)); pcie->ob_wins_configured++; } @@ -469,7 +551,9 @@ static int mobiveil_bringup_link(struct mobiveil_pcie *pcie) usleep_range(LINK_WAIT_MIN, LINK_WAIT_MAX); } + dev_err(&pcie->pdev->dev, "link never came up\n"); + return -ETIMEDOUT; } @@ -482,50 +566,55 @@ static void mobiveil_pcie_enable_msi(struct mobiveil_pcie *pcie) msi->msi_pages_phys = (phys_addr_t)msg_addr; writel_relaxed(lower_32_bits(msg_addr), - pcie->apb_csr_base + MSI_BASE_LO_OFFSET); + pcie->apb_csr_base + MSI_BASE_LO_OFFSET); writel_relaxed(upper_32_bits(msg_addr), - pcie->apb_csr_base + MSI_BASE_HI_OFFSET); + pcie->apb_csr_base + MSI_BASE_HI_OFFSET); writel_relaxed(4096, pcie->apb_csr_base + MSI_SIZE_OFFSET); writel_relaxed(1, pcie->apb_csr_base + MSI_ENABLE_OFFSET); } static int mobiveil_host_init(struct mobiveil_pcie *pcie) { - u32 value, pab_ctrl, type = 0; - int err; - struct resource_entry *win, *tmp; - - err = mobiveil_bringup_link(pcie); - if (err) { - dev_info(&pcie->pdev->dev, "link bring-up failed\n"); - return err; - } + u32 value, pab_ctrl, type; + struct resource_entry *win; + + /* setup bus numbers */ + value = csr_readl(pcie, PCI_PRIMARY_BUS); + value &= 0xff000000; + value |= 0x00ff0100; + csr_writel(pcie, value, PCI_PRIMARY_BUS); /* * program Bus Master Enable Bit in Command Register in PAB Config * Space */ value = csr_readl(pcie, PCI_COMMAND); - csr_writel(pcie, value | PCI_COMMAND_IO | PCI_COMMAND_MEMORY | - PCI_COMMAND_MASTER, PCI_COMMAND); + value |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER; + csr_writel(pcie, value, PCI_COMMAND); /* * program PIO Enable Bit to 1 (and PEX PIO Enable to 1) in PAB_CTRL * register */ pab_ctrl = csr_readl(pcie, PAB_CTRL); - csr_writel(pcie, pab_ctrl | (1 << AMBA_PIO_ENABLE_SHIFT) | - (1 << PEX_PIO_ENABLE_SHIFT), PAB_CTRL); + pab_ctrl |= (1 << AMBA_PIO_ENABLE_SHIFT) | (1 << PEX_PIO_ENABLE_SHIFT); + csr_writel(pcie, pab_ctrl, PAB_CTRL); csr_writel(pcie, (PAB_INTP_INTX_MASK | PAB_INTP_MSI_MASK), - PAB_INTP_AMBA_MISC_ENB); + PAB_INTP_AMBA_MISC_ENB); /* * program PIO Enable Bit to 1 and Config Window Enable Bit to 1 in * PAB_AXI_PIO_CTRL Register */ value = csr_readl(pcie, PAB_AXI_PIO_CTRL); - csr_writel(pcie, value | APIO_EN_MASK, PAB_AXI_PIO_CTRL); + value |= APIO_EN_MASK; + csr_writel(pcie, value, PAB_AXI_PIO_CTRL); + + /* Enable PCIe PIO master */ + value = csr_readl(pcie, PAB_PEX_PIO_CTRL); + value |= 1 << PIO_ENABLE_SHIFT; + csr_writel(pcie, value, PAB_PEX_PIO_CTRL); /* * we'll program one outbound window for config reads and @@ -535,32 +624,38 @@ static int mobiveil_host_init(struct mobiveil_pcie *pcie) */ /* config outbound translation window */ - program_ob_windows(pcie, pcie->ob_wins_configured, - pcie->ob_io_res->start, 0, CFG_WINDOW_TYPE, - resource_size(pcie->ob_io_res)); + program_ob_windows(pcie, WIN_NUM_0, pcie->ob_io_res->start, 0, + CFG_WINDOW_TYPE, resource_size(pcie->ob_io_res)); /* memory inbound translation window */ - program_ib_windows(pcie, WIN_NUM_1, 0, MEM_WINDOW_TYPE, IB_WIN_SIZE); + program_ib_windows(pcie, WIN_NUM_0, 0, 0, MEM_WINDOW_TYPE, IB_WIN_SIZE); /* Get the I/O and memory ranges from DT */ - resource_list_for_each_entry_safe(win, tmp, &pcie->resources) { - type = 0; + resource_list_for_each_entry(win, &pcie->resources) { if (resource_type(win->res) == IORESOURCE_MEM) type = MEM_WINDOW_TYPE; - if (resource_type(win->res) == IORESOURCE_IO) + else if (resource_type(win->res) == IORESOURCE_IO) type = IO_WINDOW_TYPE; - if (type) { - /* configure outbound translation window */ - program_ob_windows(pcie, pcie->ob_wins_configured, - win->res->start, 0, type, - resource_size(win->res)); - } + else + continue; + + /* configure outbound translation window */ + program_ob_windows(pcie, pcie->ob_wins_configured, + win->res->start, + win->res->start - win->offset, + type, resource_size(win->res)); } + /* fixup for PCIe class register */ + value = csr_readl(pcie, PAB_INTP_AXI_PIO_CLASS); + value &= 0xff; + value |= (PCI_CLASS_BRIDGE_PCI << 16); + csr_writel(pcie, value, PAB_INTP_AXI_PIO_CLASS); + /* setup MSI hardware registers */ mobiveil_pcie_enable_msi(pcie); - return err; + return 0; } static void mobiveil_mask_intx_irq(struct irq_data *data) @@ -574,7 +669,8 @@ static void mobiveil_mask_intx_irq(struct irq_data *data) mask = 1 << ((data->hwirq + PAB_INTX_START) - 1); raw_spin_lock_irqsave(&pcie->intx_mask_lock, flags); shifted_val = csr_readl(pcie, PAB_INTP_AMBA_MISC_ENB); - csr_writel(pcie, (shifted_val & (~mask)), PAB_INTP_AMBA_MISC_ENB); + shifted_val &= ~mask; + csr_writel(pcie, shifted_val, PAB_INTP_AMBA_MISC_ENB); raw_spin_unlock_irqrestore(&pcie->intx_mask_lock, flags); } @@ -589,7 +685,8 @@ static void mobiveil_unmask_intx_irq(struct irq_data *data) mask = 1 << ((data->hwirq + PAB_INTX_START) - 1); raw_spin_lock_irqsave(&pcie->intx_mask_lock, flags); shifted_val = csr_readl(pcie, PAB_INTP_AMBA_MISC_ENB); - csr_writel(pcie, (shifted_val | mask), PAB_INTP_AMBA_MISC_ENB); + shifted_val |= mask; + csr_writel(pcie, shifted_val, PAB_INTP_AMBA_MISC_ENB); raw_spin_unlock_irqrestore(&pcie->intx_mask_lock, flags); } @@ -603,10 +700,11 @@ static struct irq_chip intx_irq_chip = { /* routine to setup the INTx related data */ static int mobiveil_pcie_intx_map(struct irq_domain *domain, unsigned int irq, - irq_hw_number_t hwirq) + irq_hw_number_t hwirq) { irq_set_chip_and_handler(irq, &intx_irq_chip, handle_level_irq); irq_set_chip_data(irq, domain->host_data); + return 0; } @@ -623,7 +721,7 @@ static struct irq_chip mobiveil_msi_irq_chip = { static struct msi_domain_info mobiveil_msi_domain_info = { .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX), + MSI_FLAG_PCI_MSIX), .chip = &mobiveil_msi_irq_chip, }; @@ -641,7 +739,7 @@ static void mobiveil_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) } static int mobiveil_msi_set_affinity(struct irq_data *irq_data, - const struct cpumask *mask, bool force) + const struct cpumask *mask, bool force) { return -EINVAL; } @@ -653,7 +751,8 @@ static struct irq_chip mobiveil_msi_bottom_irq_chip = { }; static int mobiveil_irq_msi_domain_alloc(struct irq_domain *domain, - unsigned int virq, unsigned int nr_irqs, void *args) + unsigned int virq, + unsigned int nr_irqs, void *args) { struct mobiveil_pcie *pcie = domain->host_data; struct mobiveil_msi *msi = &pcie->msi; @@ -673,13 +772,13 @@ static int mobiveil_irq_msi_domain_alloc(struct irq_domain *domain, mutex_unlock(&msi->lock); irq_domain_set_info(domain, virq, bit, &mobiveil_msi_bottom_irq_chip, - domain->host_data, handle_level_irq, - NULL, NULL); + domain->host_data, handle_level_irq, NULL, NULL); return 0; } static void mobiveil_irq_msi_domain_free(struct irq_domain *domain, - unsigned int virq, unsigned int nr_irqs) + unsigned int virq, + unsigned int nr_irqs) { struct irq_data *d = irq_domain_get_irq_data(domain, virq); struct mobiveil_pcie *pcie = irq_data_get_irq_chip_data(d); @@ -687,12 +786,11 @@ static void mobiveil_irq_msi_domain_free(struct irq_domain *domain, mutex_lock(&msi->lock); - if (!test_bit(d->hwirq, msi->msi_irq_in_use)) { + if (!test_bit(d->hwirq, msi->msi_irq_in_use)) dev_err(&pcie->pdev->dev, "trying to free unused MSI#%lu\n", d->hwirq); - } else { + else __clear_bit(d->hwirq, msi->msi_irq_in_use); - } mutex_unlock(&msi->lock); } @@ -716,12 +814,14 @@ static int mobiveil_allocate_msi_domains(struct mobiveil_pcie *pcie) } msi->msi_domain = pci_msi_create_irq_domain(fwnode, - &mobiveil_msi_domain_info, msi->dev_domain); + &mobiveil_msi_domain_info, + msi->dev_domain); if (!msi->msi_domain) { dev_err(dev, "failed to create MSI domain\n"); irq_domain_remove(msi->dev_domain); return -ENOMEM; } + return 0; } @@ -732,12 +832,12 @@ static int mobiveil_pcie_init_irq_domain(struct mobiveil_pcie *pcie) int ret; /* setup INTx */ - pcie->intx_domain = irq_domain_add_linear(node, - PCI_NUM_INTX, &intx_domain_ops, pcie); + pcie->intx_domain = irq_domain_add_linear(node, PCI_NUM_INTX, + &intx_domain_ops, pcie); if (!pcie->intx_domain) { dev_err(dev, "Failed to get a INTx IRQ domain\n"); - return -ENODEV; + return -ENOMEM; } raw_spin_lock_init(&pcie->intx_mask_lock); @@ -763,11 +863,9 @@ static int mobiveil_pcie_probe(struct platform_device *pdev) /* allocate the PCIe port */ bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); if (!bridge) - return -ENODEV; + return -ENOMEM; pcie = pci_host_bridge_priv(bridge); - if (!pcie) - return -ENOMEM; pcie->pdev = pdev; @@ -784,7 +882,7 @@ static int mobiveil_pcie_probe(struct platform_device *pdev) &pcie->resources, &iobase); if (ret) { dev_err(dev, "Getting bridge resources failed\n"); - return -ENOMEM; + return ret; } /* @@ -797,9 +895,6 @@ static int mobiveil_pcie_probe(struct platform_device *pdev) goto error; } - /* fixup for PCIe class register */ - csr_writel(pcie, 0x060402ab, PAB_INTP_AXI_PIO_CLASS); - /* initialize the IRQ domains */ ret = mobiveil_pcie_init_irq_domain(pcie); if (ret) { @@ -807,6 +902,8 @@ static int mobiveil_pcie_probe(struct platform_device *pdev) goto error; } + irq_set_chained_handler_and_data(pcie->irq, mobiveil_pcie_isr, pcie); + ret = devm_request_pci_bus_resources(dev, &pcie->resources); if (ret) goto error; @@ -820,6 +917,12 @@ static int mobiveil_pcie_probe(struct platform_device *pdev) bridge->map_irq = of_irq_parse_and_map_pci; bridge->swizzle_irq = pci_common_swizzle; + ret = mobiveil_bringup_link(pcie); + if (ret) { + dev_info(dev, "link bring-up failed\n"); + goto error; + } + /* setup the kernel resources for the newly added PCIe root bus */ ret = pci_scan_root_bus_bridge(bridge); if (ret) @@ -848,10 +951,10 @@ MODULE_DEVICE_TABLE(of, mobiveil_pcie_of_match); static struct platform_driver mobiveil_pcie_driver = { .probe = mobiveil_pcie_probe, .driver = { - .name = "mobiveil-pcie", - .of_match_table = mobiveil_pcie_of_match, - .suppress_bind_attrs = true, - }, + .name = "mobiveil-pcie", + .of_match_table = mobiveil_pcie_of_match, + .suppress_bind_attrs = true, + }, }; builtin_platform_driver(mobiveil_pcie_driver); diff --git a/drivers/pci/controller/pcie-rcar.c b/drivers/pci/controller/pcie-rcar.c index c8febb009454..f6a669a9af41 100644 --- a/drivers/pci/controller/pcie-rcar.c +++ b/drivers/pci/controller/pcie-rcar.c @@ -46,14 +46,15 @@ /* Transfer control */ #define PCIETCTLR 0x02000 -#define CFINIT 1 +#define DL_DOWN BIT(3) +#define CFINIT BIT(0) #define PCIETSTR 0x02004 -#define DATA_LINK_ACTIVE 1 +#define DATA_LINK_ACTIVE BIT(0) #define PCIEERRFR 0x02020 #define UNSUPPORTED_REQUEST BIT(4) #define PCIEMSIFR 0x02044 #define PCIEMSIALR 0x02048 -#define MSIFE 1 +#define MSIFE BIT(0) #define PCIEMSIAUR 0x0204c #define PCIEMSIIER 0x02050 @@ -94,6 +95,7 @@ #define MACCTLR 0x011058 #define SPEED_CHANGE BIT(24) #define SCRAMBLE_DISABLE BIT(27) +#define PMSR 0x01105c #define MACS2R 0x011078 #define MACCGSPSETR 0x011084 #define SPCNGRSN BIT(31) @@ -152,14 +154,13 @@ struct rcar_pcie { struct rcar_msi msi; }; -static void rcar_pci_write_reg(struct rcar_pcie *pcie, unsigned long val, - unsigned long reg) +static void rcar_pci_write_reg(struct rcar_pcie *pcie, u32 val, + unsigned int reg) { writel(val, pcie->base + reg); } -static unsigned long rcar_pci_read_reg(struct rcar_pcie *pcie, - unsigned long reg) +static u32 rcar_pci_read_reg(struct rcar_pcie *pcie, unsigned int reg) { return readl(pcie->base + reg); } @@ -171,7 +172,7 @@ enum { static void rcar_rmw32(struct rcar_pcie *pcie, int where, u32 mask, u32 data) { - int shift = 8 * (where & 3); + unsigned int shift = BITS_PER_BYTE * (where & 3); u32 val = rcar_pci_read_reg(pcie, where & ~3); val &= ~(mask << shift); @@ -181,7 +182,7 @@ static void rcar_rmw32(struct rcar_pcie *pcie, int where, u32 mask, u32 data) static u32 rcar_read_conf(struct rcar_pcie *pcie, int where) { - int shift = 8 * (where & 3); + unsigned int shift = BITS_PER_BYTE * (where & 3); u32 val = rcar_pci_read_reg(pcie, where & ~3); return val >> shift; @@ -192,7 +193,7 @@ static int rcar_pcie_config_access(struct rcar_pcie *pcie, unsigned char access_type, struct pci_bus *bus, unsigned int devfn, int where, u32 *data) { - int dev, func, reg, index; + unsigned int dev, func, reg, index; dev = PCI_SLOT(devfn); func = PCI_FUNC(devfn); @@ -281,12 +282,12 @@ static int rcar_pcie_read_conf(struct pci_bus *bus, unsigned int devfn, } if (size == 1) - *val = (*val >> (8 * (where & 3))) & 0xff; + *val = (*val >> (BITS_PER_BYTE * (where & 3))) & 0xff; else if (size == 2) - *val = (*val >> (8 * (where & 2))) & 0xffff; + *val = (*val >> (BITS_PER_BYTE * (where & 2))) & 0xffff; - dev_dbg(&bus->dev, "pcie-config-read: bus=%3d devfn=0x%04x where=0x%04x size=%d val=0x%08lx\n", - bus->number, devfn, where, size, (unsigned long)*val); + dev_dbg(&bus->dev, "pcie-config-read: bus=%3d devfn=0x%04x where=0x%04x size=%d val=0x%08x\n", + bus->number, devfn, where, size, *val); return ret; } @@ -296,23 +297,24 @@ static int rcar_pcie_write_conf(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val) { struct rcar_pcie *pcie = bus->sysdata; - int shift, ret; + unsigned int shift; u32 data; + int ret; ret = rcar_pcie_config_access(pcie, RCAR_PCI_ACCESS_READ, bus, devfn, where, &data); if (ret != PCIBIOS_SUCCESSFUL) return ret; - dev_dbg(&bus->dev, "pcie-config-write: bus=%3d devfn=0x%04x where=0x%04x size=%d val=0x%08lx\n", - bus->number, devfn, where, size, (unsigned long)val); + dev_dbg(&bus->dev, "pcie-config-write: bus=%3d devfn=0x%04x where=0x%04x size=%d val=0x%08x\n", + bus->number, devfn, where, size, val); if (size == 1) { - shift = 8 * (where & 3); + shift = BITS_PER_BYTE * (where & 3); data &= ~(0xff << shift); data |= ((val & 0xff) << shift); } else if (size == 2) { - shift = 8 * (where & 2); + shift = BITS_PER_BYTE * (where & 2); data &= ~(0xffff << shift); data |= ((val & 0xffff) << shift); } else @@ -507,10 +509,10 @@ static int phy_wait_for_ack(struct rcar_pcie *pcie) } static void phy_write_reg(struct rcar_pcie *pcie, - unsigned int rate, unsigned int addr, - unsigned int lane, unsigned int data) + unsigned int rate, u32 addr, + unsigned int lane, u32 data) { - unsigned long phyaddr; + u32 phyaddr; phyaddr = WRITE_CMD | ((rate & 1) << RATE_POS) | @@ -738,15 +740,15 @@ static irqreturn_t rcar_pcie_msi_irq(int irq, void *data) while (reg) { unsigned int index = find_first_bit(®, 32); - unsigned int irq; + unsigned int msi_irq; /* clear the interrupt */ rcar_pci_write_reg(pcie, 1 << index, PCIEMSIFR); - irq = irq_find_mapping(msi->domain, index); - if (irq) { + msi_irq = irq_find_mapping(msi->domain, index); + if (msi_irq) { if (test_bit(index, msi->used)) - generic_handle_irq(irq); + generic_handle_irq(msi_irq); else dev_info(dev, "unhandled MSI\n"); } else { @@ -890,7 +892,7 @@ static int rcar_pcie_enable_msi(struct rcar_pcie *pcie) { struct device *dev = pcie->dev; struct rcar_msi *msi = &pcie->msi; - unsigned long base; + phys_addr_t base; int err, i; mutex_init(&msi->lock); @@ -929,10 +931,14 @@ static int rcar_pcie_enable_msi(struct rcar_pcie *pcie) /* setup MSI data target */ msi->pages = __get_free_pages(GFP_KERNEL, 0); + if (!msi->pages) { + err = -ENOMEM; + goto err; + } base = virt_to_phys((void *)msi->pages); - rcar_pci_write_reg(pcie, base | MSIFE, PCIEMSIALR); - rcar_pci_write_reg(pcie, 0, PCIEMSIAUR); + rcar_pci_write_reg(pcie, lower_32_bits(base) | MSIFE, PCIEMSIALR); + rcar_pci_write_reg(pcie, upper_32_bits(base), PCIEMSIAUR); /* enable all MSI interrupts */ rcar_pci_write_reg(pcie, 0xffffffff, PCIEMSIIER); @@ -1118,7 +1124,7 @@ static int rcar_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct rcar_pcie *pcie; - unsigned int data; + u32 data; int err; int (*phy_init_fn)(struct rcar_pcie *); struct pci_host_bridge *bridge; @@ -1130,6 +1136,7 @@ static int rcar_pcie_probe(struct platform_device *pdev) pcie = pci_host_bridge_priv(bridge); pcie->dev = dev; + platform_set_drvdata(pdev, pcie); err = pci_parse_request_of_pci_ranges(dev, &pcie->resources, NULL); if (err) @@ -1221,10 +1228,28 @@ err_free_bridge: return err; } +static int rcar_pcie_resume_noirq(struct device *dev) +{ + struct rcar_pcie *pcie = dev_get_drvdata(dev); + + if (rcar_pci_read_reg(pcie, PMSR) && + !(rcar_pci_read_reg(pcie, PCIETCTLR) & DL_DOWN)) + return 0; + + /* Re-establish the PCIe link */ + rcar_pci_write_reg(pcie, CFINIT, PCIETCTLR); + return rcar_pcie_wait_for_dl(pcie); +} + +static const struct dev_pm_ops rcar_pcie_pm_ops = { + .resume_noirq = rcar_pcie_resume_noirq, +}; + static struct platform_driver rcar_pcie_driver = { .driver = { .name = "rcar-pcie", .of_match_table = rcar_pcie_of_match, + .pm = &rcar_pcie_pm_ops, .suppress_bind_attrs = true, }, .probe = rcar_pcie_probe, diff --git a/drivers/pci/controller/pcie-rockchip-ep.c b/drivers/pci/controller/pcie-rockchip-ep.c index a5d799e2dff2..d743b0a48988 100644 --- a/drivers/pci/controller/pcie-rockchip-ep.c +++ b/drivers/pci/controller/pcie-rockchip-ep.c @@ -350,7 +350,7 @@ static void rockchip_pcie_ep_assert_intx(struct rockchip_pcie_ep *ep, u8 fn, struct rockchip_pcie *rockchip = &ep->rockchip; u32 r = ep->max_regions - 1; u32 offset; - u16 status; + u32 status; u8 msg_code; if (unlikely(ep->irq_pci_addr != ROCKCHIP_PCIE_EP_PCI_LEGACY_IRQ_ADDR || diff --git a/drivers/pci/controller/pcie-rockchip-host.c b/drivers/pci/controller/pcie-rockchip-host.c index 1372d270764f..ef8e677ce9d1 100644 --- a/drivers/pci/controller/pcie-rockchip-host.c +++ b/drivers/pci/controller/pcie-rockchip-host.c @@ -608,29 +608,29 @@ static int rockchip_pcie_parse_host_dt(struct rockchip_pcie *rockchip) rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v"); if (IS_ERR(rockchip->vpcie12v)) { - if (PTR_ERR(rockchip->vpcie12v) == -EPROBE_DEFER) - return -EPROBE_DEFER; + if (PTR_ERR(rockchip->vpcie12v) != -ENODEV) + return PTR_ERR(rockchip->vpcie12v); dev_info(dev, "no vpcie12v regulator found\n"); } rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3"); if (IS_ERR(rockchip->vpcie3v3)) { - if (PTR_ERR(rockchip->vpcie3v3) == -EPROBE_DEFER) - return -EPROBE_DEFER; + if (PTR_ERR(rockchip->vpcie3v3) != -ENODEV) + return PTR_ERR(rockchip->vpcie3v3); dev_info(dev, "no vpcie3v3 regulator found\n"); } rockchip->vpcie1v8 = devm_regulator_get_optional(dev, "vpcie1v8"); if (IS_ERR(rockchip->vpcie1v8)) { - if (PTR_ERR(rockchip->vpcie1v8) == -EPROBE_DEFER) - return -EPROBE_DEFER; + if (PTR_ERR(rockchip->vpcie1v8) != -ENODEV) + return PTR_ERR(rockchip->vpcie1v8); dev_info(dev, "no vpcie1v8 regulator found\n"); } rockchip->vpcie0v9 = devm_regulator_get_optional(dev, "vpcie0v9"); if (IS_ERR(rockchip->vpcie0v9)) { - if (PTR_ERR(rockchip->vpcie0v9) == -EPROBE_DEFER) - return -EPROBE_DEFER; + if (PTR_ERR(rockchip->vpcie0v9) != -ENODEV) + return PTR_ERR(rockchip->vpcie0v9); dev_info(dev, "no vpcie0v9 regulator found\n"); } @@ -724,6 +724,7 @@ static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip) rockchip->irq_domain = irq_domain_add_linear(intc, PCI_NUM_INTX, &intx_domain_ops, rockchip); + of_node_put(intc); if (!rockchip->irq_domain) { dev_err(dev, "failed to get a INTx IRQ domain\n"); return -EINVAL; diff --git a/drivers/pci/controller/pcie-xilinx-nwl.c b/drivers/pci/controller/pcie-xilinx-nwl.c index 81538d77f790..45c0f344ccd1 100644 --- a/drivers/pci/controller/pcie-xilinx-nwl.c +++ b/drivers/pci/controller/pcie-xilinx-nwl.c @@ -438,11 +438,10 @@ static const struct irq_domain_ops legacy_domain_ops = { #ifdef CONFIG_PCI_MSI static struct irq_chip nwl_msi_irq_chip = { .name = "nwl_pcie:msi", - .irq_enable = unmask_msi_irq, - .irq_disable = mask_msi_irq, - .irq_mask = mask_msi_irq, - .irq_unmask = unmask_msi_irq, - + .irq_enable = pci_msi_unmask_irq, + .irq_disable = pci_msi_mask_irq, + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, }; static struct msi_domain_info nwl_msi_domain_info = { @@ -483,15 +482,13 @@ static int nwl_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, int i; mutex_lock(&msi->lock); - bit = bitmap_find_next_zero_area(msi->bitmap, INT_PCI_MSI_NR, 0, - nr_irqs, 0); - if (bit >= INT_PCI_MSI_NR) { + bit = bitmap_find_free_region(msi->bitmap, INT_PCI_MSI_NR, + get_count_order(nr_irqs)); + if (bit < 0) { mutex_unlock(&msi->lock); return -ENOSPC; } - bitmap_set(msi->bitmap, bit, nr_irqs); - for (i = 0; i < nr_irqs; i++) { irq_domain_set_info(domain, virq + i, bit + i, &nwl_irq_chip, domain->host_data, handle_simple_irq, @@ -509,7 +506,8 @@ static void nwl_irq_domain_free(struct irq_domain *domain, unsigned int virq, struct nwl_msi *msi = &pcie->msi; mutex_lock(&msi->lock); - bitmap_clear(msi->bitmap, data->hwirq, nr_irqs); + bitmap_release_region(msi->bitmap, data->hwirq, + get_count_order(nr_irqs)); mutex_unlock(&msi->lock); } diff --git a/drivers/pci/controller/pcie-xilinx.c b/drivers/pci/controller/pcie-xilinx.c index 9bd1a35cd5d8..5bf3af3b28e6 100644 --- a/drivers/pci/controller/pcie-xilinx.c +++ b/drivers/pci/controller/pcie-xilinx.c @@ -336,14 +336,19 @@ static const struct irq_domain_ops msi_domain_ops = { * xilinx_pcie_enable_msi - Enable MSI support * @port: PCIe port information */ -static void xilinx_pcie_enable_msi(struct xilinx_pcie_port *port) +static int xilinx_pcie_enable_msi(struct xilinx_pcie_port *port) { phys_addr_t msg_addr; port->msi_pages = __get_free_pages(GFP_KERNEL, 0); + if (!port->msi_pages) + return -ENOMEM; + msg_addr = virt_to_phys((void *)port->msi_pages); pcie_write(port, 0x0, XILINX_PCIE_REG_MSIBASE1); pcie_write(port, msg_addr, XILINX_PCIE_REG_MSIBASE2); + + return 0; } /* INTx Functions */ @@ -498,6 +503,7 @@ static int xilinx_pcie_init_irq_domain(struct xilinx_pcie_port *port) struct device *dev = port->dev; struct device_node *node = dev->of_node; struct device_node *pcie_intc_node; + int ret; /* Setup INTx */ pcie_intc_node = of_get_next_child(node, NULL); @@ -526,7 +532,9 @@ static int xilinx_pcie_init_irq_domain(struct xilinx_pcie_port *port) return -ENODEV; } - xilinx_pcie_enable_msi(port); + ret = xilinx_pcie_enable_msi(port); + if (ret) + return ret; } return 0; diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c index cf6816b55b5e..a35d3f3996d7 100644 --- a/drivers/pci/controller/vmd.c +++ b/drivers/pci/controller/vmd.c @@ -31,6 +31,9 @@ #define PCI_REG_VMLOCK 0x70 #define MB2_SHADOW_EN(vmlock) (vmlock & 0x2) +#define MB2_SHADOW_OFFSET 0x2000 +#define MB2_SHADOW_SIZE 16 + enum vmd_features { /* * Device may contain registers which hint the physical location of the @@ -94,11 +97,10 @@ struct vmd_dev { struct resource resources[3]; struct irq_domain *irq_domain; struct pci_bus *bus; + u8 busn_start; -#ifdef CONFIG_X86_DEV_DMA_OPS struct dma_map_ops dma_ops; struct dma_domain dma_domain; -#endif }; static inline struct vmd_dev *vmd_from_bus(struct pci_bus *bus) @@ -293,7 +295,6 @@ static struct msi_domain_info vmd_msi_domain_info = { .chip = &vmd_msi_controller, }; -#ifdef CONFIG_X86_DEV_DMA_OPS /* * VMD replaces the requester ID with its own. DMA mappings for devices in a * VMD domain need to be mapped for the VMD, not the device requiring @@ -438,16 +439,13 @@ static void vmd_setup_dma_ops(struct vmd_dev *vmd) add_dma_domain(domain); } #undef ASSIGN_VMD_DMA_OPS -#else -static void vmd_teardown_dma_ops(struct vmd_dev *vmd) {} -static void vmd_setup_dma_ops(struct vmd_dev *vmd) {} -#endif static char __iomem *vmd_cfg_addr(struct vmd_dev *vmd, struct pci_bus *bus, unsigned int devfn, int reg, int len) { char __iomem *addr = vmd->cfgbar + - (bus->number << 20) + (devfn << 12) + reg; + ((bus->number - vmd->busn_start) << 20) + + (devfn << 12) + reg; if ((addr - vmd->cfgbar) + len >= resource_size(&vmd->dev->resource[VMD_CFGBAR])) @@ -570,7 +568,7 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) unsigned long flags; LIST_HEAD(resources); resource_size_t offset[2] = {0}; - resource_size_t membar2_offset = 0x2000, busn_start = 0; + resource_size_t membar2_offset = 0x2000; struct pci_bus *child; /* @@ -583,7 +581,7 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) u32 vmlock; int ret; - membar2_offset = 0x2018; + membar2_offset = MB2_SHADOW_OFFSET + MB2_SHADOW_SIZE; ret = pci_read_config_dword(vmd->dev, PCI_REG_VMLOCK, &vmlock); if (ret || vmlock == ~0) return -ENODEV; @@ -595,9 +593,9 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) if (!membar2) return -ENOMEM; offset[0] = vmd->dev->resource[VMD_MEMBAR1].start - - readq(membar2 + 0x2008); + readq(membar2 + MB2_SHADOW_OFFSET); offset[1] = vmd->dev->resource[VMD_MEMBAR2].start - - readq(membar2 + 0x2010); + readq(membar2 + MB2_SHADOW_OFFSET + 8); pci_iounmap(vmd->dev, membar2); } } @@ -613,14 +611,14 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) pci_read_config_dword(vmd->dev, PCI_REG_VMCONFIG, &vmconfig); if (BUS_RESTRICT_CAP(vmcap) && (BUS_RESTRICT_CFG(vmconfig) == 0x1)) - busn_start = 128; + vmd->busn_start = 128; } res = &vmd->dev->resource[VMD_CFGBAR]; vmd->resources[0] = (struct resource) { .name = "VMD CFGBAR", - .start = busn_start, - .end = busn_start + (resource_size(res) >> 20) - 1, + .start = vmd->busn_start, + .end = vmd->busn_start + (resource_size(res) >> 20) - 1, .flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED, }; @@ -634,7 +632,7 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) * 32-bit resources. __pci_assign_resource() enforces that * artificial restriction to make sure everything will fit. * - * The only way we could use a 64-bit non-prefechable MEMBAR is + * The only way we could use a 64-bit non-prefetchable MEMBAR is * if its address is <4GB so that we can convert it to a 32-bit * resource. To be visible to the host OS, all VMD endpoints must * be initially configured by platform BIOS, which includes setting @@ -688,8 +686,8 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) pci_add_resource_offset(&resources, &vmd->resources[1], offset[0]); pci_add_resource_offset(&resources, &vmd->resources[2], offset[1]); - vmd->bus = pci_create_root_bus(&vmd->dev->dev, busn_start, &vmd_ops, - sd, &resources); + vmd->bus = pci_create_root_bus(&vmd->dev->dev, vmd->busn_start, + &vmd_ops, sd, &resources); if (!vmd->bus) { pci_free_resource_list(&resources); irq_domain_remove(vmd->irq_domain); |