Loading...
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 | /*
* Copyright 2014-2016 IBM Corp.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version
* 2 of the License, or (at your option) any later version.
*/
#include <linux/module.h>
#include <linux/msi.h>
#include <asm/pci-bridge.h>
#include <asm/pnv-pci.h>
#include <asm/opal.h>
#include <misc/cxl.h>
#include "pci.h"
struct device_node *pnv_pci_get_phb_node(struct pci_dev *dev)
{
struct pci_controller *hose = pci_bus_to_host(dev->bus);
return of_node_get(hose->dn);
}
EXPORT_SYMBOL(pnv_pci_get_phb_node);
int pnv_phb_to_cxl_mode(struct pci_dev *dev, uint64_t mode)
{
struct pci_controller *hose = pci_bus_to_host(dev->bus);
struct pnv_phb *phb = hose->private_data;
struct pnv_ioda_pe *pe;
int rc;
pe = pnv_ioda_get_pe(dev);
if (!pe)
return -ENODEV;
pe_info(pe, "Switching PHB to CXL\n");
rc = opal_pci_set_phb_cxl_mode(phb->opal_id, mode, pe->pe_number);
if (rc == OPAL_UNSUPPORTED)
dev_err(&dev->dev, "Required cxl mode not supported by firmware - update skiboot\n");
else if (rc)
dev_err(&dev->dev, "opal_pci_set_phb_cxl_mode failed: %i\n", rc);
return rc;
}
EXPORT_SYMBOL(pnv_phb_to_cxl_mode);
/* Find PHB for cxl dev and allocate MSI hwirqs?
* Returns the absolute hardware IRQ number
*/
int pnv_cxl_alloc_hwirqs(struct pci_dev *dev, int num)
{
struct pci_controller *hose = pci_bus_to_host(dev->bus);
struct pnv_phb *phb = hose->private_data;
int hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, num);
if (hwirq < 0) {
dev_warn(&dev->dev, "Failed to find a free MSI\n");
return -ENOSPC;
}
return phb->msi_base + hwirq;
}
EXPORT_SYMBOL(pnv_cxl_alloc_hwirqs);
void pnv_cxl_release_hwirqs(struct pci_dev *dev, int hwirq, int num)
{
struct pci_controller *hose = pci_bus_to_host(dev->bus);
struct pnv_phb *phb = hose->private_data;
msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq - phb->msi_base, num);
}
EXPORT_SYMBOL(pnv_cxl_release_hwirqs);
void pnv_cxl_release_hwirq_ranges(struct cxl_irq_ranges *irqs,
struct pci_dev *dev)
{
struct pci_controller *hose = pci_bus_to_host(dev->bus);
struct pnv_phb *phb = hose->private_data;
int i, hwirq;
for (i = 1; i < CXL_IRQ_RANGES; i++) {
if (!irqs->range[i])
continue;
pr_devel("cxl release irq range 0x%x: offset: 0x%lx limit: %ld\n",
i, irqs->offset[i],
irqs->range[i]);
hwirq = irqs->offset[i] - phb->msi_base;
msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq,
irqs->range[i]);
}
}
EXPORT_SYMBOL(pnv_cxl_release_hwirq_ranges);
int pnv_cxl_alloc_hwirq_ranges(struct cxl_irq_ranges *irqs,
struct pci_dev *dev, int num)
{
struct pci_controller *hose = pci_bus_to_host(dev->bus);
struct pnv_phb *phb = hose->private_data;
int i, hwirq, try;
memset(irqs, 0, sizeof(struct cxl_irq_ranges));
/* 0 is reserved for the multiplexed PSL DSI interrupt */
for (i = 1; i < CXL_IRQ_RANGES && num; i++) {
try = num;
while (try) {
hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, try);
if (hwirq >= 0)
break;
try /= 2;
}
if (!try)
goto fail;
irqs->offset[i] = phb->msi_base + hwirq;
irqs->range[i] = try;
pr_devel("cxl alloc irq range 0x%x: offset: 0x%lx limit: %li\n",
i, irqs->offset[i], irqs->range[i]);
num -= try;
}
if (num)
goto fail;
return 0;
fail:
pnv_cxl_release_hwirq_ranges(irqs, dev);
return -ENOSPC;
}
EXPORT_SYMBOL(pnv_cxl_alloc_hwirq_ranges);
int pnv_cxl_get_irq_count(struct pci_dev *dev)
{
struct pci_controller *hose = pci_bus_to_host(dev->bus);
struct pnv_phb *phb = hose->private_data;
return phb->msi_bmp.irq_count;
}
EXPORT_SYMBOL(pnv_cxl_get_irq_count);
int pnv_cxl_ioda_msi_setup(struct pci_dev *dev, unsigned int hwirq,
unsigned int virq)
{
struct pci_controller *hose = pci_bus_to_host(dev->bus);
struct pnv_phb *phb = hose->private_data;
unsigned int xive_num = hwirq - phb->msi_base;
struct pnv_ioda_pe *pe;
int rc;
if (!(pe = pnv_ioda_get_pe(dev)))
return -ENODEV;
/* Assign XIVE to PE */
rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
if (rc) {
pe_warn(pe, "%s: OPAL error %d setting msi_base 0x%x "
"hwirq 0x%x XIVE 0x%x PE\n",
pci_name(dev), rc, phb->msi_base, hwirq, xive_num);
return -EIO;
}
pnv_set_msi_irq_chip(phb, virq);
return 0;
}
EXPORT_SYMBOL(pnv_cxl_ioda_msi_setup);
#if IS_MODULE(CONFIG_CXL)
static inline int get_cxl_module(void)
{
struct module *cxl_module;
mutex_lock(&module_mutex);
cxl_module = find_module("cxl");
if (cxl_module)
__module_get(cxl_module);
mutex_unlock(&module_mutex);
if (!cxl_module)
return -ENODEV;
return 0;
}
#else
static inline int get_cxl_module(void) { return 0; }
#endif
/*
* Sets flags and switches the controller ops to enable the cxl kernel api.
* Originally the cxl kernel API operated on a virtual PHB, but certain cards
* such as the Mellanox CX4 use a peer model instead and for these cards the
* cxl kernel api will operate on the real PHB.
*/
int pnv_cxl_enable_phb_kernel_api(struct pci_controller *hose, bool enable)
{
struct pnv_phb *phb = hose->private_data;
int rc;
if (!enable) {
/*
* Once cxl mode is enabled on the PHB, there is currently no
* known safe method to disable it again, and trying risks a
* checkstop. If we can find a way to safely disable cxl mode
* in the future we can revisit this, but for now the only sane
* thing to do is to refuse to disable cxl mode:
*/
return -EPERM;
}
/*
* Hold a reference to the cxl module since several PHB operations now
* depend on it, and it would be insane to allow it to be removed so
* long as we are in this mode (and since we can't safely disable this
* mode once enabled...).
*/
rc = get_cxl_module();
if (rc)
return rc;
phb->flags |= PNV_PHB_FLAG_CXL;
hose->controller_ops = pnv_cxl_cx4_ioda_controller_ops;
return 0;
}
EXPORT_SYMBOL_GPL(pnv_cxl_enable_phb_kernel_api);
bool pnv_pci_on_cxl_phb(struct pci_dev *dev)
{
struct pci_controller *hose = pci_bus_to_host(dev->bus);
struct pnv_phb *phb = hose->private_data;
return !!(phb->flags & PNV_PHB_FLAG_CXL);
}
EXPORT_SYMBOL_GPL(pnv_pci_on_cxl_phb);
struct cxl_afu *pnv_cxl_phb_to_afu(struct pci_controller *hose)
{
struct pnv_phb *phb = hose->private_data;
return (struct cxl_afu *)phb->cxl_afu;
}
EXPORT_SYMBOL_GPL(pnv_cxl_phb_to_afu);
void pnv_cxl_phb_set_peer_afu(struct pci_dev *dev, struct cxl_afu *afu)
{
struct pci_controller *hose = pci_bus_to_host(dev->bus);
struct pnv_phb *phb = hose->private_data;
phb->cxl_afu = afu;
}
EXPORT_SYMBOL_GPL(pnv_cxl_phb_set_peer_afu);
/*
* In the peer cxl model, the XSL/PSL is physical function 0, and will be used
* by other functions on the device for memory access and interrupts. When the
* other functions are enabled we explicitly take a reference on the cxl
* function since they will use it, and allocate a default context associated
* with that function just like the vPHB model of the cxl kernel API.
*/
bool pnv_cxl_enable_device_hook(struct pci_dev *dev)
{
struct pci_controller *hose = pci_bus_to_host(dev->bus);
struct pnv_phb *phb = hose->private_data;
struct cxl_afu *afu = phb->cxl_afu;
if (!pnv_pci_enable_device_hook(dev))
return false;
/* No special handling for the cxl function, which is always PF 0 */
if (PCI_FUNC(dev->devfn) == 0)
return true;
if (!afu) {
dev_WARN(&dev->dev, "Attempted to enable function > 0 on CXL PHB without a peer AFU\n");
return false;
}
dev_info(&dev->dev, "Enabling function on CXL enabled PHB with peer AFU\n");
/* Make sure the peer AFU can't go away while this device is active */
cxl_afu_get(afu);
return cxl_pci_associate_default_context(dev, afu);
}
void pnv_cxl_disable_device(struct pci_dev *dev)
{
struct pci_controller *hose = pci_bus_to_host(dev->bus);
struct pnv_phb *phb = hose->private_data;
struct cxl_afu *afu = phb->cxl_afu;
/* No special handling for cxl function: */
if (PCI_FUNC(dev->devfn) == 0)
return;
cxl_pci_disable_device(dev);
cxl_afu_put(afu);
}
/*
* This is a special version of pnv_setup_msi_irqs for cards in cxl mode. This
* function handles setting up the IVTE entries for the XSL to use.
*
* We are currently not filling out the MSIX table, since the only currently
* supported adapter (CX4) uses a custom MSIX table format in cxl mode and it
* is up to their driver to fill that out. In the future we may fill out the
* MSIX table (and change the IVTE entries to be an index to the MSIX table)
* for adapters implementing the Full MSI-X mode described in the CAIA.
*/
int pnv_cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
{
struct pci_controller *hose = pci_bus_to_host(pdev->bus);
struct pnv_phb *phb = hose->private_data;
struct msi_desc *entry;
struct cxl_context *ctx = NULL;
unsigned int virq;
int hwirq;
int afu_irq = 0;
int rc;
if (WARN_ON(!phb) || !phb->msi_bmp.bitmap)
return -ENODEV;
if (pdev->no_64bit_msi && !phb->msi32_support)
return -ENODEV;
rc = cxl_cx4_setup_msi_irqs(pdev, nvec, type);
if (rc)
return rc;
for_each_pci_msi_entry(entry, pdev) {
if (!entry->msi_attrib.is_64 && !phb->msi32_support) {
pr_warn("%s: Supports only 64-bit MSIs\n",
pci_name(pdev));
return -ENXIO;
}
hwirq = cxl_next_msi_hwirq(pdev, &ctx, &afu_irq);
if (WARN_ON(hwirq <= 0))
return (hwirq ? hwirq : -ENOMEM);
virq = irq_create_mapping(NULL, hwirq);
if (!virq) {
pr_warn("%s: Failed to map cxl mode MSI to linux irq\n",
pci_name(pdev));
return -ENOMEM;
}
rc = pnv_cxl_ioda_msi_setup(pdev, hwirq, virq);
if (rc) {
pr_warn("%s: Failed to setup cxl mode MSI\n", pci_name(pdev));
irq_dispose_mapping(virq);
return rc;
}
irq_set_msi_desc(virq, entry);
}
return 0;
}
void pnv_cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev)
{
struct pci_controller *hose = pci_bus_to_host(pdev->bus);
struct pnv_phb *phb = hose->private_data;
struct msi_desc *entry;
irq_hw_number_t hwirq;
if (WARN_ON(!phb))
return;
for_each_pci_msi_entry(entry, pdev) {
if (!entry->irq)
continue;
hwirq = virq_to_hw(entry->irq);
irq_set_msi_desc(entry->irq, NULL);
irq_dispose_mapping(entry->irq);
}
cxl_cx4_teardown_msi_irqs(pdev);
}
|