portdrv_core.c 15.5 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11 12 13
/*
 * File:	portdrv_core.c
 * Purpose:	PCI Express Port Bus Driver's Core Functions
 *
 * Copyright (C) 2004 Intel
 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
 */

#include <linux/module.h>
#include <linux/pci.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/pm.h>
14
#include <linux/pm_runtime.h>
Tim Schmielau's avatar
Tim Schmielau committed
15 16
#include <linux/string.h>
#include <linux/slab.h>
Linus Torvalds's avatar
Linus Torvalds committed
17
#include <linux/pcieport_if.h>
18
#include <linux/aer.h>
Linus Torvalds's avatar
Linus Torvalds committed
19

20
#include "../pci.h"
Linus Torvalds's avatar
Linus Torvalds committed
21 22
#include "portdrv.h"

23 24 25 26 27 28 29 30 31 32 33
bool pciehp_msi_disabled;

static int __init pciehp_setup(char *str)
{
	if (!strncmp(str, "nomsi", 5))
		pciehp_msi_disabled = true;

	return 1;
}
__setup("pcie_hp=", pciehp_setup);

34 35 36 37 38 39
/**
 * release_pcie_device - free PCI Express port service device structure
 * @dev: Port service device to release
 *
 * Invoked automatically when device is being removed in response to
 * device_unregister(dev).  Release all resources being claimed.
Linus Torvalds's avatar
Linus Torvalds committed
40 41 42
 */
static void release_pcie_device(struct device *dev)
{
43
	kfree(to_pcie_device(dev));
Linus Torvalds's avatar
Linus Torvalds committed
44 45
}

46 47 48 49
/**
 * pcie_port_msix_add_entry - add entry to given array of MSI-X entries
 * @entries: Array of MSI-X entries
 * @new_entry: Index of the entry to add to the array
50
 * @nr_entries: Number of entries already in the array
51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82
 *
 * Return value: Position of the added entry in the array
 */
static int pcie_port_msix_add_entry(
	struct msix_entry *entries, int new_entry, int nr_entries)
{
	int j;

	for (j = 0; j < nr_entries; j++)
		if (entries[j].entry == new_entry)
			return j;

	entries[j].entry = new_entry;
	return j;
}

/**
 * pcie_port_enable_msix - try to set up MSI-X as interrupt mode for given port
 * @dev: PCI Express port to handle
 * @vectors: Array of interrupt vectors to populate
 * @mask: Bitmask of port capabilities returned by get_port_device_capability()
 *
 * Return value: 0 on success, error code on failure
 */
static int pcie_port_enable_msix(struct pci_dev *dev, int *vectors, int mask)
{
	struct msix_entry *msix_entries;
	int idx[PCIE_PORT_DEVICE_MAXSERVICES];
	int nr_entries, status, pos, i, nvec;
	u16 reg16;
	u32 reg32;

83 84 85 86
	nr_entries = pci_msix_vec_count(dev);
	if (nr_entries < 0)
		return nr_entries;
	BUG_ON(!nr_entries);
87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102
	if (nr_entries > PCIE_PORT_MAX_MSIX_ENTRIES)
		nr_entries = PCIE_PORT_MAX_MSIX_ENTRIES;

	msix_entries = kzalloc(sizeof(*msix_entries) * nr_entries, GFP_KERNEL);
	if (!msix_entries)
		return -ENOMEM;

	/*
	 * Allocate as many entries as the port wants, so that we can check
	 * which of them will be useful.  Moreover, if nr_entries is correctly
	 * equal to the number of entries this port actually uses, we'll happily
	 * go through without any tricks.
	 */
	for (i = 0; i < nr_entries; i++)
		msix_entries[i].entry = i;

103
	status = pci_enable_msix_exact(dev, msix_entries, nr_entries);
104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124
	if (status)
		goto Exit;

	for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++)
		idx[i] = -1;
	status = -EIO;
	nvec = 0;

	if (mask & (PCIE_PORT_SERVICE_PME | PCIE_PORT_SERVICE_HP)) {
		int entry;

		/*
		 * The code below follows the PCI Express Base Specification 2.0
		 * stating in Section 6.1.6 that "PME and Hot-Plug Event
		 * interrupts (when both are implemented) always share the same
		 * MSI or MSI-X vector, as indicated by the Interrupt Message
		 * Number field in the PCI Express Capabilities register", where
		 * according to Section 7.8.2 of the specification "For MSI-X,
		 * the value in this field indicates which MSI-X Table entry is
		 * used to generate the interrupt message."
		 */
125
		pcie_capability_read_word(dev, PCI_EXP_FLAGS, &reg16);
126
		entry = (reg16 & PCI_EXP_FLAGS_IRQ) >> 9;
127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174
		if (entry >= nr_entries)
			goto Error;

		i = pcie_port_msix_add_entry(msix_entries, entry, nvec);
		if (i == nvec)
			nvec++;

		idx[PCIE_PORT_SERVICE_PME_SHIFT] = i;
		idx[PCIE_PORT_SERVICE_HP_SHIFT] = i;
	}

	if (mask & PCIE_PORT_SERVICE_AER) {
		int entry;

		/*
		 * The code below follows Section 7.10.10 of the PCI Express
		 * Base Specification 2.0 stating that bits 31-27 of the Root
		 * Error Status Register contain a value indicating which of the
		 * MSI/MSI-X vectors assigned to the port is going to be used
		 * for AER, where "For MSI-X, the value in this register
		 * indicates which MSI-X Table entry is used to generate the
		 * interrupt message."
		 */
		pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
		pci_read_config_dword(dev, pos + PCI_ERR_ROOT_STATUS, &reg32);
		entry = reg32 >> 27;
		if (entry >= nr_entries)
			goto Error;

		i = pcie_port_msix_add_entry(msix_entries, entry, nvec);
		if (i == nvec)
			nvec++;

		idx[PCIE_PORT_SERVICE_AER_SHIFT] = i;
	}

	/*
	 * If nvec is equal to the allocated number of entries, we can just use
	 * what we have.  Otherwise, the port has some extra entries not for the
	 * services we know and we need to work around that.
	 */
	if (nvec == nr_entries) {
		status = 0;
	} else {
		/* Drop the temporary MSI-X setup */
		pci_disable_msix(dev);

		/* Now allocate the MSI-X vectors for real */
175
		status = pci_enable_msix_exact(dev, msix_entries, nvec);
176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191
		if (status)
			goto Exit;
	}

	for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++)
		vectors[i] = idx[i] >= 0 ? msix_entries[idx[i]].vector : -1;

 Exit:
	kfree(msix_entries);
	return status;

 Error:
	pci_disable_msix(dev);
	goto Exit;
}

192
/**
193
 * init_service_irqs - initialize irqs for PCI Express port services
194
 * @dev: PCI Express port to handle
195
 * @irqs: Array of irqs to populate
196 197 198 199
 * @mask: Bitmask of port capabilities returned by get_port_device_capability()
 *
 * Return value: Interrupt mode associated with the port
 */
200
static int init_service_irqs(struct pci_dev *dev, int *irqs, int mask)
Linus Torvalds's avatar
Linus Torvalds committed
201
{
202 203
	int i, irq = -1;

204 205 206 207
	/*
	 * If MSI cannot be used for PCIe PME or hotplug, we have to use
	 * INTx or other interrupts, e.g. system shared interrupt.
	 */
208 209
	if (((mask & PCIE_PORT_SERVICE_PME) && pcie_pme_no_msi()) ||
	    ((mask & PCIE_PORT_SERVICE_HP) && pciehp_no_msi())) {
210
		if (dev->irq)
211 212 213
			irq = dev->irq;
		goto no_msi;
	}
214

215
	/* Try to use MSI-X if supported */
216 217
	if (!pcie_port_enable_msix(dev, irqs, mask))
		return 0;
218

219 220 221 222 223 224
	/*
	 * We're not going to use MSI-X, so try MSI and fall back to INTx.
	 * If neither MSI/MSI-X nor INTx available, try other interrupt.  On
	 * some platforms, root port doesn't support MSI/MSI-X/INTx in RC mode.
	 */
	if (!pci_enable_msi(dev) || dev->irq)
225
		irq = dev->irq;
226

227
 no_msi:
228
	for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++)
229 230
		irqs[i] = irq;
	irqs[PCIE_PORT_SERVICE_VC_SHIFT] = -1;
Linus Torvalds's avatar
Linus Torvalds committed
231

232 233 234
	if (irq < 0)
		return -ENODEV;
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
235 236
}

237 238 239 240 241 242 243 244
static void cleanup_service_irqs(struct pci_dev *dev)
{
	if (dev->msix_enabled)
		pci_disable_msix(dev);
	else if (dev->msi_enabled)
		pci_disable_msi(dev);
}

245 246 247 248 249 250 251 252 253 254
/**
 * get_port_device_capability - discover capabilities of a PCI Express port
 * @dev: PCI Express port to examine
 *
 * The capabilities are read from the port's PCI Express configuration registers
 * as described in PCI Express Base Specification 1.0a sections 7.8.2, 7.8.9 and
 * 7.9 - 7.11.
 *
 * Return value: Bitmask of discovered port capabilities
 */
Linus Torvalds's avatar
Linus Torvalds committed
255 256
static int get_port_device_capability(struct pci_dev *dev)
{
257
	int services = 0;
258
	int cap_mask = 0;
259

260 261 262
	if (pcie_ports_disabled)
		return 0;

263
	cap_mask = PCIE_PORT_SERVICE_PME | PCIE_PORT_SERVICE_HP
264
			| PCIE_PORT_SERVICE_VC | PCIE_PORT_SERVICE_DPC;
265 266 267
	if (pci_aer_available())
		cap_mask |= PCIE_PORT_SERVICE_AER;

268 269
	if (pcie_ports_auto)
		pcie_port_platform_notify(dev, &cap_mask);
Linus Torvalds's avatar
Linus Torvalds committed
270 271

	/* Hot-Plug Capable */
272 273 274 275 276 277 278 279
	if ((cap_mask & PCIE_PORT_SERVICE_HP) && dev->is_hotplug_bridge) {
		services |= PCIE_PORT_SERVICE_HP;
		/*
		 * Disable hot-plug interrupts in case they have been enabled
		 * by the BIOS and the hot-plug service driver is not loaded.
		 */
		pcie_capability_clear_word(dev, PCI_EXP_SLTCTL,
			  PCI_EXP_SLTCTL_CCIE | PCI_EXP_SLTCTL_HPIE);
280 281
	}
	/* AER capable */
282
	if ((cap_mask & PCIE_PORT_SERVICE_AER)
283
	    && pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR)) {
284
		services |= PCIE_PORT_SERVICE_AER;
285 286 287 288 289 290
		/*
		 * Disable AER on this port in case it's been enabled by the
		 * BIOS (the AER service driver will enable it when necessary).
		 */
		pci_disable_pcie_error_reporting(dev);
	}
291
	/* VC support */
292 293
	if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_VC))
		services |= PCIE_PORT_SERVICE_VC;
294
	/* Root ports are capable of generating PME too */
295
	if ((cap_mask & PCIE_PORT_SERVICE_PME)
296
	    && pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) {
297
		services |= PCIE_PORT_SERVICE_PME;
298 299 300 301 302 303 304
		/*
		 * Disable PME interrupt on this port in case it's been enabled
		 * by the BIOS (the PME service driver will enable it when
		 * necessary).
		 */
		pcie_pme_interrupt_enable(dev, false);
	}
305 306
	if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DPC))
		services |= PCIE_PORT_SERVICE_DPC;
Linus Torvalds's avatar
Linus Torvalds committed
307 308 309 310

	return services;
}

311
/**
312 313 314
 * pcie_device_init - allocate and initialize PCI Express port service device
 * @pdev: PCI Express port to associate the service device with
 * @service: Type of service to associate with the service device
315 316
 * @irq: Interrupt vector to associate with the service device
 */
317
static int pcie_device_init(struct pci_dev *pdev, int service, int irq)
Linus Torvalds's avatar
Linus Torvalds committed
318
{
319 320
	int retval;
	struct pcie_device *pcie;
Linus Torvalds's avatar
Linus Torvalds committed
321 322
	struct device *device;

323 324 325 326 327 328
	pcie = kzalloc(sizeof(*pcie), GFP_KERNEL);
	if (!pcie)
		return -ENOMEM;
	pcie->port = pdev;
	pcie->irq = irq;
	pcie->service = service;
Linus Torvalds's avatar
Linus Torvalds committed
329 330

	/* Initialize generic device interface */
331
	device = &pcie->device;
Linus Torvalds's avatar
Linus Torvalds committed
332 333
	device->bus = &pcie_port_bus_type;
	device->release = release_pcie_device;	/* callback to free pcie dev */
334
	dev_set_name(device, "%s:pcie%03x",
335
		     pci_name(pdev),
336
		     get_descriptor_id(pci_pcie_type(pdev), service));
337
	device->parent = &pdev->dev;
338
	device_enable_async_suspend(device);
339 340

	retval = device_register(device);
341
	if (retval) {
342
		put_device(device);
343 344 345
		return retval;
	}

346 347
	pm_runtime_no_callbacks(device);

348
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
349 350
}

351 352 353 354 355 356 357
/**
 * pcie_port_device_register - register PCI Express port
 * @dev: PCI Express port to register
 *
 * Allocate the port extension structure and register services associated with
 * the port.
 */
Linus Torvalds's avatar
Linus Torvalds committed
358 359
int pcie_port_device_register(struct pci_dev *dev)
{
360
	int status, capabilities, i, nr_service;
361
	int irqs[PCIE_PORT_DEVICE_MAXSERVICES];
Linus Torvalds's avatar
Linus Torvalds committed
362

363 364 365
	/* Enable PCI Express port device */
	status = pci_enable_device(dev);
	if (status)
366
		return status;
367 368 369

	/* Get and check PCI Express port services */
	capabilities = get_port_device_capability(dev);
370
	if (!capabilities)
371 372
		return 0;

373
	pci_set_master(dev);
374 375 376
	/*
	 * Initialize service irqs. Don't use service devices that
	 * require interrupts if there is no way to generate them.
377 378 379
	 * However, some drivers may have a polling mode (e.g. pciehp_poll_mode)
	 * that can be used in the absence of irqs.  Allow them to determine
	 * if that is to be used.
380 381 382
	 */
	status = init_service_irqs(dev, irqs, capabilities);
	if (status) {
383
		capabilities &= PCIE_PORT_SERVICE_VC | PCIE_PORT_SERVICE_HP;
384
		if (!capabilities)
385
			goto error_disable;
386
	}
Linus Torvalds's avatar
Linus Torvalds committed
387 388

	/* Allocate child services if any */
389 390 391
	status = -ENODEV;
	nr_service = 0;
	for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++) {
392 393 394
		int service = 1 << i;
		if (!(capabilities & service))
			continue;
395 396
		if (!pcie_device_init(dev, service, irqs[i]))
			nr_service++;
397
	}
398
	if (!nr_service)
399
		goto error_cleanup_irqs;
400

Linus Torvalds's avatar
Linus Torvalds committed
401
	return 0;
402

403 404
error_cleanup_irqs:
	cleanup_service_irqs(dev);
405 406
error_disable:
	pci_disable_device(dev);
407
	return status;
Linus Torvalds's avatar
Linus Torvalds committed
408 409 410
}

#ifdef CONFIG_PM
411
static int suspend_iter(struct device *dev, void *data)
Linus Torvalds's avatar
Linus Torvalds committed
412 413
{
	struct pcie_port_service_driver *service_driver;
414

415 416 417 418 419
	if ((dev->bus == &pcie_port_bus_type) && dev->driver) {
		service_driver = to_service_driver(dev->driver);
		if (service_driver->suspend)
			service_driver->suspend(to_pcie_device(dev));
	}
420 421
	return 0;
}
Linus Torvalds's avatar
Linus Torvalds committed
422

423 424 425 426
/**
 * pcie_port_device_suspend - suspend port services associated with a PCIe port
 * @dev: PCI Express port to handle
 */
427
int pcie_port_device_suspend(struct device *dev)
428
{
429
	return device_for_each_child(dev, NULL, suspend_iter);
Linus Torvalds's avatar
Linus Torvalds committed
430 431
}

432 433
static int resume_iter(struct device *dev, void *data)
{
Linus Torvalds's avatar
Linus Torvalds committed
434 435
	struct pcie_port_service_driver *service_driver;

436 437 438 439 440
	if ((dev->bus == &pcie_port_bus_type) &&
	    (dev->driver)) {
		service_driver = to_service_driver(dev->driver);
		if (service_driver->resume)
			service_driver->resume(to_pcie_device(dev));
Linus Torvalds's avatar
Linus Torvalds committed
441
	}
442 443
	return 0;
}
Linus Torvalds's avatar
Linus Torvalds committed
444

445
/**
446
 * pcie_port_device_resume - resume port services associated with a PCIe port
447 448
 * @dev: PCI Express port to handle
 */
449
int pcie_port_device_resume(struct device *dev)
450
{
451
	return device_for_each_child(dev, NULL, resume_iter);
Linus Torvalds's avatar
Linus Torvalds committed
452
}
453
#endif /* PM */
Linus Torvalds's avatar
Linus Torvalds committed
454

455
static int remove_iter(struct device *dev, void *data)
Linus Torvalds's avatar
Linus Torvalds committed
456
{
457
	if (dev->bus == &pcie_port_bus_type)
458
		device_unregister(dev);
459 460 461
	return 0;
}

462 463 464 465 466 467 468
/**
 * pcie_port_device_remove - unregister PCI Express port service devices
 * @dev: PCI Express port the service devices to unregister are associated with
 *
 * Remove PCI Express port service devices associated with given port and
 * disable MSI-X or MSI for the port.
 */
469 470
void pcie_port_device_remove(struct pci_dev *dev)
{
471
	device_for_each_child(&dev->dev, NULL, remove_iter);
472
	cleanup_service_irqs(dev);
473
	pci_disable_device(dev);
Linus Torvalds's avatar
Linus Torvalds committed
474 475
}

476 477 478 479 480 481 482 483
/**
 * pcie_port_probe_service - probe driver for given PCI Express port service
 * @dev: PCI Express port service device to probe against
 *
 * If PCI Express port service driver is registered with
 * pcie_port_service_register(), this function will be called by the driver core
 * whenever match is found between the driver and a port service device.
 */
484
static int pcie_port_probe_service(struct device *dev)
Linus Torvalds's avatar
Linus Torvalds committed
485
{
486 487 488 489 490 491 492 493 494 495 496 497
	struct pcie_device *pciedev;
	struct pcie_port_service_driver *driver;
	int status;

	if (!dev || !dev->driver)
		return -ENODEV;

	driver = to_service_driver(dev->driver);
	if (!driver || !driver->probe)
		return -ENODEV;

	pciedev = to_pcie_device(dev);
498
	status = driver->probe(pciedev);
499 500 501 502 503 504
	if (status)
		return status;

	dev_printk(KERN_DEBUG, dev, "service driver %s loaded\n", driver->name);
	get_device(dev);
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
505 506
}

507 508 509 510 511 512 513 514 515
/**
 * pcie_port_remove_service - detach driver from given PCI Express port service
 * @dev: PCI Express port service device to handle
 *
 * If PCI Express port service driver is registered with
 * pcie_port_service_register(), this function will be called by the driver core
 * when device_unregister() is called for the port service device associated
 * with the driver.
 */
516
static int pcie_port_remove_service(struct device *dev)
Linus Torvalds's avatar
Linus Torvalds committed
517
{
518 519 520 521 522 523 524 525 526 527 528 529 530 531 532
	struct pcie_device *pciedev;
	struct pcie_port_service_driver *driver;

	if (!dev || !dev->driver)
		return 0;

	pciedev = to_pcie_device(dev);
	driver = to_service_driver(dev->driver);
	if (driver && driver->remove) {
		dev_printk(KERN_DEBUG, dev, "unloading service driver %s\n",
			driver->name);
		driver->remove(pciedev);
		put_device(dev);
	}
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
533 534
}

535 536 537 538 539 540 541 542 543
/**
 * pcie_port_shutdown_service - shut down given PCI Express port service
 * @dev: PCI Express port service device to handle
 *
 * If PCI Express port service driver is registered with
 * pcie_port_service_register(), this function will be called by the driver core
 * when device_shutdown() is called for the port service device associated
 * with the driver.
 */
544 545
static void pcie_port_shutdown_service(struct device *dev) {}

546 547 548 549
/**
 * pcie_port_service_register - register PCI Express port service driver
 * @new: PCI Express port service driver to register
 */
Linus Torvalds's avatar
Linus Torvalds committed
550 551
int pcie_port_service_register(struct pcie_port_service_driver *new)
{
552 553 554
	if (pcie_ports_disabled)
		return -ENODEV;

555
	new->driver.name = new->name;
Linus Torvalds's avatar
Linus Torvalds committed
556 557 558 559 560 561
	new->driver.bus = &pcie_port_bus_type;
	new->driver.probe = pcie_port_probe_service;
	new->driver.remove = pcie_port_remove_service;
	new->driver.shutdown = pcie_port_shutdown_service;

	return driver_register(&new->driver);
562
}
563
EXPORT_SYMBOL(pcie_port_service_register);
Linus Torvalds's avatar
Linus Torvalds committed
564

565 566 567 568
/**
 * pcie_port_service_unregister - unregister PCI Express port service driver
 * @drv: PCI Express port service driver to unregister
 */
569
void pcie_port_service_unregister(struct pcie_port_service_driver *drv)
Linus Torvalds's avatar
Linus Torvalds committed
570
{
571
	driver_unregister(&drv->driver);
Linus Torvalds's avatar
Linus Torvalds committed
572 573
}
EXPORT_SYMBOL(pcie_port_service_unregister);