aer_inject.c 12.5 KB
Newer Older
1
/*
2
 * PCIe AER software error injection support.
3
 *
4
 * Debuging PCIe AER code is quite difficult because it is hard to
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 * trigger various real hardware errors. Software based error
 * injection can fake almost all kinds of errors with the help of a
 * user space helper tool aer-inject, which can be gotten from:
 *   http://www.kernel.org/pub/linux/utils/pci/aer-inject/
 *
 * Copyright 2009 Intel Corporation.
 *     Huang Ying <ying.huang@intel.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; version 2
 * of the License.
 *
 */

#include <linux/module.h>
#include <linux/init.h>
#include <linux/miscdevice.h>
#include <linux/pci.h>
24
#include <linux/slab.h>
25
#include <linux/fs.h>
26
#include <linux/uaccess.h>
27
#include <linux/stddef.h>
28 29
#include "aerdrv.h"

30
/* Override the existing corrected and uncorrected error masks */
31
static bool aer_mask_override;
32 33
module_param(aer_mask_override, bool, 0);

34
struct aer_error_inj {
35 36 37 38 39 40 41 42 43
	u8 bus;
	u8 dev;
	u8 fn;
	u32 uncor_status;
	u32 cor_status;
	u32 header_log0;
	u32 header_log1;
	u32 header_log2;
	u32 header_log3;
44
	u16 domain;
45 46
};

47
struct aer_error {
48
	struct list_head list;
49
	u16 domain;
50 51 52 53 54 55 56 57 58 59 60 61 62 63
	unsigned int bus;
	unsigned int devfn;
	int pos_cap_err;

	u32 uncor_status;
	u32 cor_status;
	u32 header_log0;
	u32 header_log1;
	u32 header_log2;
	u32 header_log3;
	u32 root_status;
	u32 source_id;
};

64
struct pci_bus_ops {
65 66 67 68 69 70 71 72 73 74 75 76
	struct list_head list;
	struct pci_bus *bus;
	struct pci_ops *ops;
};

static LIST_HEAD(einjected);

static LIST_HEAD(pci_bus_ops_list);

/* Protect einjected and pci_bus_ops_list */
static DEFINE_SPINLOCK(inject_lock);

77 78 79
static void aer_error_init(struct aer_error *err, u16 domain,
			   unsigned int bus, unsigned int devfn,
			   int pos_cap_err)
80 81
{
	INIT_LIST_HEAD(&err->list);
82
	err->domain = domain;
83 84 85 86 87 88
	err->bus = bus;
	err->devfn = devfn;
	err->pos_cap_err = pos_cap_err;
}

/* inject_lock must be held before calling */
89 90
static struct aer_error *__find_aer_error(u16 domain, unsigned int bus,
					  unsigned int devfn)
91 92 93 94
{
	struct aer_error *err;

	list_for_each_entry(err, &einjected, list) {
95 96 97
		if (domain == err->domain &&
		    bus == err->bus &&
		    devfn == err->devfn)
98 99 100 101 102 103 104 105
			return err;
	}
	return NULL;
}

/* inject_lock must be held before calling */
static struct aer_error *__find_aer_error_by_dev(struct pci_dev *dev)
{
106 107 108 109
	int domain = pci_domain_nr(dev->bus);
	if (domain < 0)
		return NULL;
	return __find_aer_error((u16)domain, dev->bus->number, dev->devfn);
110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165
}

/* inject_lock must be held before calling */
static struct pci_ops *__find_pci_bus_ops(struct pci_bus *bus)
{
	struct pci_bus_ops *bus_ops;

	list_for_each_entry(bus_ops, &pci_bus_ops_list, list) {
		if (bus_ops->bus == bus)
			return bus_ops->ops;
	}
	return NULL;
}

static struct pci_bus_ops *pci_bus_ops_pop(void)
{
	unsigned long flags;
	struct pci_bus_ops *bus_ops = NULL;

	spin_lock_irqsave(&inject_lock, flags);
	if (list_empty(&pci_bus_ops_list))
		bus_ops = NULL;
	else {
		struct list_head *lh = pci_bus_ops_list.next;
		list_del(lh);
		bus_ops = list_entry(lh, struct pci_bus_ops, list);
	}
	spin_unlock_irqrestore(&inject_lock, flags);
	return bus_ops;
}

static u32 *find_pci_config_dword(struct aer_error *err, int where,
				  int *prw1cs)
{
	int rw1cs = 0;
	u32 *target = NULL;

	if (err->pos_cap_err == -1)
		return NULL;

	switch (where - err->pos_cap_err) {
	case PCI_ERR_UNCOR_STATUS:
		target = &err->uncor_status;
		rw1cs = 1;
		break;
	case PCI_ERR_COR_STATUS:
		target = &err->cor_status;
		rw1cs = 1;
		break;
	case PCI_ERR_HEADER_LOG:
		target = &err->header_log0;
		break;
	case PCI_ERR_HEADER_LOG+4:
		target = &err->header_log1;
		break;
	case PCI_ERR_HEADER_LOG+8:
166
		target = &err->header_log2;
167 168 169 170 171 172 173 174
		break;
	case PCI_ERR_HEADER_LOG+12:
		target = &err->header_log3;
		break;
	case PCI_ERR_ROOT_STATUS:
		target = &err->root_status;
		rw1cs = 1;
		break;
175
	case PCI_ERR_ROOT_ERR_SRC:
176 177 178 179 180 181 182 183 184 185 186 187 188 189 190
		target = &err->source_id;
		break;
	}
	if (prw1cs)
		*prw1cs = rw1cs;
	return target;
}

static int pci_read_aer(struct pci_bus *bus, unsigned int devfn, int where,
			int size, u32 *val)
{
	u32 *sim;
	struct aer_error *err;
	unsigned long flags;
	struct pci_ops *ops;
191
	int domain;
192 193 194 195

	spin_lock_irqsave(&inject_lock, flags);
	if (size != sizeof(u32))
		goto out;
196 197 198 199
	domain = pci_domain_nr(bus);
	if (domain < 0)
		goto out;
	err = __find_aer_error((u16)domain, bus->number, devfn);
200 201 202 203 204 205 206 207 208 209 210 211 212 213 214
	if (!err)
		goto out;

	sim = find_pci_config_dword(err, where, NULL);
	if (sim) {
		*val = *sim;
		spin_unlock_irqrestore(&inject_lock, flags);
		return 0;
	}
out:
	ops = __find_pci_bus_ops(bus);
	spin_unlock_irqrestore(&inject_lock, flags);
	return ops->read(bus, devfn, where, size, val);
}

215 216
static int pci_write_aer(struct pci_bus *bus, unsigned int devfn, int where,
			 int size, u32 val)
217 218 219 220 221 222
{
	u32 *sim;
	struct aer_error *err;
	unsigned long flags;
	int rw1cs;
	struct pci_ops *ops;
223
	int domain;
224 225 226 227

	spin_lock_irqsave(&inject_lock, flags);
	if (size != sizeof(u32))
		goto out;
228 229 230 231
	domain = pci_domain_nr(bus);
	if (domain < 0)
		goto out;
	err = __find_aer_error((u16)domain, bus->number, devfn);
232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281
	if (!err)
		goto out;

	sim = find_pci_config_dword(err, where, &rw1cs);
	if (sim) {
		if (rw1cs)
			*sim ^= val;
		else
			*sim = val;
		spin_unlock_irqrestore(&inject_lock, flags);
		return 0;
	}
out:
	ops = __find_pci_bus_ops(bus);
	spin_unlock_irqrestore(&inject_lock, flags);
	return ops->write(bus, devfn, where, size, val);
}

static struct pci_ops pci_ops_aer = {
	.read = pci_read_aer,
	.write = pci_write_aer,
};

static void pci_bus_ops_init(struct pci_bus_ops *bus_ops,
			     struct pci_bus *bus,
			     struct pci_ops *ops)
{
	INIT_LIST_HEAD(&bus_ops->list);
	bus_ops->bus = bus;
	bus_ops->ops = ops;
}

static int pci_bus_set_aer_ops(struct pci_bus *bus)
{
	struct pci_ops *ops;
	struct pci_bus_ops *bus_ops;
	unsigned long flags;

	bus_ops = kmalloc(sizeof(*bus_ops), GFP_KERNEL);
	if (!bus_ops)
		return -ENOMEM;
	ops = pci_bus_set_ops(bus, &pci_ops_aer);
	spin_lock_irqsave(&inject_lock, flags);
	if (ops == &pci_ops_aer)
		goto out;
	pci_bus_ops_init(bus_ops, bus, ops);
	list_add(&bus_ops->list, &pci_bus_ops_list);
	bus_ops = NULL;
out:
	spin_unlock_irqrestore(&inject_lock, flags);
282
	kfree(bus_ops);
283 284 285 286 287 288
	return 0;
}

static struct pci_dev *pcie_find_root_port(struct pci_dev *dev)
{
	while (1) {
289
		if (!pci_is_pcie(dev))
290
			break;
291
		if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT)
292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328
			return dev;
		if (!dev->bus->self)
			break;
		dev = dev->bus->self;
	}
	return NULL;
}

static int find_aer_device_iter(struct device *device, void *data)
{
	struct pcie_device **result = data;
	struct pcie_device *pcie_dev;

	if (device->bus == &pcie_port_bus_type) {
		pcie_dev = to_pcie_device(device);
		if (pcie_dev->service & PCIE_PORT_SERVICE_AER) {
			*result = pcie_dev;
			return 1;
		}
	}
	return 0;
}

static int find_aer_device(struct pci_dev *dev, struct pcie_device **result)
{
	return device_for_each_child(&dev->dev, result, find_aer_device_iter);
}

static int aer_inject(struct aer_error_inj *einj)
{
	struct aer_error *err, *rperr;
	struct aer_error *err_alloc = NULL, *rperr_alloc = NULL;
	struct pci_dev *dev, *rpdev;
	struct pcie_device *edev;
	unsigned long flags;
	unsigned int devfn = PCI_DEVFN(einj->dev, einj->fn);
	int pos_cap_err, rp_pos_cap_err;
329
	u32 sever, cor_mask, uncor_mask, cor_mask_orig = 0, uncor_mask_orig = 0;
330 331
	int ret = 0;

332
	dev = pci_get_domain_bus_and_slot((int)einj->domain, einj->bus, devfn);
333
	if (!dev)
334
		return -ENODEV;
335 336
	rpdev = pcie_find_root_port(dev);
	if (!rpdev) {
337
		ret = -ENODEV;
338 339 340 341 342
		goto out_put;
	}

	pos_cap_err = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
	if (!pos_cap_err) {
343
		ret = -EPERM;
344 345 346
		goto out_put;
	}
	pci_read_config_dword(dev, pos_cap_err + PCI_ERR_UNCOR_SEVER, &sever);
347 348 349
	pci_read_config_dword(dev, pos_cap_err + PCI_ERR_COR_MASK, &cor_mask);
	pci_read_config_dword(dev, pos_cap_err + PCI_ERR_UNCOR_MASK,
			      &uncor_mask);
350 351 352

	rp_pos_cap_err = pci_find_ext_capability(rpdev, PCI_EXT_CAP_ID_ERR);
	if (!rp_pos_cap_err) {
353
		ret = -EPERM;
354 355 356 357 358 359 360 361 362 363 364 365 366 367
		goto out_put;
	}

	err_alloc =  kzalloc(sizeof(struct aer_error), GFP_KERNEL);
	if (!err_alloc) {
		ret = -ENOMEM;
		goto out_put;
	}
	rperr_alloc =  kzalloc(sizeof(struct aer_error), GFP_KERNEL);
	if (!rperr_alloc) {
		ret = -ENOMEM;
		goto out_put;
	}

368 369 370 371 372 373 374 375 376 377 378 379
	if (aer_mask_override) {
		cor_mask_orig = cor_mask;
		cor_mask &= !(einj->cor_status);
		pci_write_config_dword(dev, pos_cap_err + PCI_ERR_COR_MASK,
				       cor_mask);

		uncor_mask_orig = uncor_mask;
		uncor_mask &= !(einj->uncor_status);
		pci_write_config_dword(dev, pos_cap_err + PCI_ERR_UNCOR_MASK,
				       uncor_mask);
	}

380 381 382 383 384 385
	spin_lock_irqsave(&inject_lock, flags);

	err = __find_aer_error_by_dev(dev);
	if (!err) {
		err = err_alloc;
		err_alloc = NULL;
386 387
		aer_error_init(err, einj->domain, einj->bus, devfn,
			       pos_cap_err);
388 389 390 391 392 393 394 395 396
		list_add(&err->list, &einjected);
	}
	err->uncor_status |= einj->uncor_status;
	err->cor_status |= einj->cor_status;
	err->header_log0 = einj->header_log0;
	err->header_log1 = einj->header_log1;
	err->header_log2 = einj->header_log2;
	err->header_log3 = einj->header_log3;

397 398
	if (!aer_mask_override && einj->cor_status &&
	    !(einj->cor_status & ~cor_mask)) {
399
		ret = -EINVAL;
400
		printk(KERN_WARNING "The correctable error(s) is masked by device\n");
401 402 403
		spin_unlock_irqrestore(&inject_lock, flags);
		goto out_put;
	}
404 405
	if (!aer_mask_override && einj->uncor_status &&
	    !(einj->uncor_status & ~uncor_mask)) {
406
		ret = -EINVAL;
407
		printk(KERN_WARNING "The uncorrectable error(s) is masked by device\n");
408 409 410 411
		spin_unlock_irqrestore(&inject_lock, flags);
		goto out_put;
	}

412 413 414 415
	rperr = __find_aer_error_by_dev(rpdev);
	if (!rperr) {
		rperr = rperr_alloc;
		rperr_alloc = NULL;
416 417
		aer_error_init(rperr, pci_domain_nr(rpdev->bus),
			       rpdev->bus->number, rpdev->devfn,
418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443
			       rp_pos_cap_err);
		list_add(&rperr->list, &einjected);
	}
	if (einj->cor_status) {
		if (rperr->root_status & PCI_ERR_ROOT_COR_RCV)
			rperr->root_status |= PCI_ERR_ROOT_MULTI_COR_RCV;
		else
			rperr->root_status |= PCI_ERR_ROOT_COR_RCV;
		rperr->source_id &= 0xffff0000;
		rperr->source_id |= (einj->bus << 8) | devfn;
	}
	if (einj->uncor_status) {
		if (rperr->root_status & PCI_ERR_ROOT_UNCOR_RCV)
			rperr->root_status |= PCI_ERR_ROOT_MULTI_UNCOR_RCV;
		if (sever & einj->uncor_status) {
			rperr->root_status |= PCI_ERR_ROOT_FATAL_RCV;
			if (!(rperr->root_status & PCI_ERR_ROOT_UNCOR_RCV))
				rperr->root_status |= PCI_ERR_ROOT_FIRST_FATAL;
		} else
			rperr->root_status |= PCI_ERR_ROOT_NONFATAL_RCV;
		rperr->root_status |= PCI_ERR_ROOT_UNCOR_RCV;
		rperr->source_id &= 0x0000ffff;
		rperr->source_id |= ((einj->bus << 8) | devfn) << 16;
	}
	spin_unlock_irqrestore(&inject_lock, flags);

444 445 446 447 448 449 450
	if (aer_mask_override) {
		pci_write_config_dword(dev, pos_cap_err + PCI_ERR_COR_MASK,
				       cor_mask_orig);
		pci_write_config_dword(dev, pos_cap_err + PCI_ERR_UNCOR_MASK,
				       uncor_mask_orig);
	}

451 452 453 454 455 456 457
	ret = pci_bus_set_aer_ops(dev->bus);
	if (ret)
		goto out_put;
	ret = pci_bus_set_aer_ops(rpdev->bus);
	if (ret)
		goto out_put;

458 459 460 461 462 463
	if (find_aer_device(rpdev, &edev)) {
		if (!get_service_data(edev)) {
			printk(KERN_WARNING "AER service is not initialized\n");
			ret = -EINVAL;
			goto out_put;
		}
464
		aer_irq(-1, edev);
Ryan Desfosses's avatar
Ryan Desfosses committed
465
	} else
466 467
		ret = -EINVAL;
out_put:
468 469
	kfree(err_alloc);
	kfree(rperr_alloc);
470 471 472 473 474 475 476 477 478 479 480 481
	pci_dev_put(dev);
	return ret;
}

static ssize_t aer_inject_write(struct file *filp, const char __user *ubuf,
				size_t usize, loff_t *off)
{
	struct aer_error_inj einj;
	int ret;

	if (!capable(CAP_SYS_ADMIN))
		return -EPERM;
482 483
	if (usize < offsetof(struct aer_error_inj, domain) ||
	    usize > sizeof(einj))
484 485
		return -EINVAL;

486
	memset(&einj, 0, sizeof(einj));
487 488 489 490 491 492 493 494 495 496
	if (copy_from_user(&einj, ubuf, usize))
		return -EFAULT;

	ret = aer_inject(&einj);
	return ret ? ret : usize;
}

static const struct file_operations aer_inject_fops = {
	.write = aer_inject_write,
	.owner = THIS_MODULE,
497
	.llseek = noop_llseek,
498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524
};

static struct miscdevice aer_inject_device = {
	.minor = MISC_DYNAMIC_MINOR,
	.name = "aer_inject",
	.fops = &aer_inject_fops,
};

static int __init aer_inject_init(void)
{
	return misc_register(&aer_inject_device);
}

static void __exit aer_inject_exit(void)
{
	struct aer_error *err, *err_next;
	unsigned long flags;
	struct pci_bus_ops *bus_ops;

	misc_deregister(&aer_inject_device);

	while ((bus_ops = pci_bus_ops_pop())) {
		pci_bus_set_ops(bus_ops->bus, bus_ops->ops);
		kfree(bus_ops);
	}

	spin_lock_irqsave(&inject_lock, flags);
525
	list_for_each_entry_safe(err, err_next, &einjected, list) {
526 527 528 529 530 531 532 533 534
		list_del(&err->list);
		kfree(err);
	}
	spin_unlock_irqrestore(&inject_lock, flags);
}

module_init(aer_inject_init);
module_exit(aer_inject_exit);

535
MODULE_DESCRIPTION("PCIe AER software error injector");
536
MODULE_LICENSE("GPL");