All new accounts created on Gitlab now require administrator approval. If you invite any collaborators, please let Flux staff know so they can approve the accounts.

arm_pmu.c 25.1 KB
Newer Older
1 2 3 4 5 6
#undef DEBUG

/*
 * ARM performance counter support.
 *
 * Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles
7
 * Copyright (C) 2010 ARM Ltd., Will Deacon <will.deacon@arm.com>
8
 *
9
 * This code is based on the sparc64 perf event code, which is in turn based
10
 * on the x86 code.
11 12 13
 */
#define pr_fmt(fmt) "hw perfevents: " fmt

14
#include <linux/bitmap.h>
15
#include <linux/cpumask.h>
16
#include <linux/cpu_pm.h>
17
#include <linux/export.h>
18
#include <linux/kernel.h>
19
#include <linux/of_device.h>
20
#include <linux/perf/arm_pmu.h>
21
#include <linux/platform_device.h>
22 23
#include <linux/slab.h>
#include <linux/spinlock.h>
24 25
#include <linux/irq.h>
#include <linux/irqdesc.h>
26

27
#include <asm/cputype.h>
28 29 30
#include <asm/irq_regs.h>

static int
31 32 33 34 35
armpmu_map_cache_event(const unsigned (*cache_map)
				      [PERF_COUNT_HW_CACHE_MAX]
				      [PERF_COUNT_HW_CACHE_OP_MAX]
				      [PERF_COUNT_HW_CACHE_RESULT_MAX],
		       u64 config)
36 37 38 39 40 41 42 43 44 45 46 47 48 49 50
{
	unsigned int cache_type, cache_op, cache_result, ret;

	cache_type = (config >>  0) & 0xff;
	if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
		return -EINVAL;

	cache_op = (config >>  8) & 0xff;
	if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
		return -EINVAL;

	cache_result = (config >> 16) & 0xff;
	if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
		return -EINVAL;

51
	ret = (int)(*cache_map)[cache_type][cache_op][cache_result];
52 53 54 55 56 57 58

	if (ret == CACHE_OP_UNSUPPORTED)
		return -ENOENT;

	return ret;
}

59
static int
60
armpmu_map_hw_event(const unsigned (*event_map)[PERF_COUNT_HW_MAX], u64 config)
61
{
62 63 64 65 66 67
	int mapping;

	if (config >= PERF_COUNT_HW_MAX)
		return -EINVAL;

	mapping = (*event_map)[config];
68
	return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping;
69 70 71
}

static int
72
armpmu_map_raw_event(u32 raw_event_mask, u64 config)
73
{
74 75 76
	return (int)(config & raw_event_mask);
}

77 78 79 80 81 82 83 84
int
armpmu_map_event(struct perf_event *event,
		 const unsigned (*event_map)[PERF_COUNT_HW_MAX],
		 const unsigned (*cache_map)
				[PERF_COUNT_HW_CACHE_MAX]
				[PERF_COUNT_HW_CACHE_OP_MAX]
				[PERF_COUNT_HW_CACHE_RESULT_MAX],
		 u32 raw_event_mask)
85 86
{
	u64 config = event->attr.config;
87
	int type = event->attr.type;
88

89 90 91 92
	if (type == event->pmu->type)
		return armpmu_map_raw_event(raw_event_mask, config);

	switch (type) {
93
	case PERF_TYPE_HARDWARE:
94
		return armpmu_map_hw_event(event_map, config);
95 96 97 98 99 100 101
	case PERF_TYPE_HW_CACHE:
		return armpmu_map_cache_event(cache_map, config);
	case PERF_TYPE_RAW:
		return armpmu_map_raw_event(raw_event_mask, config);
	}

	return -ENOENT;
102 103
}

104
int armpmu_event_set_period(struct perf_event *event)
105
{
106
	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
107
	struct hw_perf_event *hwc = &event->hw;
108
	s64 left = local64_read(&hwc->period_left);
109 110 111 112 113
	s64 period = hwc->sample_period;
	int ret = 0;

	if (unlikely(left <= -period)) {
		left = period;
114
		local64_set(&hwc->period_left, left);
115 116 117 118 119 120
		hwc->last_period = period;
		ret = 1;
	}

	if (unlikely(left <= 0)) {
		left += period;
121
		local64_set(&hwc->period_left, left);
122 123 124 125
		hwc->last_period = period;
		ret = 1;
	}

126 127 128 129 130 131 132 133
	/*
	 * Limit the maximum period to prevent the counter value
	 * from overtaking the one we are about to program. In
	 * effect we are reducing max_period to account for
	 * interrupt latency (and we are being very conservative).
	 */
	if (left > (armpmu->max_period >> 1))
		left = armpmu->max_period >> 1;
134

135
	local64_set(&hwc->prev_count, (u64)-left);
136

137
	armpmu->write_counter(event, (u64)(-left) & 0xffffffff);
138 139 140 141 142 143

	perf_event_update_userpage(event);

	return ret;
}

144
u64 armpmu_event_update(struct perf_event *event)
145
{
146
	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
147
	struct hw_perf_event *hwc = &event->hw;
148
	u64 delta, prev_raw_count, new_raw_count;
149 150

again:
151
	prev_raw_count = local64_read(&hwc->prev_count);
152
	new_raw_count = armpmu->read_counter(event);
153

154
	if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
155 156 157
			     new_raw_count) != prev_raw_count)
		goto again;

158
	delta = (new_raw_count - prev_raw_count) & armpmu->max_period;
159

160 161
	local64_add(delta, &event->count);
	local64_sub(delta, &hwc->period_left);
162 163 164 165 166

	return new_raw_count;
}

static void
Peter Zijlstra's avatar
Peter Zijlstra committed
167
armpmu_read(struct perf_event *event)
168
{
169
	armpmu_event_update(event);
170 171 172
}

static void
Peter Zijlstra's avatar
Peter Zijlstra committed
173
armpmu_stop(struct perf_event *event, int flags)
174
{
175
	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
176 177
	struct hw_perf_event *hwc = &event->hw;

Peter Zijlstra's avatar
Peter Zijlstra committed
178 179 180 181 182
	/*
	 * ARM pmu always has to update the counter, so ignore
	 * PERF_EF_UPDATE, see comments in armpmu_start().
	 */
	if (!(hwc->state & PERF_HES_STOPPED)) {
183 184
		armpmu->disable(event);
		armpmu_event_update(event);
Peter Zijlstra's avatar
Peter Zijlstra committed
185 186
		hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
	}
187 188
}

189
static void armpmu_start(struct perf_event *event, int flags)
190
{
191
	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
192 193
	struct hw_perf_event *hwc = &event->hw;

Peter Zijlstra's avatar
Peter Zijlstra committed
194 195 196 197 198 199 200 201
	/*
	 * ARM pmu always has to reprogram the period, so ignore
	 * PERF_EF_RELOAD, see the comment below.
	 */
	if (flags & PERF_EF_RELOAD)
		WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));

	hwc->state = 0;
202 203
	/*
	 * Set the period again. Some counters can't be stopped, so when we
Peter Zijlstra's avatar
Peter Zijlstra committed
204
	 * were stopped we simply disabled the IRQ source and the counter
205 206 207 208
	 * may have been left counting. If we don't do this step then we may
	 * get an interrupt too soon or *way* too late if the overflow has
	 * happened since disabling.
	 */
209 210
	armpmu_event_set_period(event);
	armpmu->enable(event);
211 212
}

Peter Zijlstra's avatar
Peter Zijlstra committed
213 214 215
static void
armpmu_del(struct perf_event *event, int flags)
{
216
	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
Mark Rutland's avatar
Mark Rutland committed
217
	struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events);
Peter Zijlstra's avatar
Peter Zijlstra committed
218 219 220 221
	struct hw_perf_event *hwc = &event->hw;
	int idx = hwc->idx;

	armpmu_stop(event, PERF_EF_UPDATE);
222 223
	hw_events->events[idx] = NULL;
	clear_bit(idx, hw_events->used_mask);
224 225
	if (armpmu->clear_event_idx)
		armpmu->clear_event_idx(hw_events, event);
Peter Zijlstra's avatar
Peter Zijlstra committed
226 227 228 229

	perf_event_update_userpage(event);
}

230
static int
Peter Zijlstra's avatar
Peter Zijlstra committed
231
armpmu_add(struct perf_event *event, int flags)
232
{
233
	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
Mark Rutland's avatar
Mark Rutland committed
234
	struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events);
235 236 237 238
	struct hw_perf_event *hwc = &event->hw;
	int idx;
	int err = 0;

239 240 241 242
	/* An event following a process won't be stopped earlier */
	if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus))
		return -ENOENT;

Peter Zijlstra's avatar
Peter Zijlstra committed
243
	perf_pmu_disable(event->pmu);
244

245
	/* If we don't have a space for the counter then finish early. */
246
	idx = armpmu->get_event_idx(hw_events, event);
247 248 249 250 251 252 253 254 255 256
	if (idx < 0) {
		err = idx;
		goto out;
	}

	/*
	 * If there is an event in the counter we are going to use then make
	 * sure it is disabled.
	 */
	event->hw.idx = idx;
257
	armpmu->disable(event);
258
	hw_events->events[idx] = event;
259

Peter Zijlstra's avatar
Peter Zijlstra committed
260 261 262
	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
	if (flags & PERF_EF_START)
		armpmu_start(event, PERF_EF_RELOAD);
263 264 265 266 267

	/* Propagate our changes to the userspace mapping. */
	perf_event_update_userpage(event);

out:
Peter Zijlstra's avatar
Peter Zijlstra committed
268
	perf_pmu_enable(event->pmu);
269 270 271 272
	return err;
}

static int
273 274
validate_event(struct pmu *pmu, struct pmu_hw_events *hw_events,
			       struct perf_event *event)
275
{
276
	struct arm_pmu *armpmu;
277

278 279 280
	if (is_software_event(event))
		return 1;

281 282 283 284 285 286 287 288
	/*
	 * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The
	 * core perf code won't check that the pmu->ctx == leader->ctx
	 * until after pmu->event_init(event).
	 */
	if (event->pmu != pmu)
		return 0;

289
	if (event->state < PERF_EVENT_STATE_OFF)
290 291 292
		return 1;

	if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec)
293
		return 1;
294

295
	armpmu = to_arm_pmu(event->pmu);
296
	return armpmu->get_event_idx(hw_events, event) >= 0;
297 298 299 300 301 302
}

static int
validate_group(struct perf_event *event)
{
	struct perf_event *sibling, *leader = event->group_leader;
303
	struct pmu_hw_events fake_pmu;
304

305 306 307 308
	/*
	 * Initialise the fake PMU. We only need to populate the
	 * used_mask for the purposes of validation.
	 */
309
	memset(&fake_pmu.used_mask, 0, sizeof(fake_pmu.used_mask));
310

311
	if (!validate_event(event->pmu, &fake_pmu, leader))
312
		return -EINVAL;
313 314

	list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
315
		if (!validate_event(event->pmu, &fake_pmu, sibling))
316
			return -EINVAL;
317 318
	}

319
	if (!validate_event(event->pmu, &fake_pmu, event))
320
		return -EINVAL;
321 322 323 324

	return 0;
}

325
static irqreturn_t armpmu_dispatch_irq(int irq, void *dev)
326
{
327 328 329
	struct arm_pmu *armpmu;
	struct platform_device *plat_device;
	struct arm_pmu_platdata *plat;
330 331
	int ret;
	u64 start_clock, finish_clock;
332

333 334 335 336 337 338 339
	/*
	 * we request the IRQ with a (possibly percpu) struct arm_pmu**, but
	 * the handlers expect a struct arm_pmu*. The percpu_irq framework will
	 * do any necessary shifting, we just need to perform the first
	 * dereference.
	 */
	armpmu = *(void **)dev;
340 341
	plat_device = armpmu->plat_device;
	plat = dev_get_platdata(&plat_device->dev);
342

343
	start_clock = sched_clock();
344
	if (plat && plat->handle_irq)
345
		ret = plat->handle_irq(irq, armpmu, armpmu->handle_irq);
346
	else
347
		ret = armpmu->handle_irq(irq, armpmu);
348 349 350 351
	finish_clock = sched_clock();

	perf_sample_event_took(finish_clock - start_clock);
	return ret;
352 353
}

354
static void
355
armpmu_release_hardware(struct arm_pmu *armpmu)
356
{
357
	armpmu->free_irq(armpmu);
358 359
}

360
static int
361
armpmu_reserve_hardware(struct arm_pmu *armpmu)
362
{
363
	int err = armpmu->request_irq(armpmu, armpmu_dispatch_irq);
364 365 366
	if (err) {
		armpmu_release_hardware(armpmu);
		return err;
367
	}
368

369
	return 0;
370 371 372 373 374
}

static void
hw_perf_event_destroy(struct perf_event *event)
{
375
	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
376 377 378 379
	atomic_t *active_events	 = &armpmu->active_events;
	struct mutex *pmu_reserve_mutex = &armpmu->reserve_mutex;

	if (atomic_dec_and_mutex_lock(active_events, pmu_reserve_mutex)) {
380
		armpmu_release_hardware(armpmu);
381
		mutex_unlock(pmu_reserve_mutex);
382 383 384
	}
}

385 386 387 388 389 390 391
static int
event_requires_mode_exclusion(struct perf_event_attr *attr)
{
	return attr->exclude_idle || attr->exclude_user ||
	       attr->exclude_kernel || attr->exclude_hv;
}

392 393 394
static int
__hw_perf_event_init(struct perf_event *event)
{
395
	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
396
	struct hw_perf_event *hwc = &event->hw;
397
	int mapping;
398

399
	mapping = armpmu->map_event(event);
400 401 402 403 404 405 406

	if (mapping < 0) {
		pr_debug("event %x:%llx not supported\n", event->attr.type,
			 event->attr.config);
		return mapping;
	}

407 408 409 410 411 412 413 414 415 416 417
	/*
	 * We don't assign an index until we actually place the event onto
	 * hardware. Use -1 to signify that we haven't decided where to put it
	 * yet. For SMP systems, each core has it's own PMU so we can't do any
	 * clever allocation or constraints checking at this point.
	 */
	hwc->idx		= -1;
	hwc->config_base	= 0;
	hwc->config		= 0;
	hwc->event_base		= 0;

418 419 420
	/*
	 * Check whether we need to exclude the counter from certain modes.
	 */
421 422 423
	if ((!armpmu->set_event_filter ||
	     armpmu->set_event_filter(hwc, &event->attr)) &&
	     event_requires_mode_exclusion(&event->attr)) {
424 425
		pr_debug("ARM performance counters do not support "
			 "mode exclusion\n");
426
		return -EOPNOTSUPP;
427 428 429
	}

	/*
430
	 * Store the event encoding into the config_base field.
431
	 */
432
	hwc->config_base	    |= (unsigned long)mapping;
433

434
	if (!is_sampling_event(event)) {
435 436 437 438 439 440 441
		/*
		 * For non-sampling runs, limit the sample_period to half
		 * of the counter width. That way, the new counter value
		 * is far less likely to overtake the previous one unless
		 * you have some serious IRQ latency issues.
		 */
		hwc->sample_period  = armpmu->max_period >> 1;
442
		hwc->last_period    = hwc->sample_period;
443
		local64_set(&hwc->period_left, hwc->sample_period);
444 445 446
	}

	if (event->group_leader != event) {
447
		if (validate_group(event) != 0)
448 449 450
			return -EINVAL;
	}

451
	return 0;
452 453
}

454
static int armpmu_event_init(struct perf_event *event)
455
{
456
	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
457
	int err = 0;
458
	atomic_t *active_events = &armpmu->active_events;
459

460 461 462 463 464 465 466 467 468 469 470
	/*
	 * Reject CPU-affine events for CPUs that are of a different class to
	 * that which this PMU handles. Process-following events (where
	 * event->cpu == -1) can be migrated between CPUs, and thus we have to
	 * reject them later (in armpmu_add) if they're scheduled on a
	 * different class of CPU.
	 */
	if (event->cpu != -1 &&
		!cpumask_test_cpu(event->cpu, &armpmu->supported_cpus))
		return -ENOENT;

471 472 473 474
	/* does not support taken branch sampling */
	if (has_branch_stack(event))
		return -EOPNOTSUPP;

475
	if (armpmu->map_event(event) == -ENOENT)
476 477
		return -ENOENT;

478 479
	event->destroy = hw_perf_event_destroy;

480 481 482
	if (!atomic_inc_not_zero(active_events)) {
		mutex_lock(&armpmu->reserve_mutex);
		if (atomic_read(active_events) == 0)
483
			err = armpmu_reserve_hardware(armpmu);
484 485

		if (!err)
486 487
			atomic_inc(active_events);
		mutex_unlock(&armpmu->reserve_mutex);
488 489 490
	}

	if (err)
491
		return err;
492 493 494 495 496

	err = __hw_perf_event_init(event);
	if (err)
		hw_perf_event_destroy(event);

497
	return err;
498 499
}

Peter Zijlstra's avatar
Peter Zijlstra committed
500
static void armpmu_enable(struct pmu *pmu)
501
{
502
	struct arm_pmu *armpmu = to_arm_pmu(pmu);
Mark Rutland's avatar
Mark Rutland committed
503
	struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events);
504
	int enabled = bitmap_weight(hw_events->used_mask, armpmu->num_events);
505

506 507 508 509
	/* For task-bound events we may be called on other CPUs */
	if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus))
		return;

510
	if (enabled)
511
		armpmu->start(armpmu);
512 513
}

Peter Zijlstra's avatar
Peter Zijlstra committed
514
static void armpmu_disable(struct pmu *pmu)
515
{
516
	struct arm_pmu *armpmu = to_arm_pmu(pmu);
517 518 519 520 521

	/* For task-bound events we may be called on other CPUs */
	if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus))
		return;

522
	armpmu->stop(armpmu);
523 524
}

525 526 527 528 529 530 531 532 533 534 535 536
/*
 * In heterogeneous systems, events are specific to a particular
 * microarchitecture, and aren't suitable for another. Thus, only match CPUs of
 * the same microarchitecture.
 */
static int armpmu_filter_match(struct perf_event *event)
{
	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
	unsigned int cpu = smp_processor_id();
	return cpumask_test_cpu(cpu, &armpmu->supported_cpus);
}

537
static void armpmu_init(struct arm_pmu *armpmu)
538 539 540
{
	atomic_set(&armpmu->active_events, 0);
	mutex_init(&armpmu->reserve_mutex);
541 542 543 544 545 546 547 548 549 550

	armpmu->pmu = (struct pmu) {
		.pmu_enable	= armpmu_enable,
		.pmu_disable	= armpmu_disable,
		.event_init	= armpmu_event_init,
		.add		= armpmu_add,
		.del		= armpmu_del,
		.start		= armpmu_start,
		.stop		= armpmu_stop,
		.read		= armpmu_read,
551
		.filter_match	= armpmu_filter_match,
552 553 554
	};
}

555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713
/* Set at runtime when we know what CPU type we are. */
static struct arm_pmu *__oprofile_cpu_pmu;

/*
 * Despite the names, these two functions are CPU-specific and are used
 * by the OProfile/perf code.
 */
const char *perf_pmu_name(void)
{
	if (!__oprofile_cpu_pmu)
		return NULL;

	return __oprofile_cpu_pmu->name;
}
EXPORT_SYMBOL_GPL(perf_pmu_name);

int perf_num_counters(void)
{
	int max_events = 0;

	if (__oprofile_cpu_pmu != NULL)
		max_events = __oprofile_cpu_pmu->num_events;

	return max_events;
}
EXPORT_SYMBOL_GPL(perf_num_counters);

static void cpu_pmu_enable_percpu_irq(void *data)
{
	int irq = *(int *)data;

	enable_percpu_irq(irq, IRQ_TYPE_NONE);
}

static void cpu_pmu_disable_percpu_irq(void *data)
{
	int irq = *(int *)data;

	disable_percpu_irq(irq);
}

static void cpu_pmu_free_irq(struct arm_pmu *cpu_pmu)
{
	int i, irq, irqs;
	struct platform_device *pmu_device = cpu_pmu->plat_device;
	struct pmu_hw_events __percpu *hw_events = cpu_pmu->hw_events;

	irqs = min(pmu_device->num_resources, num_possible_cpus());

	irq = platform_get_irq(pmu_device, 0);
	if (irq >= 0 && irq_is_percpu(irq)) {
		on_each_cpu(cpu_pmu_disable_percpu_irq, &irq, 1);
		free_percpu_irq(irq, &hw_events->percpu_pmu);
	} else {
		for (i = 0; i < irqs; ++i) {
			int cpu = i;

			if (cpu_pmu->irq_affinity)
				cpu = cpu_pmu->irq_affinity[i];

			if (!cpumask_test_and_clear_cpu(cpu, &cpu_pmu->active_irqs))
				continue;
			irq = platform_get_irq(pmu_device, i);
			if (irq >= 0)
				free_irq(irq, per_cpu_ptr(&hw_events->percpu_pmu, cpu));
		}
	}
}

static int cpu_pmu_request_irq(struct arm_pmu *cpu_pmu, irq_handler_t handler)
{
	int i, err, irq, irqs;
	struct platform_device *pmu_device = cpu_pmu->plat_device;
	struct pmu_hw_events __percpu *hw_events = cpu_pmu->hw_events;

	if (!pmu_device)
		return -ENODEV;

	irqs = min(pmu_device->num_resources, num_possible_cpus());
	if (irqs < 1) {
		pr_warn_once("perf/ARM: No irqs for PMU defined, sampling events not supported\n");
		return 0;
	}

	irq = platform_get_irq(pmu_device, 0);
	if (irq >= 0 && irq_is_percpu(irq)) {
		err = request_percpu_irq(irq, handler, "arm-pmu",
					 &hw_events->percpu_pmu);
		if (err) {
			pr_err("unable to request IRQ%d for ARM PMU counters\n",
				irq);
			return err;
		}
		on_each_cpu(cpu_pmu_enable_percpu_irq, &irq, 1);
	} else {
		for (i = 0; i < irqs; ++i) {
			int cpu = i;

			err = 0;
			irq = platform_get_irq(pmu_device, i);
			if (irq < 0)
				continue;

			if (cpu_pmu->irq_affinity)
				cpu = cpu_pmu->irq_affinity[i];

			/*
			 * If we have a single PMU interrupt that we can't shift,
			 * assume that we're running on a uniprocessor machine and
			 * continue. Otherwise, continue without this interrupt.
			 */
			if (irq_set_affinity(irq, cpumask_of(cpu)) && irqs > 1) {
				pr_warn("unable to set irq affinity (irq=%d, cpu=%u)\n",
					irq, cpu);
				continue;
			}

			err = request_irq(irq, handler,
					  IRQF_NOBALANCING | IRQF_NO_THREAD, "arm-pmu",
					  per_cpu_ptr(&hw_events->percpu_pmu, cpu));
			if (err) {
				pr_err("unable to request IRQ%d for ARM PMU counters\n",
					irq);
				return err;
			}

			cpumask_set_cpu(cpu, &cpu_pmu->active_irqs);
		}
	}

	return 0;
}

/*
 * PMU hardware loses all context when a CPU goes offline.
 * When a CPU is hotplugged back in, since some hardware registers are
 * UNKNOWN at reset, the PMU must be explicitly reset to avoid reading
 * junk values out of them.
 */
static int cpu_pmu_notify(struct notifier_block *b, unsigned long action,
			  void *hcpu)
{
	int cpu = (unsigned long)hcpu;
	struct arm_pmu *pmu = container_of(b, struct arm_pmu, hotplug_nb);

	if ((action & ~CPU_TASKS_FROZEN) != CPU_STARTING)
		return NOTIFY_DONE;

	if (!cpumask_test_cpu(cpu, &pmu->supported_cpus))
		return NOTIFY_DONE;

	if (pmu->reset)
		pmu->reset(pmu);
	else
		return NOTIFY_DONE;

	return NOTIFY_OK;
}

714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739
#ifdef CONFIG_CPU_PM
static void cpu_pm_pmu_setup(struct arm_pmu *armpmu, unsigned long cmd)
{
	struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events);
	struct perf_event *event;
	int idx;

	for (idx = 0; idx < armpmu->num_events; idx++) {
		/*
		 * If the counter is not used skip it, there is no
		 * need of stopping/restarting it.
		 */
		if (!test_bit(idx, hw_events->used_mask))
			continue;

		event = hw_events->events[idx];

		switch (cmd) {
		case CPU_PM_ENTER:
			/*
			 * Stop and update the counter
			 */
			armpmu_stop(event, PERF_EF_UPDATE);
			break;
		case CPU_PM_EXIT:
		case CPU_PM_ENTER_FAILED:
740 741 742 743 744 745 746 747 748 749 750 751 752
			 /*
			  * Restore and enable the counter.
			  * armpmu_start() indirectly calls
			  *
			  * perf_event_update_userpage()
			  *
			  * that requires RCU read locking to be functional,
			  * wrap the call within RCU_NONIDLE to make the
			  * RCU subsystem aware this cpu is not idle from
			  * an RCU perspective for the armpmu_start() call
			  * duration.
			  */
			RCU_NONIDLE(armpmu_start(event, PERF_EF_RELOAD));
753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811
			break;
		default:
			break;
		}
	}
}

static int cpu_pm_pmu_notify(struct notifier_block *b, unsigned long cmd,
			     void *v)
{
	struct arm_pmu *armpmu = container_of(b, struct arm_pmu, cpu_pm_nb);
	struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events);
	int enabled = bitmap_weight(hw_events->used_mask, armpmu->num_events);

	if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus))
		return NOTIFY_DONE;

	/*
	 * Always reset the PMU registers on power-up even if
	 * there are no events running.
	 */
	if (cmd == CPU_PM_EXIT && armpmu->reset)
		armpmu->reset(armpmu);

	if (!enabled)
		return NOTIFY_OK;

	switch (cmd) {
	case CPU_PM_ENTER:
		armpmu->stop(armpmu);
		cpu_pm_pmu_setup(armpmu, cmd);
		break;
	case CPU_PM_EXIT:
		cpu_pm_pmu_setup(armpmu, cmd);
	case CPU_PM_ENTER_FAILED:
		armpmu->start(armpmu);
		break;
	default:
		return NOTIFY_DONE;
	}

	return NOTIFY_OK;
}

static int cpu_pm_pmu_register(struct arm_pmu *cpu_pmu)
{
	cpu_pmu->cpu_pm_nb.notifier_call = cpu_pm_pmu_notify;
	return cpu_pm_register_notifier(&cpu_pmu->cpu_pm_nb);
}

static void cpu_pm_pmu_unregister(struct arm_pmu *cpu_pmu)
{
	cpu_pm_unregister_notifier(&cpu_pmu->cpu_pm_nb);
}
#else
static inline int cpu_pm_pmu_register(struct arm_pmu *cpu_pmu) { return 0; }
static inline void cpu_pm_pmu_unregister(struct arm_pmu *cpu_pmu) { }
#endif

812 813 814 815 816 817 818 819 820 821 822 823 824 825 826
static int cpu_pmu_init(struct arm_pmu *cpu_pmu)
{
	int err;
	int cpu;
	struct pmu_hw_events __percpu *cpu_hw_events;

	cpu_hw_events = alloc_percpu(struct pmu_hw_events);
	if (!cpu_hw_events)
		return -ENOMEM;

	cpu_pmu->hotplug_nb.notifier_call = cpu_pmu_notify;
	err = register_cpu_notifier(&cpu_pmu->hotplug_nb);
	if (err)
		goto out_hw_events;

827 828 829 830
	err = cpu_pm_pmu_register(cpu_pmu);
	if (err)
		goto out_unregister;

831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849
	for_each_possible_cpu(cpu) {
		struct pmu_hw_events *events = per_cpu_ptr(cpu_hw_events, cpu);
		raw_spin_lock_init(&events->pmu_lock);
		events->percpu_pmu = cpu_pmu;
	}

	cpu_pmu->hw_events	= cpu_hw_events;
	cpu_pmu->request_irq	= cpu_pmu_request_irq;
	cpu_pmu->free_irq	= cpu_pmu_free_irq;

	/* Ensure the PMU has sane values out of reset. */
	if (cpu_pmu->reset)
		on_each_cpu_mask(&cpu_pmu->supported_cpus, cpu_pmu->reset,
			 cpu_pmu, 1);

	/* If no interrupts available, set the corresponding capability flag */
	if (!platform_get_irq(cpu_pmu->plat_device, 0))
		cpu_pmu->pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;

850 851 852 853 854 855 856 857
	/*
	 * This is a CPU PMU potentially in a heterogeneous configuration (e.g.
	 * big.LITTLE). This is not an uncore PMU, and we have taken ctx
	 * sharing into account (e.g. with our pmu::filter_match callback and
	 * pmu::event_init group validation).
	 */
	cpu_pmu->pmu.capabilities |= PERF_PMU_CAP_HETEROGENEOUS_CPUS;

858 859
	return 0;

860 861
out_unregister:
	unregister_cpu_notifier(&cpu_pmu->hotplug_nb);
862 863 864 865 866 867 868
out_hw_events:
	free_percpu(cpu_hw_events);
	return err;
}

static void cpu_pmu_destroy(struct arm_pmu *cpu_pmu)
{
869
	cpu_pm_pmu_unregister(cpu_pmu);
870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898
	unregister_cpu_notifier(&cpu_pmu->hotplug_nb);
	free_percpu(cpu_pmu->hw_events);
}

/*
 * CPU PMU identification and probing.
 */
static int probe_current_pmu(struct arm_pmu *pmu,
			     const struct pmu_probe_info *info)
{
	int cpu = get_cpu();
	unsigned int cpuid = read_cpuid_id();
	int ret = -ENODEV;

	pr_info("probing PMU on CPU %d\n", cpu);

	for (; info->init != NULL; info++) {
		if ((cpuid & info->mask) != info->cpuid)
			continue;
		ret = info->init(pmu);
		break;
	}

	put_cpu();
	return ret;
}

static int of_pmu_irq_cfg(struct arm_pmu *pmu)
{
899 900
	int *irqs, i = 0;
	bool using_spi = false;
901 902 903 904 905 906
	struct platform_device *pdev = pmu->plat_device;

	irqs = kcalloc(pdev->num_resources, sizeof(*irqs), GFP_KERNEL);
	if (!irqs)
		return -ENOMEM;

907
	do {
908
		struct device_node *dn;
909
		int cpu, irq;
910

911 912 913
		/* See if we have an affinity entry */
		dn = of_parse_phandle(pdev->dev.of_node, "interrupt-affinity", i);
		if (!dn)
914
			break;
915 916 917 918 919 920 921 922 923 924 925 926 927 928

		/* Check the IRQ type and prohibit a mix of PPIs and SPIs */
		irq = platform_get_irq(pdev, i);
		if (irq >= 0) {
			bool spi = !irq_is_percpu(irq);

			if (i > 0 && spi != using_spi) {
				pr_err("PPI/SPI IRQ type mismatch for %s!\n",
					dn->name);
				kfree(irqs);
				return -EINVAL;
			}

			using_spi = spi;
929 930
		}

931
		/* Now look up the logical CPU number */
932 933 934 935 936 937 938
		for_each_possible_cpu(cpu) {
			struct device_node *cpu_dn;

			cpu_dn = of_cpu_device_node_get(cpu);
			of_node_put(cpu_dn);

			if (dn == cpu_dn)
939
				break;
940
		}
941 942 943 944

		if (cpu >= nr_cpu_ids) {
			pr_warn("Failed to find logical CPU for %s\n",
				dn->name);
945
			of_node_put(dn);
946
			cpumask_setall(&pmu->supported_cpus);
947 948
			break;
		}
949
		of_node_put(dn);
950

951 952
		/* For SPIs, we need to track the affinity per IRQ */
		if (using_spi) {
953
			if (i >= pdev->num_resources)
954 955 956 957 958 959
				break;

			irqs[i] = cpu;
		}

		/* Keep track of the CPUs containing this PMU type */
960
		cpumask_set_cpu(cpu, &pmu->supported_cpus);
961 962
		i++;
	} while (1);
963

964 965 966 967 968 969
	/* If we didn't manage to parse anything, claim to support all CPUs */
	if (cpumask_weight(&pmu->supported_cpus) == 0)
		cpumask_setall(&pmu->supported_cpus);

	/* If we matched up the IRQ affinities, use them to route the SPIs */
	if (using_spi && i == pdev->num_resources)
970
		pmu->irq_affinity = irqs;
971
	else
972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992
		kfree(irqs);

	return 0;
}

int arm_pmu_device_probe(struct platform_device *pdev,
			 const struct of_device_id *of_table,
			 const struct pmu_probe_info *probe_table)
{
	const struct of_device_id *of_id;
	const int (*init_fn)(struct arm_pmu *);
	struct device_node *node = pdev->dev.of_node;
	struct arm_pmu *pmu;
	int ret = -ENODEV;

	pmu = kzalloc(sizeof(struct arm_pmu), GFP_KERNEL);
	if (!pmu) {
		pr_info("failed to allocate PMU device!\n");
		return -ENOMEM;
	}

993 994
	armpmu_init(pmu);

995 996 997 998 999
	pmu->plat_device = pdev;

	if (node && (of_id = of_match_node(of_table, pdev->dev.of_node))) {
		init_fn = of_id->data;

1000 1001 1002 1003 1004 1005 1006 1007 1008
		pmu->secure_access = of_property_read_bool(pdev->dev.of_node,
							   "secure-reg-access");

		/* arm64 systems boot only as non-secure */
		if (IS_ENABLED(CONFIG_ARM64) && pmu->secure_access) {
			pr_warn("ignoring \"secure-reg-access\" property for arm64\n");
			pmu->secure_access = false;
		}

1009 1010 1011 1012 1013 1014 1015 1016 1017
		ret = of_pmu_irq_cfg(pmu);
		if (!ret)
			ret = init_fn(pmu);
	} else {
		ret = probe_current_pmu(pmu, probe_table);
		cpumask_setall(&pmu->supported_cpus);
	}

	if (ret) {
1018
		pr_info("%s: failed to probe PMU!\n", of_node_full_name(node));
1019 1020 1021 1022 1023 1024 1025
		goto out_free;
	}

	ret = cpu_pmu_init(pmu);
	if (ret)
		goto out_free;

1026
	ret = perf_pmu_register(&pmu->pmu, pmu->name, -1);
1027 1028 1029
	if (ret)
		goto out_destroy;

1030 1031 1032
	if (!__oprofile_cpu_pmu)
		__oprofile_cpu_pmu = pmu;

1033 1034 1035
	pr_info("enabled with %s PMU driver, %d counters available\n",
			pmu->name, pmu->num_events);

1036 1037 1038 1039 1040
	return 0;

out_destroy:
	cpu_pmu_destroy(pmu);
out_free:
1041 1042
	pr_info("%s: failed to register PMU devices!\n",
		of_node_full_name(node));
1043
	kfree(pmu->irq_affinity);
1044 1045 1046
	kfree(pmu);
	return ret;
}