intel_idle.c 36 KB
Newer Older
1 2 3
/*
 * intel_idle.c - native hardware idle loop for modern Intel processors
 *
4
 * Copyright (c) 2013, Intel Corporation.
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55
 * Len Brown <len.brown@intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 */

/*
 * intel_idle is a cpuidle driver that loads on specific Intel processors
 * in lieu of the legacy ACPI processor_idle driver.  The intent is to
 * make Linux more efficient on these processors, as intel_idle knows
 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.
 */

/*
 * Design Assumptions
 *
 * All CPUs have same idle states as boot CPU
 *
 * Chipset BM_STS (bus master status) bit is a NOP
 *	for preventing entry into deep C-stats
 */

/*
 * Known limitations
 *
 * The driver currently initializes for_each_online_cpu() upon modprobe.
 * It it unaware of subsequent processors hot-added to the system.
 * This means that if you boot with maxcpus=n and later online
 * processors above n, those processors will use C1 only.
 *
 * ACPI has a .suspend hack to turn off deep c-statees during suspend
 * to avoid complications with the lapic timer workaround.
 * Have not seen issues with suspend, but may need same workaround here.
 *
 */

/* un-comment DEBUG to enable pr_debug() statements */
#define DEBUG

#include <linux/kernel.h>
#include <linux/cpuidle.h>
56
#include <linux/tick.h>
57 58
#include <trace/events/power.h>
#include <linux/sched.h>
59 60
#include <linux/notifier.h>
#include <linux/cpu.h>
61
#include <linux/moduleparam.h>
62
#include <asm/cpu_device_id.h>
63
#include <asm/intel-family.h>
64
#include <asm/mwait.h>
65
#include <asm/msr.h>
66

67
#define INTEL_IDLE_VERSION "0.4.1"
68 69 70 71 72 73 74
#define PREFIX "intel_idle: "

static struct cpuidle_driver intel_idle_driver = {
	.name = "intel_idle",
	.owner = THIS_MODULE,
};
/* intel_idle.max_cstate=0 disables driver */
75
static int max_cstate = CPUIDLE_STATE_MAX - 1;
76

77
static unsigned int mwait_substates;
78

79
#define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF
80
/* Reliable LAPIC Timer States, bit 1 for C1 etc.  */
81
static unsigned int lapic_timer_reliable_states = (1 << 1);	 /* Default to only C1 */
82

83 84 85 86 87 88 89 90
struct idle_cpu {
	struct cpuidle_state *state_table;

	/*
	 * Hardware C-state auto-demotion may not always be optimal.
	 * Indicate which enable bits to clear here.
	 */
	unsigned long auto_demotion_disable_flags;
91
	bool byt_auto_demotion_disable_flag;
92
	bool disable_promotion_to_c1e;
93 94 95
};

static const struct idle_cpu *icpu;
96
static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
97 98
static int intel_idle(struct cpuidle_device *dev,
			struct cpuidle_driver *drv, int index);
99 100
static void intel_idle_freeze(struct cpuidle_device *dev,
			      struct cpuidle_driver *drv, int index);
101
static int intel_idle_cpu_init(int cpu);
102 103 104

static struct cpuidle_state *cpuidle_state_table;

105 106 107 108 109 110 111 112
/*
 * Set this flag for states where the HW flushes the TLB for us
 * and so we don't need cross-calls to keep it consistent.
 * If this flag is set, SW flushes the TLB, so even if the
 * HW doesn't do the flushing, this flag is safe to use.
 */
#define CPUIDLE_FLAG_TLB_FLUSHED	0x10000

113 114 115 116 117 118 119 120 121 122
/*
 * MWAIT takes an 8-bit "hint" in EAX "suggesting"
 * the C-state (top nibble) and sub-state (bottom nibble)
 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc.
 *
 * We store the hint at the top of our "flags" for each state.
 */
#define flg2MWAIT(flags) (((flags) >> 24) & 0xFF)
#define MWAIT2flg(eax) ((eax & 0xFF) << 24)

123 124 125 126 127
/*
 * States are indexed by the cstate number,
 * which is also the index into the MWAIT hint array.
 * Thus C0 is a dummy.
 */
128
static struct cpuidle_state nehalem_cstates[] = {
129
	{
130
		.name = "C1-NHM",
131
		.desc = "MWAIT 0x00",
132
		.flags = MWAIT2flg(0x00),
133 134
		.exit_latency = 3,
		.target_residency = 6,
135 136
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
137 138 139
	{
		.name = "C1E-NHM",
		.desc = "MWAIT 0x01",
140
		.flags = MWAIT2flg(0x01),
141 142
		.exit_latency = 10,
		.target_residency = 20,
143 144
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
145
	{
146
		.name = "C3-NHM",
147
		.desc = "MWAIT 0x10",
148
		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
149 150
		.exit_latency = 20,
		.target_residency = 80,
151 152
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
153
	{
154
		.name = "C6-NHM",
155
		.desc = "MWAIT 0x20",
156
		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
157 158
		.exit_latency = 200,
		.target_residency = 800,
159 160
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
161 162
	{
		.enter = NULL }
163 164
};

165
static struct cpuidle_state snb_cstates[] = {
166
	{
167
		.name = "C1-SNB",
168
		.desc = "MWAIT 0x00",
169
		.flags = MWAIT2flg(0x00),
170 171
		.exit_latency = 2,
		.target_residency = 2,
172 173
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
174 175 176
	{
		.name = "C1E-SNB",
		.desc = "MWAIT 0x01",
177
		.flags = MWAIT2flg(0x01),
178 179
		.exit_latency = 10,
		.target_residency = 20,
180 181
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
182
	{
183
		.name = "C3-SNB",
184
		.desc = "MWAIT 0x10",
185
		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
186
		.exit_latency = 80,
187
		.target_residency = 211,
188 189
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
190
	{
191
		.name = "C6-SNB",
192
		.desc = "MWAIT 0x20",
193
		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
194
		.exit_latency = 104,
195
		.target_residency = 345,
196 197
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
198
	{
199
		.name = "C7-SNB",
200
		.desc = "MWAIT 0x30",
201
		.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
202
		.exit_latency = 109,
203
		.target_residency = 345,
204 205
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
206 207
	{
		.enter = NULL }
208 209
};

Len Brown's avatar
Len Brown committed
210 211 212 213
static struct cpuidle_state byt_cstates[] = {
	{
		.name = "C1-BYT",
		.desc = "MWAIT 0x00",
214
		.flags = MWAIT2flg(0x00),
Len Brown's avatar
Len Brown committed
215 216
		.exit_latency = 1,
		.target_residency = 1,
217 218
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
Len Brown's avatar
Len Brown committed
219 220 221
	{
		.name = "C6N-BYT",
		.desc = "MWAIT 0x58",
222
		.flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
223
		.exit_latency = 300,
Len Brown's avatar
Len Brown committed
224
		.target_residency = 275,
225 226
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
Len Brown's avatar
Len Brown committed
227 228 229
	{
		.name = "C6S-BYT",
		.desc = "MWAIT 0x52",
230
		.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
231
		.exit_latency = 500,
Len Brown's avatar
Len Brown committed
232
		.target_residency = 560,
233 234
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
Len Brown's avatar
Len Brown committed
235 236 237
	{
		.name = "C7-BYT",
		.desc = "MWAIT 0x60",
238
		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
Len Brown's avatar
Len Brown committed
239
		.exit_latency = 1200,
240
		.target_residency = 4000,
241 242
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
Len Brown's avatar
Len Brown committed
243 244 245
	{
		.name = "C7S-BYT",
		.desc = "MWAIT 0x64",
246
		.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
Len Brown's avatar
Len Brown committed
247 248
		.exit_latency = 10000,
		.target_residency = 20000,
249 250
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
Len Brown's avatar
Len Brown committed
251 252 253 254
	{
		.enter = NULL }
};

255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299
static struct cpuidle_state cht_cstates[] = {
	{
		.name = "C1-CHT",
		.desc = "MWAIT 0x00",
		.flags = MWAIT2flg(0x00),
		.exit_latency = 1,
		.target_residency = 1,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.name = "C6N-CHT",
		.desc = "MWAIT 0x58",
		.flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
		.exit_latency = 80,
		.target_residency = 275,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.name = "C6S-CHT",
		.desc = "MWAIT 0x52",
		.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
		.exit_latency = 200,
		.target_residency = 560,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.name = "C7-CHT",
		.desc = "MWAIT 0x60",
		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
		.exit_latency = 1200,
		.target_residency = 4000,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.name = "C7S-CHT",
		.desc = "MWAIT 0x64",
		.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
		.exit_latency = 10000,
		.target_residency = 20000,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.enter = NULL }
};

300
static struct cpuidle_state ivb_cstates[] = {
301
	{
Len Brown's avatar
Len Brown committed
302 303
		.name = "C1-IVB",
		.desc = "MWAIT 0x00",
304
		.flags = MWAIT2flg(0x00),
Len Brown's avatar
Len Brown committed
305 306
		.exit_latency = 1,
		.target_residency = 1,
307 308
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
309 310 311
	{
		.name = "C1E-IVB",
		.desc = "MWAIT 0x01",
312
		.flags = MWAIT2flg(0x01),
313 314
		.exit_latency = 10,
		.target_residency = 20,
315 316
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
317
	{
Len Brown's avatar
Len Brown committed
318 319
		.name = "C3-IVB",
		.desc = "MWAIT 0x10",
320
		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
Len Brown's avatar
Len Brown committed
321 322
		.exit_latency = 59,
		.target_residency = 156,
323 324
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
325
	{
Len Brown's avatar
Len Brown committed
326 327
		.name = "C6-IVB",
		.desc = "MWAIT 0x20",
328
		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
Len Brown's avatar
Len Brown committed
329 330
		.exit_latency = 80,
		.target_residency = 300,
331 332
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
333
	{
Len Brown's avatar
Len Brown committed
334 335
		.name = "C7-IVB",
		.desc = "MWAIT 0x30",
336
		.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
Len Brown's avatar
Len Brown committed
337 338
		.exit_latency = 87,
		.target_residency = 300,
339 340
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
341 342
	{
		.enter = NULL }
Len Brown's avatar
Len Brown committed
343 344
};

345 346 347 348
static struct cpuidle_state ivt_cstates[] = {
	{
		.name = "C1-IVT",
		.desc = "MWAIT 0x00",
349
		.flags = MWAIT2flg(0x00),
350 351
		.exit_latency = 1,
		.target_residency = 1,
352 353
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
354 355 356
	{
		.name = "C1E-IVT",
		.desc = "MWAIT 0x01",
357
		.flags = MWAIT2flg(0x01),
358 359
		.exit_latency = 10,
		.target_residency = 80,
360 361
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
362 363 364
	{
		.name = "C3-IVT",
		.desc = "MWAIT 0x10",
365
		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
366 367
		.exit_latency = 59,
		.target_residency = 156,
368 369
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
370 371 372
	{
		.name = "C6-IVT",
		.desc = "MWAIT 0x20",
373
		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
374 375
		.exit_latency = 82,
		.target_residency = 300,
376 377
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
378 379 380 381 382 383 384 385
	{
		.enter = NULL }
};

static struct cpuidle_state ivt_cstates_4s[] = {
	{
		.name = "C1-IVT-4S",
		.desc = "MWAIT 0x00",
386
		.flags = MWAIT2flg(0x00),
387 388
		.exit_latency = 1,
		.target_residency = 1,
389 390
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
391 392 393
	{
		.name = "C1E-IVT-4S",
		.desc = "MWAIT 0x01",
394
		.flags = MWAIT2flg(0x01),
395 396
		.exit_latency = 10,
		.target_residency = 250,
397 398
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
399 400 401
	{
		.name = "C3-IVT-4S",
		.desc = "MWAIT 0x10",
402
		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
403 404
		.exit_latency = 59,
		.target_residency = 300,
405 406
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
407 408 409
	{
		.name = "C6-IVT-4S",
		.desc = "MWAIT 0x20",
410
		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
411 412
		.exit_latency = 84,
		.target_residency = 400,
413 414
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
415 416 417 418 419 420 421 422
	{
		.enter = NULL }
};

static struct cpuidle_state ivt_cstates_8s[] = {
	{
		.name = "C1-IVT-8S",
		.desc = "MWAIT 0x00",
423
		.flags = MWAIT2flg(0x00),
424 425
		.exit_latency = 1,
		.target_residency = 1,
426 427
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
428 429 430
	{
		.name = "C1E-IVT-8S",
		.desc = "MWAIT 0x01",
431
		.flags = MWAIT2flg(0x01),
432 433
		.exit_latency = 10,
		.target_residency = 500,
434 435
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
436 437 438
	{
		.name = "C3-IVT-8S",
		.desc = "MWAIT 0x10",
439
		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
440 441
		.exit_latency = 59,
		.target_residency = 600,
442 443
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
444 445 446
	{
		.name = "C6-IVT-8S",
		.desc = "MWAIT 0x20",
447
		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
448 449
		.exit_latency = 88,
		.target_residency = 700,
450 451
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
452 453 454 455
	{
		.enter = NULL }
};

456
static struct cpuidle_state hsw_cstates[] = {
457
	{
Len Brown's avatar
Len Brown committed
458 459
		.name = "C1-HSW",
		.desc = "MWAIT 0x00",
460
		.flags = MWAIT2flg(0x00),
Len Brown's avatar
Len Brown committed
461 462
		.exit_latency = 2,
		.target_residency = 2,
463 464
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
465 466 467
	{
		.name = "C1E-HSW",
		.desc = "MWAIT 0x01",
468
		.flags = MWAIT2flg(0x01),
469 470
		.exit_latency = 10,
		.target_residency = 20,
471 472
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
473
	{
Len Brown's avatar
Len Brown committed
474 475
		.name = "C3-HSW",
		.desc = "MWAIT 0x10",
476
		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
Len Brown's avatar
Len Brown committed
477 478
		.exit_latency = 33,
		.target_residency = 100,
479 480
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
481
	{
Len Brown's avatar
Len Brown committed
482 483
		.name = "C6-HSW",
		.desc = "MWAIT 0x20",
484
		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
Len Brown's avatar
Len Brown committed
485 486
		.exit_latency = 133,
		.target_residency = 400,
487 488
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
489
	{
Len Brown's avatar
Len Brown committed
490 491
		.name = "C7s-HSW",
		.desc = "MWAIT 0x32",
492
		.flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
Len Brown's avatar
Len Brown committed
493 494
		.exit_latency = 166,
		.target_residency = 500,
495 496
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
497 498 499
	{
		.name = "C8-HSW",
		.desc = "MWAIT 0x40",
500
		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
501 502
		.exit_latency = 300,
		.target_residency = 900,
503 504
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
505 506 507
	{
		.name = "C9-HSW",
		.desc = "MWAIT 0x50",
508
		.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
509 510
		.exit_latency = 600,
		.target_residency = 1800,
511 512
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
513 514 515
	{
		.name = "C10-HSW",
		.desc = "MWAIT 0x60",
516
		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
517 518
		.exit_latency = 2600,
		.target_residency = 7700,
519 520
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
521 522
	{
		.enter = NULL }
Len Brown's avatar
Len Brown committed
523
};
Len Brown's avatar
Len Brown committed
524 525 526 527
static struct cpuidle_state bdw_cstates[] = {
	{
		.name = "C1-BDW",
		.desc = "MWAIT 0x00",
528
		.flags = MWAIT2flg(0x00),
Len Brown's avatar
Len Brown committed
529 530
		.exit_latency = 2,
		.target_residency = 2,
531 532
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
Len Brown's avatar
Len Brown committed
533 534 535
	{
		.name = "C1E-BDW",
		.desc = "MWAIT 0x01",
536
		.flags = MWAIT2flg(0x01),
Len Brown's avatar
Len Brown committed
537 538
		.exit_latency = 10,
		.target_residency = 20,
539 540
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
Len Brown's avatar
Len Brown committed
541 542 543
	{
		.name = "C3-BDW",
		.desc = "MWAIT 0x10",
544
		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
Len Brown's avatar
Len Brown committed
545 546
		.exit_latency = 40,
		.target_residency = 100,
547 548
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
Len Brown's avatar
Len Brown committed
549 550 551
	{
		.name = "C6-BDW",
		.desc = "MWAIT 0x20",
552
		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
Len Brown's avatar
Len Brown committed
553 554
		.exit_latency = 133,
		.target_residency = 400,
555 556
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
Len Brown's avatar
Len Brown committed
557 558 559
	{
		.name = "C7s-BDW",
		.desc = "MWAIT 0x32",
560
		.flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
Len Brown's avatar
Len Brown committed
561 562
		.exit_latency = 166,
		.target_residency = 500,
563 564
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
Len Brown's avatar
Len Brown committed
565 566 567
	{
		.name = "C8-BDW",
		.desc = "MWAIT 0x40",
568
		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
Len Brown's avatar
Len Brown committed
569 570
		.exit_latency = 300,
		.target_residency = 900,
571 572
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
Len Brown's avatar
Len Brown committed
573 574 575
	{
		.name = "C9-BDW",
		.desc = "MWAIT 0x50",
576
		.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
Len Brown's avatar
Len Brown committed
577 578
		.exit_latency = 600,
		.target_residency = 1800,
579 580
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
Len Brown's avatar
Len Brown committed
581 582 583
	{
		.name = "C10-BDW",
		.desc = "MWAIT 0x60",
584
		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
Len Brown's avatar
Len Brown committed
585 586
		.exit_latency = 2600,
		.target_residency = 7700,
587 588
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
Len Brown's avatar
Len Brown committed
589 590 591
	{
		.enter = NULL }
};
Len Brown's avatar
Len Brown committed
592

593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621
static struct cpuidle_state skl_cstates[] = {
	{
		.name = "C1-SKL",
		.desc = "MWAIT 0x00",
		.flags = MWAIT2flg(0x00),
		.exit_latency = 2,
		.target_residency = 2,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.name = "C1E-SKL",
		.desc = "MWAIT 0x01",
		.flags = MWAIT2flg(0x01),
		.exit_latency = 10,
		.target_residency = 20,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.name = "C3-SKL",
		.desc = "MWAIT 0x10",
		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
		.exit_latency = 70,
		.target_residency = 100,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.name = "C6-SKL",
		.desc = "MWAIT 0x20",
		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
622
		.exit_latency = 85,
623 624 625 626 627 628 629 630 631 632 633 634 635 636 637
		.target_residency = 200,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.name = "C7s-SKL",
		.desc = "MWAIT 0x33",
		.flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED,
		.exit_latency = 124,
		.target_residency = 800,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.name = "C8-SKL",
		.desc = "MWAIT 0x40",
		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
638
		.exit_latency = 200,
639 640 641
		.target_residency = 800,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
642 643 644 645 646 647 648 649
	{
		.name = "C9-SKL",
		.desc = "MWAIT 0x50",
		.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
		.exit_latency = 480,
		.target_residency = 5000,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
650 651 652 653 654 655 656 657 658 659 660 661
	{
		.name = "C10-SKL",
		.desc = "MWAIT 0x60",
		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
		.exit_latency = 890,
		.target_residency = 5000,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.enter = NULL }
};

Len Brown's avatar
Len Brown committed
662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690
static struct cpuidle_state skx_cstates[] = {
	{
		.name = "C1-SKX",
		.desc = "MWAIT 0x00",
		.flags = MWAIT2flg(0x00),
		.exit_latency = 2,
		.target_residency = 2,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.name = "C1E-SKX",
		.desc = "MWAIT 0x01",
		.flags = MWAIT2flg(0x01),
		.exit_latency = 10,
		.target_residency = 20,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.name = "C6-SKX",
		.desc = "MWAIT 0x20",
		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
		.exit_latency = 133,
		.target_residency = 600,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.enter = NULL }
};

691
static struct cpuidle_state atom_cstates[] = {
692
	{
693
		.name = "C1E-ATM",
694
		.desc = "MWAIT 0x00",
695
		.flags = MWAIT2flg(0x00),
696 697
		.exit_latency = 10,
		.target_residency = 20,
698 699
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
700
	{
701
		.name = "C2-ATM",
702
		.desc = "MWAIT 0x10",
703
		.flags = MWAIT2flg(0x10),
704 705
		.exit_latency = 20,
		.target_residency = 80,
706 707
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
708
	{
709
		.name = "C4-ATM",
710
		.desc = "MWAIT 0x30",
711
		.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
712 713
		.exit_latency = 100,
		.target_residency = 400,
714 715
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
716
	{
717
		.name = "C6-ATM",
Len Brown's avatar
Len Brown committed
718
		.desc = "MWAIT 0x52",
719
		.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
Len Brown's avatar
Len Brown committed
720 721
		.exit_latency = 140,
		.target_residency = 560,
722 723
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
724 725
	{
		.enter = NULL }
726
};
727
static struct cpuidle_state avn_cstates[] = {
728 729 730
	{
		.name = "C1-AVN",
		.desc = "MWAIT 0x00",
731
		.flags = MWAIT2flg(0x00),
732 733
		.exit_latency = 2,
		.target_residency = 2,
734 735
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
736 737 738
	{
		.name = "C6-AVN",
		.desc = "MWAIT 0x51",
739
		.flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED,
740 741
		.exit_latency = 15,
		.target_residency = 45,
742 743
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
744 745
	{
		.enter = NULL }
746
};
747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766
static struct cpuidle_state knl_cstates[] = {
	{
		.name = "C1-KNL",
		.desc = "MWAIT 0x00",
		.flags = MWAIT2flg(0x00),
		.exit_latency = 1,
		.target_residency = 2,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze },
	{
		.name = "C6-KNL",
		.desc = "MWAIT 0x10",
		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
		.exit_latency = 120,
		.target_residency = 500,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze },
	{
		.enter = NULL }
};
767

Len Brown's avatar
Len Brown committed
768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828
static struct cpuidle_state bxt_cstates[] = {
	{
		.name = "C1-BXT",
		.desc = "MWAIT 0x00",
		.flags = MWAIT2flg(0x00),
		.exit_latency = 2,
		.target_residency = 2,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.name = "C1E-BXT",
		.desc = "MWAIT 0x01",
		.flags = MWAIT2flg(0x01),
		.exit_latency = 10,
		.target_residency = 20,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.name = "C6-BXT",
		.desc = "MWAIT 0x20",
		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
		.exit_latency = 133,
		.target_residency = 133,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.name = "C7s-BXT",
		.desc = "MWAIT 0x31",
		.flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED,
		.exit_latency = 155,
		.target_residency = 155,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.name = "C8-BXT",
		.desc = "MWAIT 0x40",
		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
		.exit_latency = 1000,
		.target_residency = 1000,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.name = "C9-BXT",
		.desc = "MWAIT 0x50",
		.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
		.exit_latency = 2000,
		.target_residency = 2000,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.name = "C10-BXT",
		.desc = "MWAIT 0x60",
		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
		.exit_latency = 10000,
		.target_residency = 10000,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.enter = NULL }
};

Jacob Pan's avatar
Jacob Pan committed
829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857
static struct cpuidle_state dnv_cstates[] = {
	{
		.name = "C1-DNV",
		.desc = "MWAIT 0x00",
		.flags = MWAIT2flg(0x00),
		.exit_latency = 2,
		.target_residency = 2,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.name = "C1E-DNV",
		.desc = "MWAIT 0x01",
		.flags = MWAIT2flg(0x01),
		.exit_latency = 10,
		.target_residency = 20,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.name = "C6-DNV",
		.desc = "MWAIT 0x20",
		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
		.exit_latency = 50,
		.target_residency = 500,
		.enter = &intel_idle,
		.enter_freeze = intel_idle_freeze, },
	{
		.enter = NULL }
};

858 859 860
/**
 * intel_idle
 * @dev: cpuidle_device
861
 * @drv: cpuidle driver
862
 * @index: index of cpuidle state
863
 *
864
 * Must be called under local_irq_disable().
865
 */
866 867
static int intel_idle(struct cpuidle_device *dev,
		struct cpuidle_driver *drv, int index)
868 869
{
	unsigned long ecx = 1; /* break on interrupt flag */
870
	struct cpuidle_state *state = &drv->states[index];
871
	unsigned long eax = flg2MWAIT(state->flags);
872 873 874 875 876
	unsigned int cstate;
	int cpu = smp_processor_id();

	cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) + 1;

877
	/*
878 879
	 * leave_mm() to avoid costly and often unnecessary wakeups
	 * for flushing the user TLB's associated with the active mm.
880
	 */
881
	if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED)
882 883
		leave_mm(cpu);

884
	if (!(lapic_timer_reliable_states & (1 << (cstate))))
885
		tick_broadcast_enter();
886

887
	mwait_idle_with_hints(eax, ecx);
888 889

	if (!(lapic_timer_reliable_states & (1 << (cstate))))
890
		tick_broadcast_exit();
891

892
	return index;
893 894
}

895 896 897 898 899 900 901 902 903 904 905 906 907 908 909
/**
 * intel_idle_freeze - simplified "enter" callback routine for suspend-to-idle
 * @dev: cpuidle_device
 * @drv: cpuidle driver
 * @index: state index
 */
static void intel_idle_freeze(struct cpuidle_device *dev,
			     struct cpuidle_driver *drv, int index)
{
	unsigned long ecx = 1; /* break on interrupt flag */
	unsigned long eax = flg2MWAIT(drv->states[index].flags);

	mwait_idle_with_hints(eax, ecx);
}

910 911
static void __setup_broadcast_timer(void *arg)
{
912
	unsigned long on = (unsigned long)arg;
913

914 915 916 917
	if (on)
		tick_broadcast_enable();
	else
		tick_broadcast_disable();
918 919
}

920 921
static int cpu_hotplug_notify(struct notifier_block *n,
			      unsigned long action, void *hcpu)
922 923
{
	int hotcpu = (unsigned long)hcpu;
924
	struct cpuidle_device *dev;
925

926
	switch (action & ~CPU_TASKS_FROZEN) {
927
	case CPU_ONLINE:
928 929 930 931 932 933 934 935 936 937 938

		if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE)
			smp_call_function_single(hotcpu, __setup_broadcast_timer,
						 (void *)true, 1);

		/*
		 * Some systems can hotplug a cpu at runtime after
		 * the kernel has booted, we have to initialize the
		 * driver in this case
		 */
		dev = per_cpu_ptr(intel_idle_cpuidle_devices, hotcpu);
939 940 941 942 943
		if (dev->registered)
			break;

		if (intel_idle_cpu_init(hotcpu))
			return NOTIFY_BAD;
944

945 946 947 948 949
		break;
	}
	return NOTIFY_OK;
}

950 951
static struct notifier_block cpu_hotplug_notifier = {
	.notifier_call = cpu_hotplug_notify,
952 953
};

954 955 956 957 958
static void auto_demotion_disable(void *dummy)
{
	unsigned long long msr_bits;

	rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits);
959
	msr_bits &= ~(icpu->auto_demotion_disable_flags);
960 961
	wrmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits);
}
962 963 964 965 966 967 968 969
static void c1e_promotion_disable(void *dummy)
{
	unsigned long long msr_bits;

	rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
	msr_bits &= ~0x2;
	wrmsrl(MSR_IA32_POWER_CTL, msr_bits);
}
970

971 972 973
static const struct idle_cpu idle_cpu_nehalem = {
	.state_table = nehalem_cstates,
	.auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
974
	.disable_promotion_to_c1e = true,
975 976 977 978 979 980 981 982 983 984 985 986 987
};

static const struct idle_cpu idle_cpu_atom = {
	.state_table = atom_cstates,
};

static const struct idle_cpu idle_cpu_lincroft = {
	.state_table = atom_cstates,
	.auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE,
};

static const struct idle_cpu idle_cpu_snb = {
	.state_table = snb_cstates,
988
	.disable_promotion_to_c1e = true,
989 990
};

Len Brown's avatar
Len Brown committed
991 992 993
static const struct idle_cpu idle_cpu_byt = {
	.state_table = byt_cstates,
	.disable_promotion_to_c1e = true,
994
	.byt_auto_demotion_disable_flag = true,
Len Brown's avatar
Len Brown committed
995 996
};

997 998 999 1000 1001 1002
static const struct idle_cpu idle_cpu_cht = {
	.state_table = cht_cstates,
	.disable_promotion_to_c1e = true,
	.byt_auto_demotion_disable_flag = true,
};

Len Brown's avatar
Len Brown committed
1003 1004
static const struct idle_cpu idle_cpu_ivb = {
	.state_table = ivb_cstates,
1005
	.disable_promotion_to_c1e = true,
Len Brown's avatar
Len Brown committed
1006 1007
};

1008 1009 1010 1011 1012
static const struct idle_cpu idle_cpu_ivt = {
	.state_table = ivt_cstates,
	.disable_promotion_to_c1e = true,
};

Len Brown's avatar
Len Brown committed
1013 1014
static const struct idle_cpu idle_cpu_hsw = {
	.state_table = hsw_cstates,
1015
	.disable_promotion_to_c1e = true,
Len Brown's avatar
Len Brown committed
1016 1017
};

Len Brown's avatar
Len Brown committed
1018 1019 1020 1021 1022
static const struct idle_cpu idle_cpu_bdw = {
	.state_table = bdw_cstates,
	.disable_promotion_to_c1e = true,
};

1023 1024 1025 1026 1027
static const struct idle_cpu idle_cpu_skl = {
	.state_table = skl_cstates,
	.disable_promotion_to_c1e = true,
};

Len Brown's avatar
Len Brown committed
1028 1029 1030 1031
static const struct idle_cpu idle_cpu_skx = {
	.state_table = skx_cstates,
	.disable_promotion_to_c1e = true,
};
1032

1033 1034 1035 1036 1037
static const struct idle_cpu idle_cpu_avn = {
	.state_table = avn_cstates,
	.disable_promotion_to_c1e = true,
};

1038 1039 1040 1041
static const struct idle_cpu idle_cpu_knl = {
	.state_table = knl_cstates,
};

Len Brown's avatar
Len Brown committed
1042 1043 1044 1045 1046
static const struct idle_cpu idle_cpu_bxt = {
	.state_table = bxt_cstates,
	.disable_promotion_to_c1e = true,
};

Jacob Pan's avatar
Jacob Pan committed
1047 1048 1049 1050 1051
static const struct idle_cpu idle_cpu_dnv = {
	.state_table = dnv_cstates,
	.disable_promotion_to_c1e = true,
};

1052 1053 1054
#define ICPU(model, cpu) \
	{ X86_VENDOR_INTEL, 6, model, X86_FEATURE_MWAIT, (unsigned long)&cpu }

1055
static const struct x86_cpu_id intel_idle_ids[] __initconst = {
1056 1057
	ICPU(INTEL_FAM6_NEHALEM_EP,		idle_cpu_nehalem),
	ICPU(INTEL_FAM6_NEHALEM,		idle_cpu_nehalem),
1058
	ICPU(INTEL_FAM6_NEHALEM_G,		idle_cpu_nehalem),
1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087
	ICPU(INTEL_FAM6_WESTMERE,		idle_cpu_nehalem),
	ICPU(INTEL_FAM6_WESTMERE_EP,		idle_cpu_nehalem),
	ICPU(INTEL_FAM6_NEHALEM_EX,		idle_cpu_nehalem),
	ICPU(INTEL_FAM6_ATOM_PINEVIEW,		idle_cpu_atom),
	ICPU(INTEL_FAM6_ATOM_LINCROFT,		idle_cpu_lincroft),
	ICPU(INTEL_FAM6_WESTMERE_EX,		idle_cpu_nehalem),
	ICPU(INTEL_FAM6_SANDYBRIDGE,		idle_cpu_snb),
	ICPU(INTEL_FAM6_SANDYBRIDGE_X,		idle_cpu_snb),
	ICPU(INTEL_FAM6_ATOM_CEDARVIEW,		idle_cpu_atom),
	ICPU(INTEL_FAM6_ATOM_SILVERMONT1,	idle_cpu_byt),
	ICPU(INTEL_FAM6_ATOM_AIRMONT,		idle_cpu_cht),
	ICPU(INTEL_FAM6_IVYBRIDGE,		idle_cpu_ivb),
	ICPU(INTEL_FAM6_IVYBRIDGE_X,		idle_cpu_ivt),
	ICPU(INTEL_FAM6_HASWELL_CORE,		idle_cpu_hsw),
	ICPU(INTEL_FAM6_HASWELL_X,		idle_cpu_hsw),
	ICPU(INTEL_FAM6_HASWELL_ULT,		idle_cpu_hsw),
	ICPU(INTEL_FAM6_HASWELL_GT3E,		idle_cpu_hsw),
	ICPU(INTEL_FAM6_ATOM_SILVERMONT2,	idle_cpu_avn),
	ICPU(INTEL_FAM6_BROADWELL_CORE,		idle_cpu_bdw),
	ICPU(INTEL_FAM6_BROADWELL_GT3E,		idle_cpu_bdw),
	ICPU(INTEL_FAM6_BROADWELL_X,		idle_cpu_bdw),
	ICPU(INTEL_FAM6_BROADWELL_XEON_D,	idle_cpu_bdw),
	ICPU(INTEL_FAM6_SKYLAKE_MOBILE,		idle_cpu_skl),
	ICPU(INTEL_FAM6_SKYLAKE_DESKTOP,	idle_cpu_skl),
	ICPU(INTEL_FAM6_KABYLAKE_MOBILE,	idle_cpu_skl),
	ICPU(INTEL_FAM6_KABYLAKE_DESKTOP,	idle_cpu_skl),
	ICPU(INTEL_FAM6_SKYLAKE_X,		idle_cpu_skx),
	ICPU(INTEL_FAM6_XEON_PHI_KNL,		idle_cpu_knl),
	ICPU(INTEL_FAM6_ATOM_GOLDMONT,		idle_cpu_bxt),
Jacob Pan's avatar
Jacob Pan committed
1088
	ICPU(INTEL_FAM6_ATOM_DENVERTON,		idle_cpu_dnv),
1089 1090 1091
	{}
};

1092 1093 1094
/*
 * intel_idle_probe()
 */
1095
static int __init intel_idle_probe(void)
1096
{
1097
	unsigned int eax, ebx, ecx;
1098
	const struct x86_cpu_id *id;
1099 1100 1101 1102 1103 1104

	if (max_cstate == 0) {
		pr_debug(PREFIX "disabled\n");
		return -EPERM;
	}

1105 1106 1107 1108 1109 1110
	id = x86_match_cpu(intel_idle_ids);
	if (!id) {
		if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
		    boot_cpu_data.x86 == 6)
			pr_debug(PREFIX "does not run on family %d model %d\n",
				boot_cpu_data.x86, boot_cpu_data.x86_model);
1111
		return -ENODEV;
1112
	}
1113 1114 1115 1116

	if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
		return -ENODEV;

1117
	cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates);
1118 1119

	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
1120 1121
	    !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||
	    !mwait_substates)
1122 1123
			return -ENODEV;

1124
	pr_debug(PREFIX "MWAIT substates: 0x%x\n", mwait_substates);
1125

1126 1127
	icpu = (const struct idle_cpu *)id->driver_data;
	cpuidle_state_table = icpu->state_table;
1128 1129 1130 1131 1132 1133 1134 1135 1136

	pr_debug(PREFIX "v" INTEL_IDLE_VERSION
		" model 0x%X\n", boot_cpu_data.x86_model);

	return 0;
}

/*
 * intel_idle_cpuidle_devices_uninit()
1137
 * Unregisters the cpuidle devices.
1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148
 */
static void intel_idle_cpuidle_devices_uninit(void)
{
	int i;
	struct cpuidle_device *dev;

	for_each_online_cpu(i) {
		dev = per_cpu_ptr(intel_idle_cpuidle_devices, i);
		cpuidle_unregister_device(dev);
	}
}
1149 1150

/*
1151
 * ivt_idle_state_table_update(void)
1152
 *
1153
 * Tune IVT multi-socket targets
1154 1155
 * Assumption: num_sockets == (max_package_num + 1)
 */
1156
static void ivt_idle_state_table_update(void)
1157 1158
{
	/* IVT uses a different table for 1-2, 3-4, and > 4 sockets */
1159 1160 1161 1162 1163 1164 1165 1166 1167 1168
	int cpu, package_num, num_sockets = 1;

	for_each_online_cpu(cpu) {
		package_num = topology_physical_package_id(cpu);
		if (package_num + 1 > num_sockets) {
			num_sockets = package_num + 1;

			if (num_sockets > 4) {
				cpuidle_state_table = ivt_cstates_8s;
				return;
1169 1170
			}
		}
1171 1172 1173 1174 1175 1176 1177
	}

	if (num_sockets > 2)
		cpuidle_state_table = ivt_cstates_4s;

	/* else, 1 and 2 socket systems use default ivt_cstates */
}
Len Brown's avatar
Len Brown committed
1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189

/*
 * Translate IRTL (Interrupt Response Time Limit) MSR to usec
 */

static unsigned int irtl_ns_units[] = {
	1, 32, 1024, 32768, 1048576, 33554432, 0, 0 };

static unsigned long long irtl_2_usec(unsigned long long irtl)
{
	unsigned long long ns;

1190 1191 1192
	if (!irtl)
		return 0;