amd.c 22.9 KB
Newer Older
1
#include <linux/export.h>
Linus Torvalds's avatar
Linus Torvalds committed
2
#include <linux/bitops.h>
3
#include <linux/elf.h>
Linus Torvalds's avatar
Linus Torvalds committed
4
#include <linux/mm.h>
Yinghai Lu's avatar
Yinghai Lu committed
5

6
#include <linux/io.h>
7
#include <linux/sched.h>
8
#include <linux/random.h>
Linus Torvalds's avatar
Linus Torvalds committed
9
#include <asm/processor.h>
10
#include <asm/apic.h>
11
#include <asm/cpu.h>
Borislav Petkov's avatar
Borislav Petkov committed
12
#include <asm/smp.h>
13
#include <asm/pci-direct.h>
Linus Torvalds's avatar
Linus Torvalds committed
14

Yinghai Lu's avatar
Yinghai Lu committed
15
16
17
18
19
#ifdef CONFIG_X86_64
# include <asm/mmconfig.h>
# include <asm/cacheflush.h>
#endif

Linus Torvalds's avatar
Linus Torvalds committed
20
21
#include "cpu.h"

22
23
24
25
26
27
28
/*
 * nodes_per_socket: Stores the number of nodes per socket.
 * Refer to Fam15h Models 00-0fh BKDG - CPUID Fn8000_001E_ECX
 * Node Identifiers[10:8]
 */
static u32 nodes_per_socket = 1;

29
30
31
32
33
static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p)
{
	u32 gprs[8] = { 0 };
	int err;

34
35
	WARN_ONCE((boot_cpu_data.x86 != 0xf),
		  "%s should only be used on K8!\n", __func__);
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

	gprs[1] = msr;
	gprs[7] = 0x9c5a203a;

	err = rdmsr_safe_regs(gprs);

	*p = gprs[0] | ((u64)gprs[2] << 32);

	return err;
}

static inline int wrmsrl_amd_safe(unsigned msr, unsigned long long val)
{
	u32 gprs[8] = { 0 };

51
52
	WARN_ONCE((boot_cpu_data.x86 != 0xf),
		  "%s should only be used on K8!\n", __func__);
53
54
55
56
57
58
59
60
61

	gprs[0] = (u32)val;
	gprs[1] = msr;
	gprs[2] = val >> 32;
	gprs[7] = 0x9c5a203a;

	return wrmsr_safe_regs(gprs);
}

Linus Torvalds's avatar
Linus Torvalds committed
62
63
64
65
66
67
/*
 *	B step AMD K6 before B 9730xxxx have hardware bugs that can cause
 *	misexecution of code under Linux. Owners of such processors should
 *	contact AMD for precise details and a CPU swap.
 *
 *	See	http://www.multimania.com/poulot/k6bug.html
68
69
 *	and	section 2.6.2 of "AMD-K6 Processor Revision Guide - Model 6"
 *		(Publication # 21266  Issue Date: August 1998)
Linus Torvalds's avatar
Linus Torvalds committed
70
71
72
73
74
 *
 *	The following test is erm.. interesting. AMD neglected to up
 *	the chip setting when fixing the bug but they also tweaked some
 *	performance at the same time..
 */
75

76
77
extern __visible void vide(void);
__asm__(".globl vide\n\t.align 4\nvide: ret");
Linus Torvalds's avatar
Linus Torvalds committed
78

79
static void init_amd_k5(struct cpuinfo_x86 *c)
80
{
Borislav Petkov's avatar
Borislav Petkov committed
81
#ifdef CONFIG_X86_32
82
83
84
85
86
87
88
89
90
91
/*
 * General Systems BIOSen alias the cpu frequency registers
 * of the Elan at 0x000df000. Unfortuantly, one of the Linux
 * drivers subsequently pokes it, and changes the CPU speed.
 * Workaround : Remove the unneeded alias.
 */
#define CBAR		(0xfffc) /* Configuration Base Address  (32-bit) */
#define CBAR_ENB	(0x80000000)
#define CBAR_KEY	(0X000000CB)
	if (c->x86_model == 9 || c->x86_model == 10) {
92
93
		if (inl(CBAR) & CBAR_ENB)
			outl(0 | CBAR_KEY, CBAR);
94
	}
Borislav Petkov's avatar
Borislav Petkov committed
95
#endif
96
97
}

98
static void init_amd_k6(struct cpuinfo_x86 *c)
99
{
Borislav Petkov's avatar
Borislav Petkov committed
100
#ifdef CONFIG_X86_32
101
	u32 l, h;
102
	int mbytes = get_num_physpages() >> (20-PAGE_SHIFT);
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134

	if (c->x86_model < 6) {
		/* Based on AMD doc 20734R - June 2000 */
		if (c->x86_model == 0) {
			clear_cpu_cap(c, X86_FEATURE_APIC);
			set_cpu_cap(c, X86_FEATURE_PGE);
		}
		return;
	}

	if (c->x86_model == 6 && c->x86_mask == 1) {
		const int K6_BUG_LOOP = 1000000;
		int n;
		void (*f_vide)(void);
		unsigned long d, d2;

		printk(KERN_INFO "AMD K6 stepping B detected - ");

		/*
		 * It looks like AMD fixed the 2.6.2 bug and improved indirect
		 * calls at the same time.
		 */

		n = K6_BUG_LOOP;
		f_vide = vide;
		rdtscl(d);
		while (n--)
			f_vide();
		rdtscl(d2);
		d = d2-d;

		if (d > 20*K6_BUG_LOOP)
135
136
			printk(KERN_CONT
				"system stability may be impaired when more than 32 MB are used.\n");
137
		else
138
			printk(KERN_CONT "probably OK (after B9730xxxx).\n");
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
	}

	/* K6 with old style WHCR */
	if (c->x86_model < 8 ||
	   (c->x86_model == 8 && c->x86_mask < 8)) {
		/* We can only write allocate on the low 508Mb */
		if (mbytes > 508)
			mbytes = 508;

		rdmsr(MSR_K6_WHCR, l, h);
		if ((l&0x0000FFFF) == 0) {
			unsigned long flags;
			l = (1<<0)|((mbytes/4)<<1);
			local_irq_save(flags);
			wbinvd();
			wrmsr(MSR_K6_WHCR, l, h);
			local_irq_restore(flags);
			printk(KERN_INFO "Enabling old style K6 write allocation for %d Mb\n",
				mbytes);
		}
		return;
	}

	if ((c->x86_model == 8 && c->x86_mask > 7) ||
	     c->x86_model == 9 || c->x86_model == 13) {
		/* The more serious chips .. */

		if (mbytes > 4092)
			mbytes = 4092;

		rdmsr(MSR_K6_WHCR, l, h);
		if ((l&0xFFFF0000) == 0) {
			unsigned long flags;
			l = ((mbytes>>2)<<22)|(1<<16);
			local_irq_save(flags);
			wbinvd();
			wrmsr(MSR_K6_WHCR, l, h);
			local_irq_restore(flags);
			printk(KERN_INFO "Enabling new style K6 write allocation for %d Mb\n",
				mbytes);
		}

		return;
	}

	if (c->x86_model == 10) {
		/* AMD Geode LX is model 10 */
		/* placeholder for any needed mods */
		return;
	}
Borislav Petkov's avatar
Borislav Petkov committed
189
#endif
190
191
}

Borislav Petkov's avatar
Borislav Petkov committed
192
static void init_amd_k7(struct cpuinfo_x86 *c)
193
{
Borislav Petkov's avatar
Borislav Petkov committed
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
#ifdef CONFIG_X86_32
	u32 l, h;

	/*
	 * Bit 15 of Athlon specific MSR 15, needs to be 0
	 * to enable SSE on Palomino/Morgan/Barton CPU's.
	 * If the BIOS didn't enable it already, enable it here.
	 */
	if (c->x86_model >= 6 && c->x86_model <= 10) {
		if (!cpu_has(c, X86_FEATURE_XMM)) {
			printk(KERN_INFO "Enabling disabled K7/SSE Support.\n");
			msr_clear_bit(MSR_K7_HWCR, 15);
			set_cpu_cap(c, X86_FEATURE_XMM);
		}
	}

	/*
	 * It's been determined by AMD that Athlons since model 8 stepping 1
	 * are more robust with CLK_CTL set to 200xxxxx instead of 600xxxxx
	 * As per AMD technical note 27212 0.2
	 */
	if ((c->x86_model == 8 && c->x86_mask >= 1) || (c->x86_model > 8)) {
		rdmsr(MSR_K7_CLK_CTL, l, h);
		if ((l & 0xfff00000) != 0x20000000) {
			printk(KERN_INFO
			    "CPU: CLK_CTL MSR was %x. Reprogramming to %x\n",
					l, ((l & 0x000fffff)|0x20000000));
			wrmsr(MSR_K7_CLK_CTL, (l & 0x000fffff)|0x20000000, h);
		}
	}

	set_cpu_cap(c, X86_FEATURE_K7);

227
	/* calling is from identify_secondary_cpu() ? */
228
	if (!c->cpu_index)
229
230
231
232
233
234
235
236
237
		return;

	/*
	 * Certain Athlons might work (for various values of 'work') in SMP
	 * but they are not certified as MP capable.
	 */
	/* Athlon 660/661 is valid. */
	if ((c->x86_model == 6) && ((c->x86_mask == 0) ||
	    (c->x86_mask == 1)))
238
		return;
239
240
241

	/* Duron 670 is valid */
	if ((c->x86_model == 7) && (c->x86_mask == 0))
242
		return;
243
244
245
246
247
248
249
250
251
252
253

	/*
	 * Athlon 662, Duron 671, and Athlon >model 7 have capability
	 * bit. It's worth noting that the A5 stepping (662) of some
	 * Athlon XP's have the MP bit set.
	 * See http://www.heise.de/newsticker/data/jow-18.10.01-000 for
	 * more.
	 */
	if (((c->x86_model == 6) && (c->x86_mask >= 2)) ||
	    ((c->x86_model == 7) && (c->x86_mask >= 1)) ||
	     (c->x86_model > 7))
Borislav Petkov's avatar
Borislav Petkov committed
254
		if (cpu_has(c, X86_FEATURE_MP))
255
			return;
256
257
258
259
260
261
262
263

	/* If we get here, not a certified SMP capable AMD system. */

	/*
	 * Don't taint if we are running SMP kernel on a single non-MP
	 * approved Athlon
	 */
	WARN_ONCE(1, "WARNING: This combination of AMD"
264
		" processors is not suitable for SMP.\n");
265
	add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_NOW_UNRELIABLE);
266
#endif
Borislav Petkov's avatar
Borislav Petkov committed
267
}
268

269
#ifdef CONFIG_NUMA
270
271
272
273
/*
 * To workaround broken NUMA config.  Read the comment in
 * srat_detect_node().
 */
274
static int nearby_node(int apicid)
275
276
277
278
{
	int i, node;

	for (i = apicid - 1; i >= 0; i--) {
279
		node = __apicid_to_node[i];
280
281
282
283
		if (node != NUMA_NO_NODE && node_online(node))
			return node;
	}
	for (i = apicid + 1; i < MAX_LOCAL_APIC; i++) {
284
		node = __apicid_to_node[i];
285
286
287
288
289
290
		if (node != NUMA_NO_NODE && node_online(node))
			return node;
	}
	return first_node(node_online_map); /* Shouldn't happen */
}
#endif
291

292
/*
293
294
295
 * Fixup core topology information for
 * (1) AMD multi-node processors
 *     Assumption: Number of cores in each internal node is the same.
296
 * (2) AMD processors supporting compute units
297
298
 */
#ifdef CONFIG_X86_HT
299
static void amd_get_topology(struct cpuinfo_x86 *c)
300
{
301
	u32 cores_per_cu = 1;
302
	u8 node_id;
303
304
	int cpu = smp_processor_id();

305
	/* get information required for multi-node processors */
Andreas Herrmann's avatar
Andreas Herrmann committed
306
	if (cpu_has_topoext) {
307
308
309
		u32 eax, ebx, ecx, edx;

		cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
310
		nodes_per_socket = ((ecx >> 8) & 7) + 1;
311
312
313
314
315
		node_id = ecx & 7;

		/* get compute unit information */
		smp_num_siblings = ((ebx >> 8) & 3) + 1;
		c->compute_unit_id = ebx & 0xff;
316
		cores_per_cu += ((ebx >> 8) & 3);
317
	} else if (cpu_has(c, X86_FEATURE_NODEID_MSR)) {
318
319
		u64 value;

320
		rdmsrl(MSR_FAM10H_NODE_ID, value);
321
		nodes_per_socket = ((value >> 3) & 7) + 1;
322
323
		node_id = value & 7;
	} else
324
325
		return;

326
	/* fixup multi-node processor information */
327
	if (nodes_per_socket > 1) {
328
		u32 cores_per_node;
329
		u32 cus_per_node;
330

331
		set_cpu_cap(c, X86_FEATURE_AMD_DCM);
332
		cores_per_node = c->x86_max_cores / nodes_per_socket;
333
		cus_per_node = cores_per_node / cores_per_cu;
334

335
336
		/* store NodeID, use llc_shared_map to store sibling info */
		per_cpu(cpu_llc_id, cpu) = node_id;
337

338
		/* core id has to be in the [0 .. cores_per_node - 1] range */
339
340
		c->cpu_core_id %= cores_per_node;
		c->compute_unit_id %= cus_per_node;
341
	}
342
343
344
}
#endif

345
/*
346
 * On a AMD dual core setup the lower bits of the APIC id distinguish the cores.
347
348
 * Assumes number of cores is a power of two.
 */
349
static void amd_detect_cmp(struct cpuinfo_x86 *c)
350
351
352
{
#ifdef CONFIG_X86_HT
	unsigned bits;
353
	int cpu = smp_processor_id();
354
355
356
357
358
359

	bits = c->x86_coreid_bits;
	/* Low order bits define the core id (index of core in socket) */
	c->cpu_core_id = c->initial_apicid & ((1 << bits)-1);
	/* Convert the initial APIC ID into the socket ID */
	c->phys_proc_id = c->initial_apicid >> bits;
360
361
	/* use socket ID also for last level cache */
	per_cpu(cpu_llc_id, cpu) = c->phys_proc_id;
362
	amd_get_topology(c);
363
364
365
#endif
}

366
u16 amd_get_nb_id(int cpu)
367
{
368
	u16 id = 0;
369
370
371
372
373
374
375
#ifdef CONFIG_SMP
	id = per_cpu(cpu_llc_id, cpu);
#endif
	return id;
}
EXPORT_SYMBOL_GPL(amd_get_nb_id);

376
377
378
379
380
381
u32 amd_get_nodes_per_socket(void)
{
	return nodes_per_socket;
}
EXPORT_SYMBOL_GPL(amd_get_nodes_per_socket);

382
static void srat_detect_node(struct cpuinfo_x86 *c)
383
{
384
#ifdef CONFIG_NUMA
385
386
	int cpu = smp_processor_id();
	int node;
387
	unsigned apicid = c->apicid;
388

389
390
391
	node = numa_cpu_node(cpu);
	if (node == NUMA_NO_NODE)
		node = per_cpu(cpu_llc_id, cpu);
392

393
	/*
394
395
396
	 * On multi-fabric platform (e.g. Numascale NumaChip) a
	 * platform-specific handler needs to be called to fixup some
	 * IDs of the CPU.
397
	 */
398
	if (x86_cpuinit.fixup_cpu_id)
399
400
		x86_cpuinit.fixup_cpu_id(c, node);

401
	if (!node_online(node)) {
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
		/*
		 * Two possibilities here:
		 *
		 * - The CPU is missing memory and no node was created.  In
		 *   that case try picking one from a nearby CPU.
		 *
		 * - The APIC IDs differ from the HyperTransport node IDs
		 *   which the K8 northbridge parsing fills in.  Assume
		 *   they are all increased by a constant offset, but in
		 *   the same order as the HT nodeids.  If that doesn't
		 *   result in a usable node fall back to the path for the
		 *   previous case.
		 *
		 * This workaround operates directly on the mapping between
		 * APIC ID and NUMA node, assuming certain relationship
		 * between APIC ID, HT node ID and NUMA topology.  As going
		 * through CPU mapping may alter the outcome, directly
		 * access __apicid_to_node[].
		 */
421
422
423
		int ht_nodeid = c->initial_apicid;

		if (ht_nodeid >= 0 &&
424
425
		    __apicid_to_node[ht_nodeid] != NUMA_NO_NODE)
			node = __apicid_to_node[ht_nodeid];
426
427
428
429
430
431
432
433
		/* Pick a nearby node */
		if (!node_online(node))
			node = nearby_node(apicid);
	}
	numa_set_node(cpu, node);
#endif
}

434
static void early_init_amd_mc(struct cpuinfo_x86 *c)
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
{
#ifdef CONFIG_X86_HT
	unsigned bits, ecx;

	/* Multi core CPU? */
	if (c->extended_cpuid_level < 0x80000008)
		return;

	ecx = cpuid_ecx(0x80000008);

	c->x86_max_cores = (ecx & 0xff) + 1;

	/* CPU telling us the core id bits shift? */
	bits = (ecx >> 12) & 0xF;

	/* Otherwise recompute */
	if (bits == 0) {
		while ((1 << bits) < c->x86_max_cores)
			bits++;
	}

	c->x86_coreid_bits = bits;
#endif
}

460
static void bsp_init_amd(struct cpuinfo_x86 *c)
461
{
Borislav Petkov's avatar
Borislav Petkov committed
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481

#ifdef CONFIG_X86_64
	if (c->x86 >= 0xf) {
		unsigned long long tseg;

		/*
		 * Split up direct mapping around the TSEG SMM area.
		 * Don't do it for gbpages because there seems very little
		 * benefit in doing so.
		 */
		if (!rdmsrl_safe(MSR_K8_TSEG_ADDR, &tseg)) {
			unsigned long pfn = tseg >> PAGE_SHIFT;

			printk(KERN_DEBUG "tseg: %010llx\n", tseg);
			if (pfn_range_is_mapped(pfn, pfn + 1))
				set_memory_4k((unsigned long)__va(tseg), 1);
		}
	}
#endif

482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
	if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) {

		if (c->x86 > 0x10 ||
		    (c->x86 == 0x10 && c->x86_model >= 0x2)) {
			u64 val;

			rdmsrl(MSR_K7_HWCR, val);
			if (!(val & BIT(24)))
				printk(KERN_WARNING FW_BUG "TSC doesn't count "
					"with P0 frequency!\n");
		}
	}

	if (c->x86 == 0x15) {
		unsigned long upperbit;
		u32 cpuid, assoc;

		cpuid	 = cpuid_edx(0x80000005);
		assoc	 = cpuid >> 16 & 0xff;
		upperbit = ((cpuid >> 24) << 10) / assoc;

		va_align.mask	  = (upperbit - 1) & PAGE_MASK;
		va_align.flags    = ALIGN_VA_32 | ALIGN_VA_64;
505
506
507

		/* A random value per boot for bit slice [12:upper_bit) */
		va_align.bits = get_random_int() & va_align.mask;
508
509
510
	}
}

511
static void early_init_amd(struct cpuinfo_x86 *c)
512
{
513
514
	early_init_amd_mc(c);

515
516
517
518
519
	/*
	 * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate
	 * with P/T states and does not stop in deep C-states
	 */
	if (c->x86_power & (1 << 8)) {
520
		set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
521
		set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);
522
		if (!check_tsc_unstable())
523
			set_sched_clock_stable();
524
	}
525

526
527
528
#ifdef CONFIG_X86_64
	set_cpu_cap(c, X86_FEATURE_SYSCALL32);
#else
529
	/*  Set MTRR capability flag if appropriate */
530
531
532
533
534
	if (c->x86 == 5)
		if (c->x86_model == 13 || c->x86_model == 9 ||
		    (c->x86_model == 8 && c->x86_mask >= 8))
			set_cpu_cap(c, X86_FEATURE_K6_MTRR);
#endif
535
#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_PCI)
536
537
538
539
540
541
542
543
544
545
	/*
	 * ApicID can always be treated as an 8-bit value for AMD APIC versions
	 * >= 0x10, but even old K8s came out of reset with version 0x10. So, we
	 * can safely set X86_FEATURE_EXTD_APICID unconditionally for families
	 * after 16h.
	 */
	if (cpu_has_apic && c->x86 > 0x16) {
		set_cpu_cap(c, X86_FEATURE_EXTD_APICID);
	} else if (cpu_has_apic && c->x86 >= 0xf) {
		/* check CPU config space for extended APIC ID */
546
547
548
549
550
551
		unsigned int val;
		val = read_pci_config(0, 24, 0, 0x68);
		if ((val & ((1 << 17) | (1 << 18))) == ((1 << 17) | (1 << 18)))
			set_cpu_cap(c, X86_FEATURE_EXTD_APICID);
	}
#endif
552

553
554
555
556
557
558
559
	/*
	 * This is only needed to tell the kernel whether to use VMCALL
	 * and VMMCALL.  VMMCALL is never executed except under virt, so
	 * we can set it unconditionally.
	 */
	set_cpu_cap(c, X86_FEATURE_VMMCALL);

560
	/* F16h erratum 793, CVE-2013-6885 */
561
562
	if (c->x86 == 0x16 && c->x86_model <= 0xf)
		msr_set_bit(MSR_AMD64_LS_CFG, 15);
563
564
}

565
static const int amd_erratum_383[];
566
static const int amd_erratum_400[];
567
static bool cpu_has_amd_erratum(struct cpuinfo_x86 *cpu, const int *erratum);
568

Borislav Petkov's avatar
Borislav Petkov committed
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
static void init_amd_k8(struct cpuinfo_x86 *c)
{
	u32 level;
	u64 value;

	/* On C+ stepping K8 rep microcode works well for copy/memset */
	level = cpuid_eax(1);
	if ((level >= 0x0f48 && level < 0x0f50) || level >= 0x0f58)
		set_cpu_cap(c, X86_FEATURE_REP_GOOD);

	/*
	 * Some BIOSes incorrectly force this feature, but only K8 revision D
	 * (model = 0x14) and later actually support it.
	 * (AMD Erratum #110, docId: 25759).
	 */
	if (c->x86_model < 0x14 && cpu_has(c, X86_FEATURE_LAHF_LM)) {
		clear_cpu_cap(c, X86_FEATURE_LAHF_LM);
		if (!rdmsrl_amd_safe(0xc001100d, &value)) {
			value &= ~BIT_64(32);
			wrmsrl_amd_safe(0xc001100d, value);
		}
	}

	if (!c->x86_model_id[0])
		strcpy(c->x86_model_id, "Hammer");
594
595
596
597
598
599
600
601
602
603
604

#ifdef CONFIG_SMP
	/*
	 * Disable TLB flush filter by setting HWCR.FFDIS on K8
	 * bit 6 of msr C001_0015
	 *
	 * Errata 63 for SH-B3 steppings
	 * Errata 122 for all steppings (F+ have it disabled by default)
	 */
	msr_set_bit(MSR_K7_HWCR, 6);
#endif
Borislav Petkov's avatar
Borislav Petkov committed
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
}

static void init_amd_gh(struct cpuinfo_x86 *c)
{
#ifdef CONFIG_X86_64
	/* do this for boot cpu */
	if (c == &boot_cpu_data)
		check_enable_amd_mmconf_dmi();

	fam10h_check_enable_mmcfg();
#endif

	/*
	 * Disable GART TLB Walk Errors on Fam10h. We do this here because this
	 * is always needed when GART is enabled, even in a kernel which has no
	 * MCE support built in. BIOS should disable GartTlbWlk Errors already.
	 * If it doesn't, we do it here as suggested by the BKDG.
	 *
	 * Fixes: https://bugzilla.kernel.org/show_bug.cgi?id=33012
	 */
	msr_set_bit(MSR_AMD64_MCx_MASK(4), 10);

	/*
	 * On family 10h BIOS may not have properly enabled WC+ support, causing
	 * it to be converted to CD memtype. This may result in performance
	 * degradation for certain nested-paging guests. Prevent this conversion
	 * by clearing bit 24 in MSR_AMD64_BU_CFG2.
	 *
	 * NOTE: we want to use the _safe accessors so as not to #GP kvm
	 * guests on older kvm hosts.
	 */
	msr_clear_bit(MSR_AMD64_BU_CFG2, 24);

	if (cpu_has_amd_erratum(c, amd_erratum_383))
		set_cpu_bug(c, X86_BUG_AMD_TLB_MMATCH);
}

static void init_amd_bd(struct cpuinfo_x86 *c)
{
	u64 value;

	/* re-enable TopologyExtensions if switched off by BIOS */
	if ((c->x86_model >= 0x10) && (c->x86_model <= 0x1f) &&
	    !cpu_has(c, X86_FEATURE_TOPOEXT)) {

		if (msr_set_bit(0xc0011005, 54) > 0) {
			rdmsrl(0xc0011005, value);
			if (value & BIT_64(54)) {
				set_cpu_cap(c, X86_FEATURE_TOPOEXT);
				pr_info(FW_INFO "CPU: Re-enabling disabled Topology Extensions Support.\n");
			}
		}
	}

	/*
	 * The way access filter has a performance penalty on some workloads.
	 * Disable it on the affected CPUs.
	 */
	if ((c->x86_model >= 0x02) && (c->x86_model < 0x20)) {
		if (!rdmsrl_safe(0xc0011021, &value) && !(value & 0x1E)) {
			value |= 0x1E;
			wrmsrl_safe(0xc0011021, value);
		}
	}
}

671
static void init_amd(struct cpuinfo_x86 *c)
Linus Torvalds's avatar
Linus Torvalds committed
672
{
673
	u32 dummy;
674

675
676
	early_init_amd(c);

677
678
	/*
	 * Bit 31 in normal CPUID used for nonstandard 3DNow ID;
679
	 * 3DNow is IDd by bit 31 in extended CPUID (1*32+31) anyway
680
	 */
681
	clear_cpu_cap(c, 0*32+31);
682

683
	if (c->x86 >= 0x10)
684
		set_cpu_cap(c, X86_FEATURE_REP_GOOD);
685
686
687

	/* get apicid instead of initial apic id from cpuid */
	c->apicid = hard_smp_processor_id();
688
689
690
691

	/* K6s reports MCEs but don't actually have all the MSRs */
	if (c->x86 < 6)
		clear_cpu_cap(c, X86_FEATURE_MCE);
Borislav Petkov's avatar
Borislav Petkov committed
692
693
694
695
696
697
698
699
700

	switch (c->x86) {
	case 4:    init_amd_k5(c); break;
	case 5:    init_amd_k6(c); break;
	case 6:	   init_amd_k7(c); break;
	case 0xf:  init_amd_k8(c); break;
	case 0x10: init_amd_gh(c); break;
	case 0x15: init_amd_bd(c); break;
	}
701

702
	/* Enable workaround for FXSAVE leak */
703
	if (c->x86 >= 6)
704
		set_cpu_bug(c, X86_BUG_FXSAVE_LEAK);
Linus Torvalds's avatar
Linus Torvalds committed
705

706
	cpu_detect_cache_sizes(c);
707

708
	/* Multi core CPU? */
709
	if (c->extended_cpuid_level >= 0x80000008) {
710
		amd_detect_cmp(c);
711
712
		srat_detect_node(c);
	}
713

714
#ifdef CONFIG_X86_32
715
	detect_ht(c);
716
#endif
717

718
	init_amd_cacheinfo(c);
719

720
	if (c->x86 >= 0xf)
721
		set_cpu_cap(c, X86_FEATURE_K8);
722

723
724
	if (cpu_has_xmm2) {
		/* MFENCE stops RDTSC speculation */
725
		set_cpu_cap(c, X86_FEATURE_MFENCE_RDTSC);
726
	}
727

728
729
730
731
732
	/*
	 * Family 0x12 and above processors have APIC timer
	 * running in deep C states.
	 */
	if (c->x86 > 0x11)
733
		set_cpu_cap(c, X86_FEATURE_ARAT);
734

735
	if (cpu_has_amd_erratum(c, amd_erratum_400))
736
737
		set_cpu_bug(c, X86_BUG_AMD_APIC_C1E);

738
	rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy);
739
740
741
742
743

	/* 3DNow or LM implies PREFETCHW */
	if (!cpu_has(c, X86_FEATURE_3DNOWPREFETCH))
		if (cpu_has(c, X86_FEATURE_3DNOW) || cpu_has(c, X86_FEATURE_LM))
			set_cpu_cap(c, X86_FEATURE_3DNOWPREFETCH);
744
745
746

	/* AMD CPUs don't reset SS attributes on SYSRET */
	set_cpu_bug(c, X86_BUG_SYSRET_SS_ATTRS);
Linus Torvalds's avatar
Linus Torvalds committed
747
748
}

749
#ifdef CONFIG_X86_32
750
static unsigned int amd_size_cache(struct cpuinfo_x86 *c, unsigned int size)
Linus Torvalds's avatar
Linus Torvalds committed
751
752
753
{
	/* AMD errata T13 (order #21922) */
	if ((c->x86 == 6)) {
754
755
		/* Duron Rev A0 */
		if (c->x86_model == 3 && c->x86_mask == 0)
Linus Torvalds's avatar
Linus Torvalds committed
756
			size = 64;
757
		/* Tbird rev A1/A2 */
Linus Torvalds's avatar
Linus Torvalds committed
758
		if (c->x86_model == 4 &&
759
			(c->x86_mask == 0 || c->x86_mask == 1))
Linus Torvalds's avatar
Linus Torvalds committed
760
761
762
763
			size = 256;
	}
	return size;
}
764
#endif
Linus Torvalds's avatar
Linus Torvalds committed
765

766
static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
{
	u32 ebx, eax, ecx, edx;
	u16 mask = 0xfff;

	if (c->x86 < 0xf)
		return;

	if (c->extended_cpuid_level < 0x80000006)
		return;

	cpuid(0x80000006, &eax, &ebx, &ecx, &edx);

	tlb_lld_4k[ENTRIES] = (ebx >> 16) & mask;
	tlb_lli_4k[ENTRIES] = ebx & mask;

	/*
	 * K8 doesn't have 2M/4M entries in the L2 TLB so read out the L1 TLB
	 * characteristics from the CPUID function 0x80000005 instead.
	 */
	if (c->x86 == 0xf) {
		cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
		mask = 0xff;
	}

	/* Handle DTLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
792
793
794
	if (!((eax >> 16) & mask))
		tlb_lld_2m[ENTRIES] = (cpuid_eax(0x80000005) >> 16) & 0xff;
	else
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
		tlb_lld_2m[ENTRIES] = (eax >> 16) & mask;

	/* a 4M entry uses two 2M entries */
	tlb_lld_4m[ENTRIES] = tlb_lld_2m[ENTRIES] >> 1;

	/* Handle ITLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
	if (!(eax & mask)) {
		/* Erratum 658 */
		if (c->x86 == 0x15 && c->x86_model <= 0x1f) {
			tlb_lli_2m[ENTRIES] = 1024;
		} else {
			cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
			tlb_lli_2m[ENTRIES] = eax & 0xff;
		}
	} else
		tlb_lli_2m[ENTRIES] = eax & mask;

	tlb_lli_4m[ENTRIES] = tlb_lli_2m[ENTRIES] >> 1;
}

815
static const struct cpu_dev amd_cpu_dev = {
Linus Torvalds's avatar
Linus Torvalds committed
816
	.c_vendor	= "AMD",
817
	.c_ident	= { "AuthenticAMD" },
818
#ifdef CONFIG_X86_32
819
820
	.legacy_models = {
		{ .family = 4, .model_names =
Linus Torvalds's avatar
Linus Torvalds committed
821
822
823
		  {
			  [3] = "486 DX/2",
			  [7] = "486 DX/2-WB",
824
825
			  [8] = "486 DX/4",
			  [9] = "486 DX/4-WB",
Linus Torvalds's avatar
Linus Torvalds committed
826
			  [14] = "Am5x86-WT",
827
			  [15] = "Am5x86-WB"
Linus Torvalds's avatar
Linus Torvalds committed
828
829
830
		  }
		},
	},
831
	.legacy_cache_size = amd_size_cache,
832
#endif
833
	.c_early_init   = early_init_amd,
834
	.c_detect_tlb	= cpu_detect_tlb_amd,
835
	.c_bsp_init	= bsp_init_amd,
Linus Torvalds's avatar
Linus Torvalds committed
836
	.c_init		= init_amd,
Yinghai Lu's avatar
Yinghai Lu committed
837
	.c_x86_vendor	= X86_VENDOR_AMD,
Linus Torvalds's avatar
Linus Torvalds committed
838
839
};

Yinghai Lu's avatar
Yinghai Lu committed
840
cpu_dev_register(amd_cpu_dev);
841
842
843
844
845
846
847
848

/*
 * AMD errata checking
 *
 * Errata are defined as arrays of ints using the AMD_LEGACY_ERRATUM() or
 * AMD_OSVW_ERRATUM() macros. The latter is intended for newer errata that
 * have an OSVW id assigned, which it takes as first argument. Both take a
 * variable number of family-specific model-stepping ranges created by
849
 * AMD_MODEL_RANGE().
850
851
852
853
854
855
856
857
858
 *
 * Example:
 *
 * const int amd_erratum_319[] =
 *	AMD_LEGACY_ERRATUM(AMD_MODEL_RANGE(0x10, 0x2, 0x1, 0x4, 0x2),
 *			   AMD_MODEL_RANGE(0x10, 0x8, 0x0, 0x8, 0x0),
 *			   AMD_MODEL_RANGE(0x10, 0x9, 0x0, 0x9, 0x0));
 */

859
860
861
862
863
864
865
866
867
#define AMD_LEGACY_ERRATUM(...)		{ -1, __VA_ARGS__, 0 }
#define AMD_OSVW_ERRATUM(osvw_id, ...)	{ osvw_id, __VA_ARGS__, 0 }
#define AMD_MODEL_RANGE(f, m_start, s_start, m_end, s_end) \
	((f << 24) | (m_start << 16) | (s_start << 12) | (m_end << 4) | (s_end))
#define AMD_MODEL_RANGE_FAMILY(range)	(((range) >> 24) & 0xff)
#define AMD_MODEL_RANGE_START(range)	(((range) >> 12) & 0xfff)
#define AMD_MODEL_RANGE_END(range)	((range) & 0xfff)

static const int amd_erratum_400[] =
868
	AMD_OSVW_ERRATUM(1, AMD_MODEL_RANGE(0xf, 0x41, 0x2, 0xff, 0xf),
869
870
			    AMD_MODEL_RANGE(0x10, 0x2, 0x1, 0xff, 0xf));

871
static const int amd_erratum_383[] =
872
	AMD_OSVW_ERRATUM(3, AMD_MODEL_RANGE(0x10, 0, 0, 0xff, 0xf));
873

874
875

static bool cpu_has_amd_erratum(struct cpuinfo_x86 *cpu, const int *erratum)
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
{
	int osvw_id = *erratum++;
	u32 range;
	u32 ms;

	if (osvw_id >= 0 && osvw_id < 65536 &&
	    cpu_has(cpu, X86_FEATURE_OSVW)) {
		u64 osvw_len;

		rdmsrl(MSR_AMD64_OSVW_ID_LENGTH, osvw_len);
		if (osvw_id < osvw_len) {
			u64 osvw_bits;

			rdmsrl(MSR_AMD64_OSVW_STATUS + (osvw_id >> 6),
			    osvw_bits);
			return osvw_bits & (1ULL << (osvw_id & 0x3f));
		}
	}

	/* OSVW unavailable or ID unknown, match family-model-stepping range */
896
	ms = (cpu->x86_model << 4) | cpu->x86_mask;
897
898
899
900
901
902
903
904
	while ((range = *erratum++))
		if ((cpu->x86 == AMD_MODEL_RANGE_FAMILY(range)) &&
		    (ms >= AMD_MODEL_RANGE_START(range)) &&
		    (ms <= AMD_MODEL_RANGE_END(range)))
			return true;

	return false;
}
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923

void set_dr_addr_mask(unsigned long mask, int dr)
{
	if (!cpu_has_bpext)
		return;

	switch (dr) {
	case 0:
		wrmsr(MSR_F16H_DR0_ADDR_MASK, mask, 0);
		break;
	case 1:
	case 2:
	case 3:
		wrmsr(MSR_F16H_DR1_ADDR_MASK - 1 + dr, mask, 0);
		break;
	default:
		break;
	}
}