c-r4k.c 32.5 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
 * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org)
 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
 */
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/mm.h>
#include <linux/bitops.h>

#include <asm/bcache.h>
#include <asm/bootinfo.h>
18
#include <asm/cache.h>
Linus Torvalds's avatar
Linus Torvalds committed
19
20
21
22
23
24
25
26
27
28
#include <asm/cacheops.h>
#include <asm/cpu.h>
#include <asm/cpu-features.h>
#include <asm/io.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/r4kcache.h>
#include <asm/system.h>
#include <asm/mmu_context.h>
#include <asm/war.h>
29
#include <asm/cacheflush.h> /* for run_uncached() */
Linus Torvalds's avatar
Linus Torvalds committed
30

31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

/*
 * Special Variant of smp_call_function for use by cache functions:
 *
 *  o No return value
 *  o collapses to normal function call on UP kernels
 *  o collapses to normal function call on systems with a single shared
 *    primary cache.
 */
static inline void r4k_on_each_cpu(void (*func) (void *info), void *info,
                                   int retry, int wait)
{
	preempt_disable();

#if !defined(CONFIG_MIPS_MT_SMP) && !defined(CONFIG_MIPS_MT_SMTC)
	smp_call_function(func, info, retry, wait);
#endif
	func(info);
	preempt_enable();
}

52
53
54
55
56
57
/*
 * Must die.
 */
static unsigned long icache_size __read_mostly;
static unsigned long dcache_size __read_mostly;
static unsigned long scache_size __read_mostly;
Linus Torvalds's avatar
Linus Torvalds committed
58
59
60
61

/*
 * Dummy cache handling routines for machines without boardcaches
 */
62
static void cache_noop(void) {}
Linus Torvalds's avatar
Linus Torvalds committed
63
64

static struct bcache_ops no_sc_ops = {
65
66
67
68
	.bc_enable = (void *)cache_noop,
	.bc_disable = (void *)cache_noop,
	.bc_wback_inv = (void *)cache_noop,
	.bc_inv = (void *)cache_noop
Linus Torvalds's avatar
Linus Torvalds committed
69
70
71
72
};

struct bcache_ops *bcops = &no_sc_ops;

73
74
#define cpu_is_r4600_v1_x()	((read_c0_prid() & 0xfffffff0) == 0x00002010)
#define cpu_is_r4600_v2_x()	((read_c0_prid() & 0xfffffff0) == 0x00002020)
Linus Torvalds's avatar
Linus Torvalds committed
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

#define R4600_HIT_CACHEOP_WAR_IMPL					\
do {									\
	if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())		\
		*(volatile unsigned long *)CKSEG1;			\
	if (R4600_V1_HIT_CACHEOP_WAR)					\
		__asm__ __volatile__("nop;nop;nop;nop");		\
} while (0)

static void (*r4k_blast_dcache_page)(unsigned long addr);

static inline void r4k_blast_dcache_page_dc32(unsigned long addr)
{
	R4600_HIT_CACHEOP_WAR_IMPL;
	blast_dcache32_page(addr);
}

92
static void __init r4k_blast_dcache_page_setup(void)
Linus Torvalds's avatar
Linus Torvalds committed
93
94
95
{
	unsigned long  dc_lsize = cpu_dcache_line_size();

96
97
98
	if (dc_lsize == 0)
		r4k_blast_dcache_page = (void *)cache_noop;
	else if (dc_lsize == 16)
Linus Torvalds's avatar
Linus Torvalds committed
99
100
101
102
103
104
105
		r4k_blast_dcache_page = blast_dcache16_page;
	else if (dc_lsize == 32)
		r4k_blast_dcache_page = r4k_blast_dcache_page_dc32;
}

static void (* r4k_blast_dcache_page_indexed)(unsigned long addr);

106
static void __init r4k_blast_dcache_page_indexed_setup(void)
Linus Torvalds's avatar
Linus Torvalds committed
107
108
109
{
	unsigned long dc_lsize = cpu_dcache_line_size();

110
111
112
	if (dc_lsize == 0)
		r4k_blast_dcache_page_indexed = (void *)cache_noop;
	else if (dc_lsize == 16)
Linus Torvalds's avatar
Linus Torvalds committed
113
114
115
116
117
118
119
		r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed;
	else if (dc_lsize == 32)
		r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed;
}

static void (* r4k_blast_dcache)(void);

120
static void __init r4k_blast_dcache_setup(void)
Linus Torvalds's avatar
Linus Torvalds committed
121
122
123
{
	unsigned long dc_lsize = cpu_dcache_line_size();

124
125
126
	if (dc_lsize == 0)
		r4k_blast_dcache = (void *)cache_noop;
	else if (dc_lsize == 16)
Linus Torvalds's avatar
Linus Torvalds committed
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
		r4k_blast_dcache = blast_dcache16;
	else if (dc_lsize == 32)
		r4k_blast_dcache = blast_dcache32;
}

/* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */
#define JUMP_TO_ALIGN(order) \
	__asm__ __volatile__( \
		"b\t1f\n\t" \
		".align\t" #order "\n\t" \
		"1:\n\t" \
		)
#define CACHE32_UNROLL32_ALIGN	JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */
#define CACHE32_UNROLL32_ALIGN2	JUMP_TO_ALIGN(11)

static inline void blast_r4600_v1_icache32(void)
{
	unsigned long flags;

	local_irq_save(flags);
	blast_icache32();
	local_irq_restore(flags);
}

static inline void tx49_blast_icache32(void)
{
	unsigned long start = INDEX_BASE;
	unsigned long end = start + current_cpu_data.icache.waysize;
	unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
	unsigned long ws_end = current_cpu_data.icache.ways <<
	                       current_cpu_data.icache.waybit;
	unsigned long ws, addr;

	CACHE32_UNROLL32_ALIGN2;
	/* I'm in even chunk.  blast odd chunks */
162
163
	for (ws = 0; ws < ws_end; ws += ws_inc)
		for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
Linus Torvalds's avatar
Linus Torvalds committed
164
165
166
			cache32_unroll32(addr|ws,Index_Invalidate_I);
	CACHE32_UNROLL32_ALIGN;
	/* I'm in odd chunk.  blast even chunks */
167
168
	for (ws = 0; ws < ws_end; ws += ws_inc)
		for (addr = start; addr < end; addr += 0x400 * 2)
Linus Torvalds's avatar
Linus Torvalds committed
169
170
171
172
173
174
175
176
177
178
179
180
181
182
			cache32_unroll32(addr|ws,Index_Invalidate_I);
}

static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page)
{
	unsigned long flags;

	local_irq_save(flags);
	blast_icache32_page_indexed(page);
	local_irq_restore(flags);
}

static inline void tx49_blast_icache32_page_indexed(unsigned long page)
{
183
184
	unsigned long indexmask = current_cpu_data.icache.waysize - 1;
	unsigned long start = INDEX_BASE + (page & indexmask);
Linus Torvalds's avatar
Linus Torvalds committed
185
186
187
188
189
190
191
192
	unsigned long end = start + PAGE_SIZE;
	unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
	unsigned long ws_end = current_cpu_data.icache.ways <<
	                       current_cpu_data.icache.waybit;
	unsigned long ws, addr;

	CACHE32_UNROLL32_ALIGN2;
	/* I'm in even chunk.  blast odd chunks */
193
194
	for (ws = 0; ws < ws_end; ws += ws_inc)
		for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
Linus Torvalds's avatar
Linus Torvalds committed
195
196
197
			cache32_unroll32(addr|ws,Index_Invalidate_I);
	CACHE32_UNROLL32_ALIGN;
	/* I'm in odd chunk.  blast even chunks */
198
199
	for (ws = 0; ws < ws_end; ws += ws_inc)
		for (addr = start; addr < end; addr += 0x400 * 2)
Linus Torvalds's avatar
Linus Torvalds committed
200
201
202
203
204
			cache32_unroll32(addr|ws,Index_Invalidate_I);
}

static void (* r4k_blast_icache_page)(unsigned long addr);

205
static void __init r4k_blast_icache_page_setup(void)
Linus Torvalds's avatar
Linus Torvalds committed
206
207
208
{
	unsigned long ic_lsize = cpu_icache_line_size();

209
210
211
	if (ic_lsize == 0)
		r4k_blast_icache_page = (void *)cache_noop;
	else if (ic_lsize == 16)
Linus Torvalds's avatar
Linus Torvalds committed
212
213
214
215
216
217
218
219
220
221
		r4k_blast_icache_page = blast_icache16_page;
	else if (ic_lsize == 32)
		r4k_blast_icache_page = blast_icache32_page;
	else if (ic_lsize == 64)
		r4k_blast_icache_page = blast_icache64_page;
}


static void (* r4k_blast_icache_page_indexed)(unsigned long addr);

222
static void __init r4k_blast_icache_page_indexed_setup(void)
Linus Torvalds's avatar
Linus Torvalds committed
223
224
225
{
	unsigned long ic_lsize = cpu_icache_line_size();

226
227
228
	if (ic_lsize == 0)
		r4k_blast_icache_page_indexed = (void *)cache_noop;
	else if (ic_lsize == 16)
Linus Torvalds's avatar
Linus Torvalds committed
229
230
		r4k_blast_icache_page_indexed = blast_icache16_page_indexed;
	else if (ic_lsize == 32) {
Thiemo Seufer's avatar
Thiemo Seufer committed
231
		if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
Linus Torvalds's avatar
Linus Torvalds committed
232
233
			r4k_blast_icache_page_indexed =
				blast_icache32_r4600_v1_page_indexed;
Thiemo Seufer's avatar
Thiemo Seufer committed
234
235
236
		else if (TX49XX_ICACHE_INDEX_INV_WAR)
			r4k_blast_icache_page_indexed =
				tx49_blast_icache32_page_indexed;
Linus Torvalds's avatar
Linus Torvalds committed
237
238
239
240
241
242
243
244
245
		else
			r4k_blast_icache_page_indexed =
				blast_icache32_page_indexed;
	} else if (ic_lsize == 64)
		r4k_blast_icache_page_indexed = blast_icache64_page_indexed;
}

static void (* r4k_blast_icache)(void);

246
static void __init r4k_blast_icache_setup(void)
Linus Torvalds's avatar
Linus Torvalds committed
247
248
249
{
	unsigned long ic_lsize = cpu_icache_line_size();

250
251
252
	if (ic_lsize == 0)
		r4k_blast_icache = (void *)cache_noop;
	else if (ic_lsize == 16)
Linus Torvalds's avatar
Linus Torvalds committed
253
254
255
256
257
258
259
260
261
262
263
264
265
266
		r4k_blast_icache = blast_icache16;
	else if (ic_lsize == 32) {
		if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
			r4k_blast_icache = blast_r4600_v1_icache32;
		else if (TX49XX_ICACHE_INDEX_INV_WAR)
			r4k_blast_icache = tx49_blast_icache32;
		else
			r4k_blast_icache = blast_icache32;
	} else if (ic_lsize == 64)
		r4k_blast_icache = blast_icache64;
}

static void (* r4k_blast_scache_page)(unsigned long addr);

267
static void __init r4k_blast_scache_page_setup(void)
Linus Torvalds's avatar
Linus Torvalds committed
268
269
270
{
	unsigned long sc_lsize = cpu_scache_line_size();

271
	if (scache_size == 0)
272
		r4k_blast_scache_page = (void *)cache_noop;
273
	else if (sc_lsize == 16)
Linus Torvalds's avatar
Linus Torvalds committed
274
275
276
277
278
279
280
281
282
283
284
		r4k_blast_scache_page = blast_scache16_page;
	else if (sc_lsize == 32)
		r4k_blast_scache_page = blast_scache32_page;
	else if (sc_lsize == 64)
		r4k_blast_scache_page = blast_scache64_page;
	else if (sc_lsize == 128)
		r4k_blast_scache_page = blast_scache128_page;
}

static void (* r4k_blast_scache_page_indexed)(unsigned long addr);

285
static void __init r4k_blast_scache_page_indexed_setup(void)
Linus Torvalds's avatar
Linus Torvalds committed
286
287
288
{
	unsigned long sc_lsize = cpu_scache_line_size();

289
	if (scache_size == 0)
290
		r4k_blast_scache_page_indexed = (void *)cache_noop;
291
	else if (sc_lsize == 16)
Linus Torvalds's avatar
Linus Torvalds committed
292
293
294
295
296
297
298
299
300
301
302
		r4k_blast_scache_page_indexed = blast_scache16_page_indexed;
	else if (sc_lsize == 32)
		r4k_blast_scache_page_indexed = blast_scache32_page_indexed;
	else if (sc_lsize == 64)
		r4k_blast_scache_page_indexed = blast_scache64_page_indexed;
	else if (sc_lsize == 128)
		r4k_blast_scache_page_indexed = blast_scache128_page_indexed;
}

static void (* r4k_blast_scache)(void);

303
static void __init r4k_blast_scache_setup(void)
Linus Torvalds's avatar
Linus Torvalds committed
304
305
306
{
	unsigned long sc_lsize = cpu_scache_line_size();

307
	if (scache_size == 0)
308
		r4k_blast_scache = (void *)cache_noop;
309
	else if (sc_lsize == 16)
Linus Torvalds's avatar
Linus Torvalds committed
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
		r4k_blast_scache = blast_scache16;
	else if (sc_lsize == 32)
		r4k_blast_scache = blast_scache32;
	else if (sc_lsize == 64)
		r4k_blast_scache = blast_scache64;
	else if (sc_lsize == 128)
		r4k_blast_scache = blast_scache128;
}

/*
 * This is former mm's flush_cache_all() which really should be
 * flush_cache_vunmap these days ...
 */
static inline void local_r4k_flush_cache_all(void * args)
{
	r4k_blast_dcache();
}

static void r4k_flush_cache_all(void)
{
	if (!cpu_has_dc_aliases)
		return;

333
	r4k_on_each_cpu(local_r4k_flush_cache_all, NULL, 1, 1);
Linus Torvalds's avatar
Linus Torvalds committed
334
335
336
337
338
339
340
341
342
343
344
345
346
347
}

static inline void local_r4k___flush_cache_all(void * args)
{
	r4k_blast_dcache();
	r4k_blast_icache();

	switch (current_cpu_data.cputype) {
	case CPU_R4000SC:
	case CPU_R4000MC:
	case CPU_R4400SC:
	case CPU_R4400MC:
	case CPU_R10000:
	case CPU_R12000:
Kumba's avatar
Kumba committed
348
	case CPU_R14000:
Linus Torvalds's avatar
Linus Torvalds committed
349
350
351
352
353
354
		r4k_blast_scache();
	}
}

static void r4k___flush_cache_all(void)
{
355
	r4k_on_each_cpu(local_r4k___flush_cache_all, NULL, 1, 1);
Linus Torvalds's avatar
Linus Torvalds committed
356
357
358
359
360
361
362
363
364
}

static inline void local_r4k_flush_cache_range(void * args)
{
	struct vm_area_struct *vma = args;

	if (!(cpu_context(smp_processor_id(), vma->vm_mm)))
		return;

365
	r4k_blast_dcache();
Linus Torvalds's avatar
Linus Torvalds committed
366
367
368
369
370
}

static void r4k_flush_cache_range(struct vm_area_struct *vma,
	unsigned long start, unsigned long end)
{
371
372
373
	if (!cpu_has_dc_aliases)
		return;

374
	r4k_on_each_cpu(local_r4k_flush_cache_range, vma, 1, 1);
Linus Torvalds's avatar
Linus Torvalds committed
375
376
377
378
379
380
381
382
383
384
385
386
}

static inline void local_r4k_flush_cache_mm(void * args)
{
	struct mm_struct *mm = args;

	if (!cpu_context(smp_processor_id(), mm))
		return;

	/*
	 * Kludge alert.  For obscure reasons R4000SC and R4400SC go nuts if we
	 * only flush the primary caches but R10000 and R12000 behave sane ...
387
388
	 * R4000SC and R4400SC indexed S-cache ops also invalidate primary
	 * caches, so we can bail out early.
Linus Torvalds's avatar
Linus Torvalds committed
389
390
391
392
	 */
	if (current_cpu_data.cputype == CPU_R4000SC ||
	    current_cpu_data.cputype == CPU_R4000MC ||
	    current_cpu_data.cputype == CPU_R4400SC ||
393
	    current_cpu_data.cputype == CPU_R4400MC) {
Linus Torvalds's avatar
Linus Torvalds committed
394
		r4k_blast_scache();
395
396
397
398
		return;
	}

	r4k_blast_dcache();
Linus Torvalds's avatar
Linus Torvalds committed
399
400
401
402
403
404
405
}

static void r4k_flush_cache_mm(struct mm_struct *mm)
{
	if (!cpu_has_dc_aliases)
		return;

406
	r4k_on_each_cpu(local_r4k_flush_cache_mm, mm, 1, 1);
Linus Torvalds's avatar
Linus Torvalds committed
407
408
409
410
}

struct flush_cache_page_args {
	struct vm_area_struct *vma;
411
	unsigned long addr;
412
	unsigned long pfn;
Linus Torvalds's avatar
Linus Torvalds committed
413
414
415
416
417
418
};

static inline void local_r4k_flush_cache_page(void *args)
{
	struct flush_cache_page_args *fcp_args = args;
	struct vm_area_struct *vma = fcp_args->vma;
419
	unsigned long addr = fcp_args->addr;
420
	unsigned long paddr = fcp_args->pfn << PAGE_SHIFT;
Linus Torvalds's avatar
Linus Torvalds committed
421
422
423
	int exec = vma->vm_flags & VM_EXEC;
	struct mm_struct *mm = vma->vm_mm;
	pgd_t *pgdp;
424
	pud_t *pudp;
Linus Torvalds's avatar
Linus Torvalds committed
425
426
427
	pmd_t *pmdp;
	pte_t *ptep;

428
429
430
431
	/*
	 * If ownes no valid ASID yet, cannot possibly have gotten
	 * this page into the cache.
	 */
Thiemo Seufer's avatar
Thiemo Seufer committed
432
	if (cpu_context(smp_processor_id(), mm) == 0)
433
434
		return;

435
436
437
438
439
	addr &= PAGE_MASK;
	pgdp = pgd_offset(mm, addr);
	pudp = pud_offset(pgdp, addr);
	pmdp = pmd_offset(pudp, addr);
	ptep = pte_offset(pmdp, addr);
Linus Torvalds's avatar
Linus Torvalds committed
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455

	/*
	 * If the page isn't marked valid, the page cannot possibly be
	 * in the cache.
	 */
	if (!(pte_val(*ptep) & _PAGE_PRESENT))
		return;

	/*
	 * Doing flushes for another ASID than the current one is
	 * too difficult since stupid R4k caches do a TLB translation
	 * for every cache flush operation.  So we do indexed flushes
	 * in that case, which doesn't overly flush the cache too much.
	 */
	if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) {
		if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
456
			r4k_blast_dcache_page(addr);
Linus Torvalds's avatar
Linus Torvalds committed
457
			if (exec && !cpu_icache_snoops_remote_store)
458
				r4k_blast_scache_page(addr);
Linus Torvalds's avatar
Linus Torvalds committed
459
460
		}
		if (exec)
461
			r4k_blast_icache_page(addr);
Linus Torvalds's avatar
Linus Torvalds committed
462
463
464
465
466
467
468
469
470

		return;
	}

	/*
	 * Do indexed flush, too much work to get the (possible) TLB refills
	 * to work correctly.
	 */
	if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
471
472
473
474
475
		r4k_blast_dcache_page_indexed(cpu_has_pindexed_dcache ?
					      paddr : addr);
		if (exec && !cpu_icache_snoops_remote_store) {
			r4k_blast_scache_page_indexed(paddr);
		}
Linus Torvalds's avatar
Linus Torvalds committed
476
477
	}
	if (exec) {
478
		if (cpu_has_vtag_icache && mm == current->active_mm) {
Linus Torvalds's avatar
Linus Torvalds committed
479
480
			int cpu = smp_processor_id();

Thiemo Seufer's avatar
Thiemo Seufer committed
481
482
			if (cpu_context(cpu, mm) != 0)
				drop_mmu_context(mm, cpu);
Linus Torvalds's avatar
Linus Torvalds committed
483
		} else
484
			r4k_blast_icache_page_indexed(addr);
Linus Torvalds's avatar
Linus Torvalds committed
485
486
487
	}
}

488
489
static void r4k_flush_cache_page(struct vm_area_struct *vma,
	unsigned long addr, unsigned long pfn)
Linus Torvalds's avatar
Linus Torvalds committed
490
491
492
493
{
	struct flush_cache_page_args args;

	args.vma = vma;
494
	args.addr = addr;
495
	args.pfn = pfn;
Linus Torvalds's avatar
Linus Torvalds committed
496

497
	r4k_on_each_cpu(local_r4k_flush_cache_page, &args, 1, 1);
Linus Torvalds's avatar
Linus Torvalds committed
498
499
500
501
502
503
504
505
506
}

static inline void local_r4k_flush_data_cache_page(void * addr)
{
	r4k_blast_dcache_page((unsigned long) addr);
}

static void r4k_flush_data_cache_page(unsigned long addr)
{
507
	r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr, 1, 1);
Linus Torvalds's avatar
Linus Torvalds committed
508
509
510
}

struct flush_icache_range_args {
511
512
	unsigned long start;
	unsigned long end;
Linus Torvalds's avatar
Linus Torvalds committed
513
514
515
516
517
518
519
520
521
};

static inline void local_r4k_flush_icache_range(void *args)
{
	struct flush_icache_range_args *fir_args = args;
	unsigned long start = fir_args->start;
	unsigned long end = fir_args->end;

	if (!cpu_has_ic_fills_f_dc) {
522
		if (end - start >= dcache_size) {
Linus Torvalds's avatar
Linus Torvalds committed
523
524
			r4k_blast_dcache();
		} else {
525
			R4600_HIT_CACHEOP_WAR_IMPL;
526
			protected_blast_dcache_range(start, end);
Linus Torvalds's avatar
Linus Torvalds committed
527
528
		}

529
		if (!cpu_icache_snoops_remote_store && scache_size) {
530
			if (end - start > scache_size)
Linus Torvalds's avatar
Linus Torvalds committed
531
				r4k_blast_scache();
532
533
			else
				protected_blast_scache_range(start, end);
Linus Torvalds's avatar
Linus Torvalds committed
534
535
536
537
538
		}
	}

	if (end - start > icache_size)
		r4k_blast_icache();
539
540
	else
		protected_blast_icache_range(start, end);
Linus Torvalds's avatar
Linus Torvalds committed
541
542
}

543
static void r4k_flush_icache_range(unsigned long start, unsigned long end)
Linus Torvalds's avatar
Linus Torvalds committed
544
545
546
547
548
549
{
	struct flush_icache_range_args args;

	args.start = start;
	args.end = end;

550
	r4k_on_each_cpu(local_r4k_flush_icache_range, &args, 1, 1);
551
	instruction_hazard();
Linus Torvalds's avatar
Linus Torvalds committed
552
553
554
555
556
557
558
559
560
}

#ifdef CONFIG_DMA_NONCOHERENT

static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size)
{
	/* Catch bad driver code */
	BUG_ON(size == 0);

561
	if (cpu_has_inclusive_pcaches) {
562
		if (size >= scache_size)
Linus Torvalds's avatar
Linus Torvalds committed
563
			r4k_blast_scache();
564
565
		else
			blast_scache_range(addr, addr + size);
Linus Torvalds's avatar
Linus Torvalds committed
566
567
568
569
570
571
572
573
574
575
576
577
		return;
	}

	/*
	 * Either no secondary cache or the available caches don't have the
	 * subset property so we have to flush the primary caches
	 * explicitly
	 */
	if (size >= dcache_size) {
		r4k_blast_dcache();
	} else {
		R4600_HIT_CACHEOP_WAR_IMPL;
578
		blast_dcache_range(addr, addr + size);
Linus Torvalds's avatar
Linus Torvalds committed
579
580
581
582
583
584
585
586
587
588
	}

	bc_wback_inv(addr, size);
}

static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
{
	/* Catch bad driver code */
	BUG_ON(size == 0);

589
	if (cpu_has_inclusive_pcaches) {
590
		if (size >= scache_size)
Linus Torvalds's avatar
Linus Torvalds committed
591
			r4k_blast_scache();
592
593
		else
			blast_scache_range(addr, addr + size);
Linus Torvalds's avatar
Linus Torvalds committed
594
595
596
597
598
599
600
		return;
	}

	if (size >= dcache_size) {
		r4k_blast_dcache();
	} else {
		R4600_HIT_CACHEOP_WAR_IMPL;
601
		blast_dcache_range(addr, addr + size);
Linus Torvalds's avatar
Linus Torvalds committed
602
603
604
605
606
607
608
609
610
611
612
613
614
	}

	bc_inv(addr, size);
}
#endif /* CONFIG_DMA_NONCOHERENT */

/*
 * While we're protected against bad userland addresses we don't care
 * very much about what happens in that case.  Usually a segmentation
 * fault will dump the process later on anyway ...
 */
static void local_r4k_flush_cache_sigtramp(void * arg)
{
Thiemo Seufer's avatar
Thiemo Seufer committed
615
616
617
	unsigned long ic_lsize = cpu_icache_line_size();
	unsigned long dc_lsize = cpu_dcache_line_size();
	unsigned long sc_lsize = cpu_scache_line_size();
Linus Torvalds's avatar
Linus Torvalds committed
618
619
620
	unsigned long addr = (unsigned long) arg;

	R4600_HIT_CACHEOP_WAR_IMPL;
621
622
	if (dc_lsize)
		protected_writeback_dcache_line(addr & ~(dc_lsize - 1));
623
	if (!cpu_icache_snoops_remote_store && scache_size)
Linus Torvalds's avatar
Linus Torvalds committed
624
		protected_writeback_scache_line(addr & ~(sc_lsize - 1));
625
626
	if (ic_lsize)
		protected_flush_icache_line(addr & ~(ic_lsize - 1));
Linus Torvalds's avatar
Linus Torvalds committed
627
628
629
630
631
	if (MIPS4K_ICACHE_REFILL_WAR) {
		__asm__ __volatile__ (
			".set push\n\t"
			".set noat\n\t"
			".set mips3\n\t"
632
#ifdef CONFIG_32BIT
Linus Torvalds's avatar
Linus Torvalds committed
633
634
			"la	$at,1f\n\t"
#endif
635
#ifdef CONFIG_64BIT
Linus Torvalds's avatar
Linus Torvalds committed
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
			"dla	$at,1f\n\t"
#endif
			"cache	%0,($at)\n\t"
			"nop; nop; nop\n"
			"1:\n\t"
			".set pop"
			:
			: "i" (Hit_Invalidate_I));
	}
	if (MIPS_CACHE_SYNC_WAR)
		__asm__ __volatile__ ("sync");
}

static void r4k_flush_cache_sigtramp(unsigned long addr)
{
651
	r4k_on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr, 1, 1);
Linus Torvalds's avatar
Linus Torvalds committed
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
}

static void r4k_flush_icache_all(void)
{
	if (cpu_has_vtag_icache)
		r4k_blast_icache();
}

static inline void rm7k_erratum31(void)
{
	const unsigned long ic_lsize = 32;
	unsigned long addr;

	/* RM7000 erratum #31. The icache is screwed at startup. */
	write_c0_taglo(0);
	write_c0_taghi(0);

	for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) {
		__asm__ __volatile__ (
Thiemo Seufer's avatar
Thiemo Seufer committed
671
			".set push\n\t"
Linus Torvalds's avatar
Linus Torvalds committed
672
673
674
675
676
677
678
679
680
681
682
683
684
685
			".set noreorder\n\t"
			".set mips3\n\t"
			"cache\t%1, 0(%0)\n\t"
			"cache\t%1, 0x1000(%0)\n\t"
			"cache\t%1, 0x2000(%0)\n\t"
			"cache\t%1, 0x3000(%0)\n\t"
			"cache\t%2, 0(%0)\n\t"
			"cache\t%2, 0x1000(%0)\n\t"
			"cache\t%2, 0x2000(%0)\n\t"
			"cache\t%2, 0x3000(%0)\n\t"
			"cache\t%1, 0(%0)\n\t"
			"cache\t%1, 0x1000(%0)\n\t"
			"cache\t%1, 0x2000(%0)\n\t"
			"cache\t%1, 0x3000(%0)\n\t"
Thiemo Seufer's avatar
Thiemo Seufer committed
686
			".set pop\n"
Linus Torvalds's avatar
Linus Torvalds committed
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
			:
			: "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill));
	}
}

static char *way_string[] __initdata = { NULL, "direct mapped", "2-way",
	"3-way", "4-way", "5-way", "6-way", "7-way", "8-way"
};

static void __init probe_pcache(void)
{
	struct cpuinfo_mips *c = &current_cpu_data;
	unsigned int config = read_c0_config();
	unsigned int prid = read_c0_prid();
	unsigned long config1;
	unsigned int lsize;

	switch (c->cputype) {
	case CPU_R4600:			/* QED style two way caches? */
	case CPU_R4700:
	case CPU_R5000:
	case CPU_NEVADA:
		icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
		c->icache.ways = 2;
712
		c->icache.waybit = __ffs(icache_size/2);
Linus Torvalds's avatar
Linus Torvalds committed
713
714
715
716

		dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
		c->dcache.ways = 2;
717
		c->dcache.waybit= __ffs(dcache_size/2);
Linus Torvalds's avatar
Linus Torvalds committed
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748

		c->options |= MIPS_CPU_CACHE_CDEX_P;
		break;

	case CPU_R5432:
	case CPU_R5500:
		icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
		c->icache.ways = 2;
		c->icache.waybit= 0;

		dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
		c->dcache.ways = 2;
		c->dcache.waybit = 0;

		c->options |= MIPS_CPU_CACHE_CDEX_P;
		break;

	case CPU_TX49XX:
		icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
		c->icache.ways = 4;
		c->icache.waybit= 0;

		dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
		c->dcache.ways = 4;
		c->dcache.waybit = 0;

		c->options |= MIPS_CPU_CACHE_CDEX_P;
Atsushi Nemoto's avatar
Atsushi Nemoto committed
749
		c->options |= MIPS_CPU_PREFETCH;
Linus Torvalds's avatar
Linus Torvalds committed
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
		break;

	case CPU_R4000PC:
	case CPU_R4000SC:
	case CPU_R4000MC:
	case CPU_R4400PC:
	case CPU_R4400SC:
	case CPU_R4400MC:
	case CPU_R4300:
		icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
		c->icache.ways = 1;
		c->icache.waybit = 0; 	/* doesn't matter */

		dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
		c->dcache.ways = 1;
		c->dcache.waybit = 0;	/* does not matter */

		c->options |= MIPS_CPU_CACHE_CDEX_P;
		break;

	case CPU_R10000:
	case CPU_R12000:
Kumba's avatar
Kumba committed
774
	case CPU_R14000:
Linus Torvalds's avatar
Linus Torvalds committed
775
776
777
778
779
780
781
782
783
784
785
786
787
788
		icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29));
		c->icache.linesz = 64;
		c->icache.ways = 2;
		c->icache.waybit = 0;

		dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26));
		c->dcache.linesz = 32;
		c->dcache.ways = 2;
		c->dcache.waybit = 0;

		c->options |= MIPS_CPU_PREFETCH;
		break;

	case CPU_VR4133:
789
		write_c0_config(config & ~VR41_CONF_P4K);
Linus Torvalds's avatar
Linus Torvalds committed
790
791
792
793
	case CPU_VR4131:
		/* Workaround for cache instruction bug of VR4131 */
		if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U ||
		    c->processor_id == 0x0c82U) {
794
795
796
			config |= 0x00400000U;
			if (c->processor_id == 0x0c80U)
				config |= VR41_CONF_BP;
Linus Torvalds's avatar
Linus Torvalds committed
797
			write_c0_config(config);
798
799
800
		} else
			c->options |= MIPS_CPU_CACHE_CDEX_P;

Linus Torvalds's avatar
Linus Torvalds committed
801
802
803
		icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
		c->icache.ways = 2;
804
		c->icache.waybit = __ffs(icache_size/2);
Linus Torvalds's avatar
Linus Torvalds committed
805
806
807
808

		dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
		c->dcache.ways = 2;
809
		c->dcache.waybit = __ffs(dcache_size/2);
Linus Torvalds's avatar
Linus Torvalds committed
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
		break;

	case CPU_VR41XX:
	case CPU_VR4111:
	case CPU_VR4121:
	case CPU_VR4122:
	case CPU_VR4181:
	case CPU_VR4181A:
		icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
		c->icache.ways = 1;
		c->icache.waybit = 0; 	/* doesn't matter */

		dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
		c->dcache.ways = 1;
		c->dcache.waybit = 0;	/* does not matter */

		c->options |= MIPS_CPU_CACHE_CDEX_P;
		break;

	case CPU_RM7000:
		rm7k_erratum31();

	case CPU_RM9000:
		icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
		c->icache.ways = 4;
838
		c->icache.waybit = __ffs(icache_size / c->icache.ways);
Linus Torvalds's avatar
Linus Torvalds committed
839
840
841
842

		dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
		c->dcache.ways = 4;
843
		c->dcache.waybit = __ffs(dcache_size / c->dcache.ways);
Linus Torvalds's avatar
Linus Torvalds committed
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870

#if !defined(CONFIG_SMP) || !defined(RM9000_CDEX_SMP_WAR)
		c->options |= MIPS_CPU_CACHE_CDEX_P;
#endif
		c->options |= MIPS_CPU_PREFETCH;
		break;

	default:
		if (!(config & MIPS_CONF_M))
			panic("Don't know how to probe P-caches on this cpu.");

		/*
		 * So we seem to be a MIPS32 or MIPS64 CPU
		 * So let's probe the I-cache ...
		 */
		config1 = read_c0_config1();

		if ((lsize = ((config1 >> 19) & 7)))
			c->icache.linesz = 2 << lsize;
		else
			c->icache.linesz = lsize;
		c->icache.sets = 64 << ((config1 >> 22) & 7);
		c->icache.ways = 1 + ((config1 >> 16) & 7);

		icache_size = c->icache.sets *
		              c->icache.ways *
		              c->icache.linesz;
871
		c->icache.waybit = __ffs(icache_size/c->icache.ways);
Linus Torvalds's avatar
Linus Torvalds committed
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890

		if (config & 0x8)		/* VI bit */
			c->icache.flags |= MIPS_CACHE_VTAG;

		/*
		 * Now probe the MIPS32 / MIPS64 data cache.
		 */
		c->dcache.flags = 0;

		if ((lsize = ((config1 >> 10) & 7)))
			c->dcache.linesz = 2 << lsize;
		else
			c->dcache.linesz= lsize;
		c->dcache.sets = 64 << ((config1 >> 13) & 7);
		c->dcache.ways = 1 + ((config1 >> 7) & 7);

		dcache_size = c->dcache.sets *
		              c->dcache.ways *
		              c->dcache.linesz;
891
		c->dcache.waybit = __ffs(dcache_size/c->dcache.ways);
Linus Torvalds's avatar
Linus Torvalds committed
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913

		c->options |= MIPS_CPU_PREFETCH;
		break;
	}

	/*
	 * Processor configuration sanity check for the R4000SC erratum
	 * #5.  With page sizes larger than 32kB there is no possibility
	 * to get a VCE exception anymore so we don't care about this
	 * misconfiguration.  The case is rather theoretical anyway;
	 * presumably no vendor is shipping his hardware in the "bad"
	 * configuration.
	 */
	if ((prid & 0xff00) == PRID_IMP_R4000 && (prid & 0xff) < 0x40 &&
	    !(config & CONF_SC) && c->icache.linesz != 16 &&
	    PAGE_SIZE <= 0x8000)
		panic("Improper R4000SC processor configuration detected");

	/* compute a couple of other cache variables */
	c->icache.waysize = icache_size / c->icache.ways;
	c->dcache.waysize = dcache_size / c->dcache.ways;

914
915
916
917
	c->icache.sets = c->icache.linesz ?
		icache_size / (c->icache.linesz * c->icache.ways) : 0;
	c->dcache.sets = c->dcache.linesz ?
		dcache_size / (c->dcache.linesz * c->dcache.ways) : 0;
Linus Torvalds's avatar
Linus Torvalds committed
918
919
920
921
922
923
924

	/*
	 * R10000 and R12000 P-caches are odd in a positive way.  They're 32kB
	 * 2-way virtually indexed so normally would suffer from aliases.  So
	 * normally they'd suffer from aliases but magic in the hardware deals
	 * with that for us so we don't need to take care ourselves.
	 */
925
	switch (c->cputype) {
926
	case CPU_20KC:
927
	case CPU_25KF:
928
		c->dcache.flags |= MIPS_CACHE_PINDEX;
929
930
	case CPU_R10000:
	case CPU_R12000:
Kumba's avatar
Kumba committed
931
	case CPU_R14000:
932
	case CPU_SB1:
933
934
		break;
	case CPU_24K:
935
	case CPU_34K:
936
	case CPU_74K:
937
938
939
940
941
942
		if ((read_c0_config7() & (1 << 16))) {
			/* effectively physically indexed dcache,
			   thus no virtual aliases. */
			c->dcache.flags |= MIPS_CACHE_PINDEX;
			break;
		}
943
	default:
944
945
		if (c->dcache.waysize > PAGE_SIZE)
			c->dcache.flags |= MIPS_CACHE_ALIASES;
946
	}
Linus Torvalds's avatar
Linus Torvalds committed
947
948
949
950
951
952
953
954
955
956

	switch (c->cputype) {
	case CPU_20KC:
		/*
		 * Some older 20Kc chips doesn't have the 'VI' bit in
		 * the config register.
		 */
		c->icache.flags |= MIPS_CACHE_VTAG;
		break;

Pete Popov's avatar
Pete Popov committed
957
	case CPU_AU1000:
Linus Torvalds's avatar
Linus Torvalds committed
958
	case CPU_AU1500:
Pete Popov's avatar
Pete Popov committed
959
960
961
	case CPU_AU1100:
	case CPU_AU1550:
	case CPU_AU1200:
Linus Torvalds's avatar
Linus Torvalds committed
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
		c->icache.flags |= MIPS_CACHE_IC_F_DC;
		break;
	}

	printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n",
	       icache_size >> 10,
	       cpu_has_vtag_icache ? "virtually tagged" : "physically tagged",
	       way_string[c->icache.ways], c->icache.linesz);

	printk("Primary data cache %ldkB, %s, linesize %d bytes.\n",
	       dcache_size >> 10, way_string[c->dcache.ways], c->dcache.linesz);
}

/*
 * If you even _breathe_ on this function, look at the gcc output and make sure
 * it does not pop things on and off the stack for the cache sizing loop that
 * executes in KSEG1 space or else you will crash and burn badly.  You have
 * been warned.
 */
static int __init probe_scache(void)
{
	extern unsigned long stext;
	unsigned long flags, addr, begin, end, pow2;
	unsigned int config = read_c0_config();
	struct cpuinfo_mips *c = &current_cpu_data;
	int tmp;

	if (config & CONF_SC)
		return 0;

	begin = (unsigned long) &stext;
	begin &= ~((4 * 1024 * 1024) - 1);
	end = begin + (4 * 1024 * 1024);

	/*
	 * This is such a bitch, you'd think they would make it easy to do
	 * this.  Away you daemons of stupidity!
	 */
	local_irq_save(flags);
For faster browsing, not all history is shown. View entire blame