ivt.S 51.8 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
/*
 * arch/ia64/kernel/ivt.S
 *
4
 * Copyright (C) 1998-2001, 2003, 2005 Hewlett-Packard Co
Linus Torvalds's avatar
Linus Torvalds committed
5
6
7
8
9
10
11
12
13
14
 *	Stephane Eranian <eranian@hpl.hp.com>
 *	David Mosberger <davidm@hpl.hp.com>
 * Copyright (C) 2000, 2002-2003 Intel Co
 *	Asit Mallick <asit.k.mallick@intel.com>
 *      Suresh Siddha <suresh.b.siddha@intel.com>
 *      Kenneth Chen <kenneth.w.chen@intel.com>
 *      Fenghua Yu <fenghua.yu@intel.com>
 *
 * 00/08/23 Asit Mallick <asit.k.mallick@intel.com> TLB handling for SMP
 * 00/12/20 David Mosberger-Tang <davidm@hpl.hp.com> DTLB/ITLB handler now uses virtual PT.
15
16
17
18
19
20
21
22
 *
 * Copyright (C) 2005 Hewlett-Packard Co
 *	Dan Magenheimer <dan.magenheimer@hp.com>
 *      Xen paravirtualization
 * Copyright (c) 2008 Isaku Yamahata <yamahata at valinux co jp>
 *                    VA Linux Systems Japan K.K.
 *                    pv_ops.
 *      Yaozu (Eddie) Dong <eddie.dong@intel.com>
Linus Torvalds's avatar
Linus Torvalds committed
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 */
/*
 * This file defines the interruption vector table used by the CPU.
 * It does not include one entry per possible cause of interruption.
 *
 * The first 20 entries of the table contain 64 bundles each while the
 * remaining 48 entries contain only 16 bundles each.
 *
 * The 64 bundles are used to allow inlining the whole handler for critical
 * interruptions like TLB misses.
 *
 *  For each entry, the comment is as follows:
 *
 *		// 0x1c00 Entry 7 (size 64 bundles) Data Key Miss (12,51)
 *  entry offset ----/     /         /                  /          /
 *  entry number ---------/         /                  /          /
 *  size of the entry -------------/                  /          /
 *  vector name -------------------------------------/          /
 *  interruptions triggering this vector ----------------------/
 *
 * The table is 32KB in size and must be aligned on 32KB boundary.
 * (The CPU ignores the 15 lower bits of the address)
 *
 * Table is based upon EAS2.6 (Oct 1999)
 */


#include <asm/asmmacro.h>
#include <asm/break.h>
#include <asm/kregs.h>
53
#include <asm/asm-offsets.h>
Linus Torvalds's avatar
Linus Torvalds committed
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
#include <asm/pgtable.h>
#include <asm/processor.h>
#include <asm/ptrace.h>
#include <asm/thread_info.h>
#include <asm/unistd.h>
#include <asm/errno.h>

#if 1
# define PSR_DEFAULT_BITS	psr.ac
#else
# define PSR_DEFAULT_BITS	0
#endif

#if 0
  /*
   * This lets you track the last eight faults that occurred on the CPU.  Make sure ar.k2 isn't
   * needed for something else before enabling this...
   */
# define DBG_FAULT(i)	mov r16=ar.k2;;	shl r16=r16,8;;	add r16=(i),r16;;mov ar.k2=r16
#else
# define DBG_FAULT(i)
#endif

#include "minstate.h"

#define FAULT(n)									\
	mov r31=pr;									\
	mov r19=n;;			/* prepare to save predicates */		\
	br.sptk.many dispatch_to_fault_handler

84
	.section .text..ivt,"ax"
Linus Torvalds's avatar
Linus Torvalds committed
85
86
87
88
89
90
91
92
93
94
95
96
97
98

	.align 32768	// align on 32KB boundary
	.global ia64_ivt
ia64_ivt:
/////////////////////////////////////////////////////////////////////////////////////////
// 0x0000 Entry 0 (size 64 bundles) VHPT Translation (8,20,47)
ENTRY(vhpt_miss)
	DBG_FAULT(0)
	/*
	 * The VHPT vector is invoked when the TLB entry for the virtual page table
	 * is missing.  This happens only as a result of a previous
	 * (the "original") TLB miss, which may either be caused by an instruction
	 * fetch or a data access (or non-access).
	 *
99
100
101
102
103
104
105
	 * What we do here is normal TLB miss handing for the _original_ miss,
	 * followed by inserting the TLB entry for the virtual page table page
	 * that the VHPT walker was attempting to access.  The latter gets
	 * inserted as long as page table entry above pte level have valid
	 * mappings for the faulting address.  The TLB entry for the original
	 * miss gets inserted only if the pte entry indicates that the page is
	 * present.
Linus Torvalds's avatar
Linus Torvalds committed
106
107
108
	 *
	 * do_page_fault gets invoked in the following cases:
	 *	- the faulting virtual address uses unimplemented address bits
109
	 *	- the faulting virtual address has no valid page table mapping
Linus Torvalds's avatar
Linus Torvalds committed
110
	 */
111
	MOV_FROM_IFA(r16)			// get address that caused the TLB miss
Linus Torvalds's avatar
Linus Torvalds committed
112
113
#ifdef CONFIG_HUGETLB_PAGE
	movl r18=PAGE_SHIFT
114
	MOV_FROM_ITIR(r25)
Linus Torvalds's avatar
Linus Torvalds committed
115
116
#endif
	;;
117
	RSM_PSR_DT				// use physical addressing for data
Linus Torvalds's avatar
Linus Torvalds committed
118
119
120
121
122
	mov r31=pr				// save the predicate registers
	mov r19=IA64_KR(PT_BASE)		// get page table base address
	shl r21=r16,3				// shift bit 60 into sign bit
	shr.u r17=r16,61			// get the region number into r17
	;;
Robin Holt's avatar
Robin Holt committed
123
	shr.u r22=r21,3
Linus Torvalds's avatar
Linus Torvalds committed
124
125
126
127
128
129
130
131
132
133
134
#ifdef CONFIG_HUGETLB_PAGE
	extr.u r26=r25,2,6
	;;
	cmp.ne p8,p0=r18,r26
	sub r27=r26,r18
	;;
(p8)	dep r25=r18,r25,2,6
(p8)	shr r22=r22,r27
#endif
	;;
	cmp.eq p6,p7=5,r17			// is IFA pointing into to region 5?
135
	shr.u r18=r22,PGDIR_SHIFT		// get bottom portion of pgd index bit
Linus Torvalds's avatar
Linus Torvalds committed
136
137
138
139
140
141
142
143
144
145
	;;
(p7)	dep r17=r17,r19,(PAGE_SHIFT-3),3	// put region number bits in place

	srlz.d
	LOAD_PHYSICAL(p6, r19, swapper_pg_dir)	// region 5 is rooted at swapper_pg_dir

	.pred.rel "mutex", p6, p7
(p6)	shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT
(p7)	shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3
	;;
146
147
(p6)	dep r17=r18,r19,3,(PAGE_SHIFT-3)	// r17=pgd_offset for region 5
(p7)	dep r17=r18,r17,3,(PAGE_SHIFT-6)	// r17=pgd_offset for region[0-4]
Linus Torvalds's avatar
Linus Torvalds committed
148
	cmp.eq p7,p6=0,r21			// unused address bits all zeroes?
Robin Holt's avatar
Robin Holt committed
149
#ifdef CONFIG_PGTABLE_4
150
	shr.u r28=r22,PUD_SHIFT			// shift pud index into position
Robin Holt's avatar
Robin Holt committed
151
#else
152
	shr.u r18=r22,PMD_SHIFT			// shift pmd index into position
Robin Holt's avatar
Robin Holt committed
153
#endif
Linus Torvalds's avatar
Linus Torvalds committed
154
	;;
155
	ld8 r17=[r17]				// get *pgd (may be 0)
Linus Torvalds's avatar
Linus Torvalds committed
156
	;;
157
(p7)	cmp.eq p6,p7=r17,r0			// was pgd_present(*pgd) == NULL?
Robin Holt's avatar
Robin Holt committed
158
#ifdef CONFIG_PGTABLE_4
159
	dep r28=r28,r17,3,(PAGE_SHIFT-3)	// r28=pud_offset(pgd,addr)
Robin Holt's avatar
Robin Holt committed
160
	;;
161
162
	shr.u r18=r22,PMD_SHIFT			// shift pmd index into position
(p7)	ld8 r29=[r28]				// get *pud (may be 0)
Linus Torvalds's avatar
Linus Torvalds committed
163
	;;
164
165
(p7)	cmp.eq.or.andcm p6,p7=r29,r0		// was pud_present(*pud) == NULL?
	dep r17=r18,r29,3,(PAGE_SHIFT-3)	// r17=pmd_offset(pud,addr)
Robin Holt's avatar
Robin Holt committed
166
#else
167
	dep r17=r18,r17,3,(PAGE_SHIFT-3)	// r17=pmd_offset(pgd,addr)
Robin Holt's avatar
Robin Holt committed
168
#endif
Linus Torvalds's avatar
Linus Torvalds committed
169
	;;
170
171
(p7)	ld8 r20=[r17]				// get *pmd (may be 0)
	shr.u r19=r22,PAGE_SHIFT		// shift pte index into position
Linus Torvalds's avatar
Linus Torvalds committed
172
	;;
173
174
(p7)	cmp.eq.or.andcm p6,p7=r20,r0		// was pmd_present(*pmd) == NULL?
	dep r21=r19,r20,3,(PAGE_SHIFT-3)	// r21=pte_offset(pmd,addr)
Robin Holt's avatar
Robin Holt committed
175
	;;
176
(p7)	ld8 r18=[r21]				// read *pte
177
	MOV_FROM_ISR(r19)			// cr.isr bit 32 tells us if this is an insn miss
Linus Torvalds's avatar
Linus Torvalds committed
178
179
	;;
(p7)	tbit.z p6,p7=r18,_PAGE_P_BIT		// page present bit cleared?
180
	MOV_FROM_IHA(r22)			// get the VHPT address that caused the TLB miss
Linus Torvalds's avatar
Linus Torvalds committed
181
182
183
184
	;;					// avoid RAW on p7
(p7)	tbit.nz.unc p10,p11=r19,32		// is it an instruction TLB miss?
	dep r23=0,r20,0,PAGE_SHIFT		// clear low bits to get page address
	;;
185
186
	ITC_I_AND_D(p10, p11, r18, r24)		// insert the instruction TLB entry and
						// insert the data TLB entry
Linus Torvalds's avatar
Linus Torvalds committed
187
(p6)	br.cond.spnt.many page_fault		// handle bad address/page not present (page fault)
188
	MOV_TO_IFA(r22, r24)
Linus Torvalds's avatar
Linus Torvalds committed
189
190

#ifdef CONFIG_HUGETLB_PAGE
191
	MOV_TO_ITIR(p8, r25, r24)		// change to default page-size for VHPT
Linus Torvalds's avatar
Linus Torvalds committed
192
193
194
195
196
197
198
199
200
#endif

	/*
	 * Now compute and insert the TLB entry for the virtual page table.  We never
	 * execute in a page table page so there is no need to set the exception deferral
	 * bit.
	 */
	adds r24=__DIRTY_BITS_NO_ED|_PAGE_PL_0|_PAGE_AR_RW,r23
	;;
201
	ITC_D(p7, r24, r25)
Linus Torvalds's avatar
Linus Torvalds committed
202
203
204
205
206
207
208
209
210
	;;
#ifdef CONFIG_SMP
	/*
	 * Tell the assemblers dependency-violation checker that the above "itc" instructions
	 * cannot possibly affect the following loads:
	 */
	dv_serialize_data

	/*
211
	 * Re-check pagetable entry.  If they changed, we may have received a ptc.g
Linus Torvalds's avatar
Linus Torvalds committed
212
	 * between reading the pagetable and the "itc".  If so, flush the entry we
213
214
215
216
217
218
219
220
221
	 * inserted and retry.  At this point, we have:
	 *
	 * r28 = equivalent of pud_offset(pgd, ifa)
	 * r17 = equivalent of pmd_offset(pud, ifa)
	 * r21 = equivalent of pte_offset(pmd, ifa)
	 *
	 * r29 = *pud
	 * r20 = *pmd
	 * r18 = *pte
Linus Torvalds's avatar
Linus Torvalds committed
222
	 */
223
224
	ld8 r25=[r21]				// read *pte again
	ld8 r26=[r17]				// read *pmd again
Robin Holt's avatar
Robin Holt committed
225
#ifdef CONFIG_PGTABLE_4
226
	ld8 r19=[r28]				// read *pud again
Robin Holt's avatar
Robin Holt committed
227
228
#endif
	cmp.ne p6,p7=r0,r0
Linus Torvalds's avatar
Linus Torvalds committed
229
	;;
230
	cmp.ne.or.andcm p6,p7=r26,r20		// did *pmd change
Robin Holt's avatar
Robin Holt committed
231
#ifdef CONFIG_PGTABLE_4
232
	cmp.ne.or.andcm p6,p7=r19,r29		// did *pud change
Robin Holt's avatar
Robin Holt committed
233
#endif
Linus Torvalds's avatar
Linus Torvalds committed
234
235
236
	mov r27=PAGE_SHIFT<<2
	;;
(p6)	ptc.l r22,r27				// purge PTE page translation
237
(p7)	cmp.ne.or.andcm p6,p7=r25,r18		// did *pte change
Linus Torvalds's avatar
Linus Torvalds committed
238
239
240
241
242
	;;
(p6)	ptc.l r16,r27				// purge translation
#endif

	mov pr=r31,-1				// restore predicate registers
243
	RFI
Linus Torvalds's avatar
Linus Torvalds committed
244
245
246
247
248
249
250
251
END(vhpt_miss)

	.org ia64_ivt+0x400
/////////////////////////////////////////////////////////////////////////////////////////
// 0x0400 Entry 1 (size 64 bundles) ITLB (21)
ENTRY(itlb_miss)
	DBG_FAULT(1)
	/*
252
	 * The ITLB handler accesses the PTE via the virtually mapped linear
Linus Torvalds's avatar
Linus Torvalds committed
253
	 * page table.  If a nested TLB miss occurs, we switch into physical
254
255
	 * mode, walk the page table, and then re-execute the PTE read and
	 * go on normally after that.
Linus Torvalds's avatar
Linus Torvalds committed
256
	 */
257
	MOV_FROM_IFA(r16)			// get virtual address
Linus Torvalds's avatar
Linus Torvalds committed
258
259
260
	mov r29=b0				// save b0
	mov r31=pr				// save predicates
.itlb_fault:
261
	MOV_FROM_IHA(r17)			// get virtual address of PTE
Linus Torvalds's avatar
Linus Torvalds committed
262
263
	movl r30=1f				// load nested fault continuation point
	;;
264
1:	ld8 r18=[r17]				// read *pte
Linus Torvalds's avatar
Linus Torvalds committed
265
266
267
268
269
	;;
	mov b0=r29
	tbit.z p6,p0=r18,_PAGE_P_BIT		// page present bit cleared?
(p6)	br.cond.spnt page_fault
	;;
270
	ITC_I(p0, r18, r19)
Linus Torvalds's avatar
Linus Torvalds committed
271
272
273
274
275
276
277
278
	;;
#ifdef CONFIG_SMP
	/*
	 * Tell the assemblers dependency-violation checker that the above "itc" instructions
	 * cannot possibly affect the following loads:
	 */
	dv_serialize_data

279
	ld8 r19=[r17]				// read *pte again and see if same
Linus Torvalds's avatar
Linus Torvalds committed
280
281
282
283
284
285
286
	mov r20=PAGE_SHIFT<<2			// setup page size for purge
	;;
	cmp.ne p7,p0=r18,r19
	;;
(p7)	ptc.l r16,r20
#endif
	mov pr=r31,-1
287
	RFI
Linus Torvalds's avatar
Linus Torvalds committed
288
289
290
291
292
293
294
295
END(itlb_miss)

	.org ia64_ivt+0x0800
/////////////////////////////////////////////////////////////////////////////////////////
// 0x0800 Entry 2 (size 64 bundles) DTLB (9,48)
ENTRY(dtlb_miss)
	DBG_FAULT(2)
	/*
296
	 * The DTLB handler accesses the PTE via the virtually mapped linear
Linus Torvalds's avatar
Linus Torvalds committed
297
	 * page table.  If a nested TLB miss occurs, we switch into physical
298
299
	 * mode, walk the page table, and then re-execute the PTE read and
	 * go on normally after that.
Linus Torvalds's avatar
Linus Torvalds committed
300
	 */
301
	MOV_FROM_IFA(r16)			// get virtual address
Linus Torvalds's avatar
Linus Torvalds committed
302
303
304
	mov r29=b0				// save b0
	mov r31=pr				// save predicates
dtlb_fault:
305
	MOV_FROM_IHA(r17)			// get virtual address of PTE
Linus Torvalds's avatar
Linus Torvalds committed
306
307
	movl r30=1f				// load nested fault continuation point
	;;
308
1:	ld8 r18=[r17]				// read *pte
Linus Torvalds's avatar
Linus Torvalds committed
309
310
311
312
313
	;;
	mov b0=r29
	tbit.z p6,p0=r18,_PAGE_P_BIT		// page present bit cleared?
(p6)	br.cond.spnt page_fault
	;;
314
	ITC_D(p0, r18, r19)
Linus Torvalds's avatar
Linus Torvalds committed
315
316
317
318
319
320
321
322
	;;
#ifdef CONFIG_SMP
	/*
	 * Tell the assemblers dependency-violation checker that the above "itc" instructions
	 * cannot possibly affect the following loads:
	 */
	dv_serialize_data

323
	ld8 r19=[r17]				// read *pte again and see if same
Linus Torvalds's avatar
Linus Torvalds committed
324
325
326
327
328
329
330
	mov r20=PAGE_SHIFT<<2			// setup page size for purge
	;;
	cmp.ne p7,p0=r18,r19
	;;
(p7)	ptc.l r16,r20
#endif
	mov pr=r31,-1
331
	RFI
Linus Torvalds's avatar
Linus Torvalds committed
332
333
334
335
336
337
338
END(dtlb_miss)

	.org ia64_ivt+0x0c00
/////////////////////////////////////////////////////////////////////////////////////////
// 0x0c00 Entry 3 (size 64 bundles) Alt ITLB (19)
ENTRY(alt_itlb_miss)
	DBG_FAULT(3)
339
	MOV_FROM_IFA(r16)	// get address that caused the TLB miss
Linus Torvalds's avatar
Linus Torvalds committed
340
	movl r17=PAGE_KERNEL
341
	MOV_FROM_IPSR(p0, r21)
Linus Torvalds's avatar
Linus Torvalds committed
342
343
344
345
346
347
348
349
	movl r19=(((1 << IA64_MAX_PHYS_BITS) - 1) & ~0xfff)
	mov r31=pr
	;;
#ifdef CONFIG_DISABLE_VHPT
	shr.u r22=r16,61			// get the region number into r21
	;;
	cmp.gt p8,p0=6,r22			// user mode
	;;
350
	THASH(p8, r17, r16, r23)
Linus Torvalds's avatar
Linus Torvalds committed
351
	;;
352
	MOV_TO_IHA(p8, r17, r23)
Linus Torvalds's avatar
Linus Torvalds committed
353
354
355
356
357
358
359
360
361
362
363
364
365
366
(p8)	mov r29=b0				// save b0
(p8)	br.cond.dptk .itlb_fault
#endif
	extr.u r23=r21,IA64_PSR_CPL0_BIT,2	// extract psr.cpl
	and r19=r19,r16		// clear ed, reserved bits, and PTE control bits
	shr.u r18=r16,57	// move address bit 61 to bit 4
	;;
	andcm r18=0x10,r18	// bit 4=~address-bit(61)
	cmp.ne p8,p0=r0,r23	// psr.cpl != 0?
	or r19=r17,r19		// insert PTE control bits into r19
	;;
	or r19=r19,r18		// set bit 4 (uncached) if the access was to region 6
(p8)	br.cond.spnt page_fault
	;;
367
	ITC_I(p0, r19, r18)	// insert the TLB entry
Linus Torvalds's avatar
Linus Torvalds committed
368
	mov pr=r31,-1
369
	RFI
Linus Torvalds's avatar
Linus Torvalds committed
370
371
372
373
374
375
376
END(alt_itlb_miss)

	.org ia64_ivt+0x1000
/////////////////////////////////////////////////////////////////////////////////////////
// 0x1000 Entry 4 (size 64 bundles) Alt DTLB (7,46)
ENTRY(alt_dtlb_miss)
	DBG_FAULT(4)
377
	MOV_FROM_IFA(r16)	// get address that caused the TLB miss
Linus Torvalds's avatar
Linus Torvalds committed
378
	movl r17=PAGE_KERNEL
379
	MOV_FROM_ISR(r20)
Linus Torvalds's avatar
Linus Torvalds committed
380
	movl r19=(((1 << IA64_MAX_PHYS_BITS) - 1) & ~0xfff)
381
	MOV_FROM_IPSR(p0, r21)
Linus Torvalds's avatar
Linus Torvalds committed
382
	mov r31=pr
383
	mov r24=PERCPU_ADDR
Linus Torvalds's avatar
Linus Torvalds committed
384
385
386
387
388
389
	;;
#ifdef CONFIG_DISABLE_VHPT
	shr.u r22=r16,61			// get the region number into r21
	;;
	cmp.gt p8,p0=6,r22			// access to region 0-5
	;;
390
	THASH(p8, r17, r16, r25)
Linus Torvalds's avatar
Linus Torvalds committed
391
	;;
392
	MOV_TO_IHA(p8, r17, r25)
Linus Torvalds's avatar
Linus Torvalds committed
393
394
395
(p8)	mov r29=b0				// save b0
(p8)	br.cond.dptk dtlb_fault
#endif
396
397
398
399
400
401
402
403
404
	cmp.ge p10,p11=r16,r24			// access to per_cpu_data?
	tbit.z p12,p0=r16,61			// access to region 6?
	mov r25=PERCPU_PAGE_SHIFT << 2
	mov r26=PERCPU_PAGE_SIZE
	nop.m 0
	nop.b 0
	;;
(p10)	mov r19=IA64_KR(PER_CPU_DATA)
(p11)	and r19=r19,r16				// clear non-ppn fields
Linus Torvalds's avatar
Linus Torvalds committed
405
406
407
408
409
	extr.u r23=r21,IA64_PSR_CPL0_BIT,2	// extract psr.cpl
	and r22=IA64_ISR_CODE_MASK,r20		// get the isr.code field
	tbit.nz p6,p7=r20,IA64_ISR_SP_BIT	// is speculation bit on?
	tbit.nz p9,p0=r20,IA64_ISR_NA_BIT	// is non-access bit on?
	;;
410
(p10)	sub r19=r19,r26
411
	MOV_TO_ITIR(p10, r25, r24)
Linus Torvalds's avatar
Linus Torvalds committed
412
413
	cmp.ne p8,p0=r0,r23
(p9)	cmp.eq.or.andcm p6,p7=IA64_ISR_CODE_LFETCH,r22	// check isr.code field
414
(p12)	dep r17=-1,r17,4,1			// set ma=UC for region 6 addr
Linus Torvalds's avatar
Linus Torvalds committed
415
416
417
418
(p8)	br.cond.spnt page_fault

	dep r21=-1,r21,IA64_PSR_ED_BIT,1
	;;
419
	or r19=r19,r17		// insert PTE control bits into r19
420
	MOV_TO_IPSR(p6, r21, r24)
Linus Torvalds's avatar
Linus Torvalds committed
421
	;;
422
	ITC_D(p7, r19, r18)	// insert the TLB entry
Linus Torvalds's avatar
Linus Torvalds committed
423
	mov pr=r31,-1
424
	RFI
Linus Torvalds's avatar
Linus Torvalds committed
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
END(alt_dtlb_miss)

	.org ia64_ivt+0x1400
/////////////////////////////////////////////////////////////////////////////////////////
// 0x1400 Entry 5 (size 64 bundles) Data nested TLB (6,45)
ENTRY(nested_dtlb_miss)
	/*
	 * In the absence of kernel bugs, we get here when the virtually mapped linear
	 * page table is accessed non-speculatively (e.g., in the Dirty-bit, Instruction
	 * Access-bit, or Data Access-bit faults).  If the DTLB entry for the virtual page
	 * table is missing, a nested TLB miss fault is triggered and control is
	 * transferred to this point.  When this happens, we lookup the pte for the
	 * faulting address by walking the page table in physical mode and return to the
	 * continuation point passed in register r30 (or call page_fault if the address is
	 * not mapped).
	 *
	 * Input:	r16:	faulting address
	 *		r29:	saved b0
	 *		r30:	continuation address
	 *		r31:	saved pr
	 *
446
	 * Output:	r17:	physical address of PTE of faulting address
Linus Torvalds's avatar
Linus Torvalds committed
447
448
449
450
	 *		r29:	saved b0
	 *		r30:	continuation address
	 *		r31:	saved pr
	 *
451
	 * Clobbered:	b0, r18, r19, r21, r22, psr.dt (cleared)
Linus Torvalds's avatar
Linus Torvalds committed
452
	 */
453
	RSM_PSR_DT				// switch to using physical data addressing
Linus Torvalds's avatar
Linus Torvalds committed
454
455
	mov r19=IA64_KR(PT_BASE)		// get the page table base address
	shl r21=r16,3				// shift bit 60 into sign bit
456
	MOV_FROM_ITIR(r18)
Linus Torvalds's avatar
Linus Torvalds committed
457
458
	;;
	shr.u r17=r16,61			// get the region number into r17
459
	extr.u r18=r18,2,6			// get the faulting page size
Linus Torvalds's avatar
Linus Torvalds committed
460
461
	;;
	cmp.eq p6,p7=5,r17			// is faulting address in region 5?
462
463
	add r22=-PAGE_SHIFT,r18			// adjustment for hugetlb address
	add r18=PGDIR_SHIFT-PAGE_SHIFT,r18
Linus Torvalds's avatar
Linus Torvalds committed
464
	;;
465
466
	shr.u r22=r16,r22
	shr.u r18=r16,r18
Linus Torvalds's avatar
Linus Torvalds committed
467
468
469
470
471
472
473
474
475
(p7)	dep r17=r17,r19,(PAGE_SHIFT-3),3	// put region number bits in place

	srlz.d
	LOAD_PHYSICAL(p6, r19, swapper_pg_dir)	// region 5 is rooted at swapper_pg_dir

	.pred.rel "mutex", p6, p7
(p6)	shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT
(p7)	shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3
	;;
476
477
(p6)	dep r17=r18,r19,3,(PAGE_SHIFT-3)	// r17=pgd_offset for region 5
(p7)	dep r17=r18,r17,3,(PAGE_SHIFT-6)	// r17=pgd_offset for region[0-4]
Linus Torvalds's avatar
Linus Torvalds committed
478
	cmp.eq p7,p6=0,r21			// unused address bits all zeroes?
Robin Holt's avatar
Robin Holt committed
479
#ifdef CONFIG_PGTABLE_4
480
	shr.u r18=r22,PUD_SHIFT			// shift pud index into position
Robin Holt's avatar
Robin Holt committed
481
#else
482
	shr.u r18=r22,PMD_SHIFT			// shift pmd index into position
Robin Holt's avatar
Robin Holt committed
483
#endif
Linus Torvalds's avatar
Linus Torvalds committed
484
	;;
485
	ld8 r17=[r17]				// get *pgd (may be 0)
Linus Torvalds's avatar
Linus Torvalds committed
486
	;;
487
488
(p7)	cmp.eq p6,p7=r17,r0			// was pgd_present(*pgd) == NULL?
	dep r17=r18,r17,3,(PAGE_SHIFT-3)	// r17=p[u|m]d_offset(pgd,addr)
Linus Torvalds's avatar
Linus Torvalds committed
489
	;;
Robin Holt's avatar
Robin Holt committed
490
#ifdef CONFIG_PGTABLE_4
491
492
(p7)	ld8 r17=[r17]				// get *pud (may be 0)
	shr.u r18=r22,PMD_SHIFT			// shift pmd index into position
Linus Torvalds's avatar
Linus Torvalds committed
493
	;;
494
495
(p7)	cmp.eq.or.andcm p6,p7=r17,r0		// was pud_present(*pud) == NULL?
	dep r17=r18,r17,3,(PAGE_SHIFT-3)	// r17=pmd_offset(pud,addr)
Robin Holt's avatar
Robin Holt committed
496
497
	;;
#endif
498
499
(p7)	ld8 r17=[r17]				// get *pmd (may be 0)
	shr.u r19=r22,PAGE_SHIFT		// shift pte index into position
Robin Holt's avatar
Robin Holt committed
500
	;;
501
502
(p7)	cmp.eq.or.andcm p6,p7=r17,r0		// was pmd_present(*pmd) == NULL?
	dep r17=r19,r17,3,(PAGE_SHIFT-3)	// r17=pte_offset(pmd,addr);
Linus Torvalds's avatar
Linus Torvalds committed
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
(p6)	br.cond.spnt page_fault
	mov b0=r30
	br.sptk.many b0				// return to continuation point
END(nested_dtlb_miss)

	.org ia64_ivt+0x1800
/////////////////////////////////////////////////////////////////////////////////////////
// 0x1800 Entry 6 (size 64 bundles) Instruction Key Miss (24)
ENTRY(ikey_miss)
	DBG_FAULT(6)
	FAULT(6)
END(ikey_miss)

	.org ia64_ivt+0x1c00
/////////////////////////////////////////////////////////////////////////////////////////
// 0x1c00 Entry 7 (size 64 bundles) Data Key Miss (12,51)
ENTRY(dkey_miss)
	DBG_FAULT(7)
	FAULT(7)
END(dkey_miss)

	.org ia64_ivt+0x2000
/////////////////////////////////////////////////////////////////////////////////////////
// 0x2000 Entry 8 (size 64 bundles) Dirty-bit (54)
ENTRY(dirty_bit)
	DBG_FAULT(8)
	/*
	 * What we do here is to simply turn on the dirty bit in the PTE.  We need to
	 * update both the page-table and the TLB entry.  To efficiently access the PTE,
	 * we address it through the virtual page table.  Most likely, the TLB entry for
	 * the relevant virtual page table page is still present in the TLB so we can
	 * normally do this without additional TLB misses.  In case the necessary virtual
	 * page table TLB entry isn't present, we take a nested TLB miss hit where we look
	 * up the physical address of the L3 PTE and then continue at label 1 below.
	 */
538
	MOV_FROM_IFA(r16)			// get the address that caused the fault
Linus Torvalds's avatar
Linus Torvalds committed
539
540
	movl r30=1f				// load continuation point in case of nested fault
	;;
541
	THASH(p0, r17, r16, r18)		// compute virtual address of L3 PTE
Linus Torvalds's avatar
Linus Torvalds committed
542
543
544
545
546
547
548
549
550
	mov r29=b0				// save b0 in case of nested fault
	mov r31=pr				// save pr
#ifdef CONFIG_SMP
	mov r28=ar.ccv				// save ar.ccv
	;;
1:	ld8 r18=[r17]
	;;					// avoid RAW on r18
	mov ar.ccv=r18				// set compare value for cmpxchg
	or r25=_PAGE_D|_PAGE_A,r18		// set the dirty and accessed bits
551
	tbit.z p7,p6 = r18,_PAGE_P_BIT		// Check present bit
Linus Torvalds's avatar
Linus Torvalds committed
552
	;;
553
(p6)	cmpxchg8.acq r26=[r17],r25,ar.ccv	// Only update if page is present
Linus Torvalds's avatar
Linus Torvalds committed
554
555
	mov r24=PAGE_SHIFT<<2
	;;
556
(p6)	cmp.eq p6,p7=r26,r18			// Only compare if page is present
Linus Torvalds's avatar
Linus Torvalds committed
557
	;;
558
	ITC_D(p6, r25, r18)			// install updated PTE
Linus Torvalds's avatar
Linus Torvalds committed
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
	;;
	/*
	 * Tell the assemblers dependency-violation checker that the above "itc" instructions
	 * cannot possibly affect the following loads:
	 */
	dv_serialize_data

	ld8 r18=[r17]				// read PTE again
	;;
	cmp.eq p6,p7=r18,r25			// is it same as the newly installed
	;;
(p7)	ptc.l r16,r24
	mov b0=r29				// restore b0
	mov ar.ccv=r28
#else
	;;
1:	ld8 r18=[r17]
	;;					// avoid RAW on r18
	or r18=_PAGE_D|_PAGE_A,r18		// set the dirty and accessed bits
	mov b0=r29				// restore b0
	;;
	st8 [r17]=r18				// store back updated PTE
581
	ITC_D(p0, r18, r16)			// install updated PTE
Linus Torvalds's avatar
Linus Torvalds committed
582
583
#endif
	mov pr=r31,-1				// restore pr
584
	RFI
Linus Torvalds's avatar
Linus Torvalds committed
585
586
587
588
589
590
591
592
END(dirty_bit)

	.org ia64_ivt+0x2400
/////////////////////////////////////////////////////////////////////////////////////////
// 0x2400 Entry 9 (size 64 bundles) Instruction Access-bit (27)
ENTRY(iaccess_bit)
	DBG_FAULT(9)
	// Like Entry 8, except for instruction access
593
	MOV_FROM_IFA(r16)			// get the address that caused the fault
Linus Torvalds's avatar
Linus Torvalds committed
594
595
596
597
598
599
	movl r30=1f				// load continuation point in case of nested fault
	mov r31=pr				// save predicates
#ifdef CONFIG_ITANIUM
	/*
	 * Erratum 10 (IFA may contain incorrect address) has "NoFix" status.
	 */
600
	MOV_FROM_IPSR(p0, r17)
Linus Torvalds's avatar
Linus Torvalds committed
601
	;;
602
	MOV_FROM_IIP(r18)
Linus Torvalds's avatar
Linus Torvalds committed
603
604
605
606
607
	tbit.z p6,p0=r17,IA64_PSR_IS_BIT	// IA64 instruction set?
	;;
(p6)	mov r16=r18				// if so, use cr.iip instead of cr.ifa
#endif /* CONFIG_ITANIUM */
	;;
608
	THASH(p0, r17, r16, r18)		// compute virtual address of L3 PTE
Linus Torvalds's avatar
Linus Torvalds committed
609
610
611
612
613
614
615
616
	mov r29=b0				// save b0 in case of nested fault)
#ifdef CONFIG_SMP
	mov r28=ar.ccv				// save ar.ccv
	;;
1:	ld8 r18=[r17]
	;;
	mov ar.ccv=r18				// set compare value for cmpxchg
	or r25=_PAGE_A,r18			// set the accessed bit
617
	tbit.z p7,p6 = r18,_PAGE_P_BIT	 	// Check present bit
Linus Torvalds's avatar
Linus Torvalds committed
618
	;;
619
(p6)	cmpxchg8.acq r26=[r17],r25,ar.ccv	// Only if page present
Linus Torvalds's avatar
Linus Torvalds committed
620
621
	mov r24=PAGE_SHIFT<<2
	;;
622
(p6)	cmp.eq p6,p7=r26,r18			// Only if page present
Linus Torvalds's avatar
Linus Torvalds committed
623
	;;
624
	ITC_I(p6, r25, r26)			// install updated PTE
Linus Torvalds's avatar
Linus Torvalds committed
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
	;;
	/*
	 * Tell the assemblers dependency-violation checker that the above "itc" instructions
	 * cannot possibly affect the following loads:
	 */
	dv_serialize_data

	ld8 r18=[r17]				// read PTE again
	;;
	cmp.eq p6,p7=r18,r25			// is it same as the newly installed
	;;
(p7)	ptc.l r16,r24
	mov b0=r29				// restore b0
	mov ar.ccv=r28
#else /* !CONFIG_SMP */
	;;
1:	ld8 r18=[r17]
	;;
	or r18=_PAGE_A,r18			// set the accessed bit
	mov b0=r29				// restore b0
	;;
	st8 [r17]=r18				// store back updated PTE
647
	ITC_I(p0, r18, r16)			// install updated PTE
Linus Torvalds's avatar
Linus Torvalds committed
648
649
#endif /* !CONFIG_SMP */
	mov pr=r31,-1
650
	RFI
Linus Torvalds's avatar
Linus Torvalds committed
651
652
653
654
655
656
657
658
END(iaccess_bit)

	.org ia64_ivt+0x2800
/////////////////////////////////////////////////////////////////////////////////////////
// 0x2800 Entry 10 (size 64 bundles) Data Access-bit (15,55)
ENTRY(daccess_bit)
	DBG_FAULT(10)
	// Like Entry 8, except for data access
659
	MOV_FROM_IFA(r16)			// get the address that caused the fault
Linus Torvalds's avatar
Linus Torvalds committed
660
661
	movl r30=1f				// load continuation point in case of nested fault
	;;
662
	THASH(p0, r17, r16, r18)		// compute virtual address of L3 PTE
Linus Torvalds's avatar
Linus Torvalds committed
663
664
665
666
667
668
669
670
671
	mov r31=pr
	mov r29=b0				// save b0 in case of nested fault)
#ifdef CONFIG_SMP
	mov r28=ar.ccv				// save ar.ccv
	;;
1:	ld8 r18=[r17]
	;;					// avoid RAW on r18
	mov ar.ccv=r18				// set compare value for cmpxchg
	or r25=_PAGE_A,r18			// set the dirty bit
672
	tbit.z p7,p6 = r18,_PAGE_P_BIT		// Check present bit
Linus Torvalds's avatar
Linus Torvalds committed
673
	;;
674
(p6)	cmpxchg8.acq r26=[r17],r25,ar.ccv	// Only if page is present
Linus Torvalds's avatar
Linus Torvalds committed
675
676
	mov r24=PAGE_SHIFT<<2
	;;
677
(p6)	cmp.eq p6,p7=r26,r18			// Only if page is present
Linus Torvalds's avatar
Linus Torvalds committed
678
	;;
679
	ITC_D(p6, r25, r26)			// install updated PTE
Linus Torvalds's avatar
Linus Torvalds committed
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
	/*
	 * Tell the assemblers dependency-violation checker that the above "itc" instructions
	 * cannot possibly affect the following loads:
	 */
	dv_serialize_data
	;;
	ld8 r18=[r17]				// read PTE again
	;;
	cmp.eq p6,p7=r18,r25			// is it same as the newly installed
	;;
(p7)	ptc.l r16,r24
	mov ar.ccv=r28
#else
	;;
1:	ld8 r18=[r17]
	;;					// avoid RAW on r18
	or r18=_PAGE_A,r18			// set the accessed bit
	;;
	st8 [r17]=r18				// store back updated PTE
699
	ITC_D(p0, r18, r16)			// install updated PTE
Linus Torvalds's avatar
Linus Torvalds committed
700
701
702
#endif
	mov b0=r29				// restore b0
	mov pr=r31,-1
703
	RFI
Linus Torvalds's avatar
Linus Torvalds committed
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
END(daccess_bit)

	.org ia64_ivt+0x2c00
/////////////////////////////////////////////////////////////////////////////////////////
// 0x2c00 Entry 11 (size 64 bundles) Break instruction (33)
ENTRY(break_fault)
	/*
	 * The streamlined system call entry/exit paths only save/restore the initial part
	 * of pt_regs.  This implies that the callers of system-calls must adhere to the
	 * normal procedure calling conventions.
	 *
	 *   Registers to be saved & restored:
	 *	CR registers: cr.ipsr, cr.iip, cr.ifs
	 *	AR registers: ar.unat, ar.pfs, ar.rsc, ar.rnat, ar.bspstore, ar.fpsr
	 * 	others: pr, b0, b6, loadrs, r1, r11, r12, r13, r15
	 *   Registers to be restored only:
	 * 	r8-r11: output value from the system call.
	 *
	 * During system call exit, scratch registers (including r15) are modified/cleared
	 * to prevent leaking bits from kernel to user level.
	 */
	DBG_FAULT(11)
726
	mov.m r16=IA64_KR(CURRENT)		// M2 r16 <- current task (12 cyc)
727
	MOV_FROM_IPSR(p0, r29)			// M2 (12 cyc)
728
	mov r31=pr				// I0 (2 cyc)
Linus Torvalds's avatar
Linus Torvalds committed
729

730
	MOV_FROM_IIM(r17)			// M2 (2 cyc)
731
732
	mov.m r27=ar.rsc			// M2 (12 cyc)
	mov r18=__IA64_BREAK_SYSCALL		// A
Linus Torvalds's avatar
Linus Torvalds committed
733

734
735
736
	mov.m ar.rsc=0				// M2
	mov.m r21=ar.fpsr			// M2 (12 cyc)
	mov r19=b6				// I0 (2 cyc)
Linus Torvalds's avatar
Linus Torvalds committed
737
	;;
738
739
740
	mov.m r23=ar.bspstore			// M2 (12 cyc)
	mov.m r24=ar.rnat			// M2 (5 cyc)
	mov.i r26=ar.pfs			// I0 (2 cyc)
Linus Torvalds's avatar
Linus Torvalds committed
741

742
743
744
745
746
747
748
	invala					// M0|1
	nop.m 0					// M
	mov r20=r1				// A			save r1

	nop.m 0
	movl r30=sys_call_table			// X

749
	MOV_FROM_IIP(r28)			// M2 (2 cyc)
750
751
752
753
754
755
756
757
758
759
760
761
762
	cmp.eq p0,p7=r18,r17			// I0 is this a system call?
(p7)	br.cond.spnt non_syscall		// B  no ->
	//
	// From this point on, we are definitely on the syscall-path
	// and we can use (non-banked) scratch registers.
	//
///////////////////////////////////////////////////////////////////////
	mov r1=r16				// A    move task-pointer to "addl"-addressable reg
	mov r2=r16				// A    setup r2 for ia64_syscall_setup
	add r9=TI_FLAGS+IA64_TASK_SIZE,r16	// A	r9 = &current_thread_info()->flags

	adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16
	adds r15=-1024,r15			// A    subtract 1024 from syscall number
Linus Torvalds's avatar
Linus Torvalds committed
763
764
	mov r3=NR_syscalls - 1
	;;
765
766
767
	ld1.bias r17=[r16]			// M0|1 r17 = current->thread.on_ustack flag
	ld4 r9=[r9]				// M0|1 r9 = current_thread_info()->flags
	extr.u r8=r29,41,2			// I0   extract ei field from cr.ipsr
Linus Torvalds's avatar
Linus Torvalds committed
768

769
770
771
	shladd r30=r15,3,r30			// A    r30 = sys_call_table + 8*(syscall-1024)
	addl r22=IA64_RBS_OFFSET,r1		// A    compute base of RBS
	cmp.leu p6,p7=r15,r3			// A    syscall number in range?
Linus Torvalds's avatar
Linus Torvalds committed
772
773
	;;

774
775
776
777
778
779
	lfetch.fault.excl.nt1 [r22]		// M0|1 prefetch RBS
(p6)	ld8 r30=[r30]				// M0|1 load address of syscall entry point
	tnat.nz.or p7,p0=r15			// I0	is syscall nr a NaT?

	mov.m ar.bspstore=r22			// M2   switch to kernel RBS
	cmp.eq p8,p9=2,r8			// A    isr.ei==2?
Linus Torvalds's avatar
Linus Torvalds committed
780
	;;
781
782
783
784
785
786

(p8)	mov r8=0				// A    clear ei to 0
(p7)	movl r30=sys_ni_syscall			// X

(p8)	adds r28=16,r28				// A    switch cr.iip to next bundle
(p9)	adds r8=1,r8				// A    increment ei to next slot
787
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
788
789
790
	;;
	mov b6=r30				// I0   setup syscall handler branch reg early
#else
791
	nop.i 0
Linus Torvalds's avatar
Linus Torvalds committed
792
	;;
793
#endif
794
795
796
797
798
799
800
801
802
803

	mov.m r25=ar.unat			// M2 (5 cyc)
	dep r29=r8,r29,41,2			// I0   insert new ei into cr.ipsr
	adds r15=1024,r15			// A    restore original syscall number
	//
	// If any of the above loads miss in L1D, we'll stall here until
	// the data arrives.
	//
///////////////////////////////////////////////////////////////////////
	st1 [r16]=r0				// M2|3 clear current->thread.on_ustack flag
804
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
805
	MOV_FROM_ITC(p0, p14, r30, r18)		// M    get cycle for accounting
806
#else
807
	mov b6=r30				// I0   setup syscall handler branch reg early
808
#endif
809
810
811
812
813
814
815
816
817
818
819
	cmp.eq pKStk,pUStk=r0,r17		// A    were we on kernel stacks already?

	and r9=_TIF_SYSCALL_TRACEAUDIT,r9	// A    mask trace or audit
	mov r18=ar.bsp				// M2 (12 cyc)
(pKStk)	br.cond.spnt .break_fixup		// B	we're already in kernel-mode -- fix up RBS
	;;
.back_from_break_fixup:
(pUStk)	addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r1 // A    compute base of memory stack
	cmp.eq p14,p0=r9,r0			// A    are syscalls being traced/audited?
	br.call.sptk.many b7=ia64_syscall_setup	// B
1:
820
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
	// mov.m r30=ar.itc is called in advance, and r13 is current
	add r16=TI_AC_STAMP+IA64_TASK_SIZE,r13	// A
	add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r13	// A
(pKStk)	br.cond.spnt .skip_accounting		// B	unlikely skip
	;;
	ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP	// M  get last stamp
	ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE	// M  time at leave
	;;
	ld8 r20=[r16],TI_AC_STAMP-TI_AC_STIME	// M  cumulated stime
	ld8 r21=[r17]				// M  cumulated utime
	sub r22=r19,r18				// A  stime before leave
	;;
	st8 [r16]=r30,TI_AC_STIME-TI_AC_STAMP	// M  update stamp
	sub r18=r30,r19				// A  elapsed time in user
	;;
	add r20=r20,r22				// A  sum stime
	add r21=r21,r18				// A  sum utime
	;;
	st8 [r16]=r20				// M  update stime
	st8 [r17]=r21				// M  update utime
	;;
.skip_accounting:
#endif
844
845
	mov ar.rsc=0x3				// M2   set eager mode, pl 0, LE, loadrs=0
	nop 0
846
	BSW_1(r2, r14)				// B (6 cyc) regs are saved, switch to bank 1
Linus Torvalds's avatar
Linus Torvalds committed
847
	;;
848

849
850
	SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r3, r16)	// M2	now it's safe to re-enable intr.-collection
						// M0   ensure interruption collection is on
851
	movl r3=ia64_ret_from_syscall		// X
Linus Torvalds's avatar
Linus Torvalds committed
852
	;;
853
854
855
	mov rp=r3				// I0   set the real return addr
(p10)	br.cond.spnt.many ia64_ret_from_syscall	// B    return if bad call-frame or r15 is a NaT

856
	SSM_PSR_I(p15, p15, r16)		// M2   restore psr.i
857
858
(p14)	br.call.sptk.many b6=b6			// B    invoke syscall-handker (ignore return addr)
	br.cond.spnt.many ia64_trace_syscall	// B	do syscall-tracing thingamagic
Linus Torvalds's avatar
Linus Torvalds committed
859
	// NOT REACHED
860
861
862
863
864
865
866
867
868
869
///////////////////////////////////////////////////////////////////////
	// On entry, we optimistically assumed that we're coming from user-space.
	// For the rare cases where a system-call is done from within the kernel,
	// we fix things up at this point:
.break_fixup:
	add r1=-IA64_PT_REGS_SIZE,sp		// A    allocate space for pt_regs structure
	mov ar.rnat=r24				// M2	restore kernel's AR.RNAT
	;;
	mov ar.bspstore=r23			// M2	restore kernel's AR.BSPSTORE
	br.cond.sptk .back_from_break_fixup
Linus Torvalds's avatar
Linus Torvalds committed
870
871
872
873
874
875
END(break_fault)

	.org ia64_ivt+0x3000
/////////////////////////////////////////////////////////////////////////////////////////
// 0x3000 Entry 12 (size 64 bundles) External Interrupt (4)
ENTRY(interrupt)
876
877
	/* interrupt handler has become too big to fit this area. */
	br.sptk.many __interrupt
Linus Torvalds's avatar
Linus Torvalds committed
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
END(interrupt)

	.org ia64_ivt+0x3400
/////////////////////////////////////////////////////////////////////////////////////////
// 0x3400 Entry 13 (size 64 bundles) Reserved
	DBG_FAULT(13)
	FAULT(13)

	.org ia64_ivt+0x3800
/////////////////////////////////////////////////////////////////////////////////////////
// 0x3800 Entry 14 (size 64 bundles) Reserved
	DBG_FAULT(14)
	FAULT(14)

	/*
	 * There is no particular reason for this code to be here, other than that
	 * there happens to be space here that would go unused otherwise.  If this
	 * fault ever gets "unreserved", simply moved the following code to a more
	 * suitable spot...
	 *
	 * ia64_syscall_setup() is a separate subroutine so that it can
	 *	allocate stacked registers so it can safely demine any
	 *	potential NaT values from the input registers.
	 *
	 * On entry:
	 *	- executing on bank 0 or bank 1 register set (doesn't matter)
	 *	-  r1: stack pointer
	 *	-  r2: current task pointer
	 *	-  r3: preserved
	 *	- r11: original contents (saved ar.pfs to be saved)
	 *	- r12: original contents (sp to be saved)
	 *	- r13: original contents (tp to be saved)
	 *	- r15: original contents (syscall # to be saved)
	 *	- r18: saved bsp (after switching to kernel stack)
	 *	- r19: saved b6
	 *	- r20: saved r1 (gp)
	 *	- r21: saved ar.fpsr
	 *	- r22: kernel's register backing store base (krbs_base)
	 *	- r23: saved ar.bspstore
	 *	- r24: saved ar.rnat
	 *	- r25: saved ar.unat
	 *	- r26: saved ar.pfs
	 *	- r27: saved ar.rsc
	 *	- r28: saved cr.iip
	 *	- r29: saved cr.ipsr
923
	 *	- r30: ar.itc for accounting (don't touch)
Linus Torvalds's avatar
Linus Torvalds committed
924
925
926
927
928
929
930
931
932
933
	 *	- r31: saved pr
	 *	-  b0: original contents (to be saved)
	 * On exit:
	 *	-  p10: TRUE if syscall is invoked with more than 8 out
	 *		registers or r15's Nat is true
	 *	-  r1: kernel's gp
	 *	-  r3: preserved (same as on entry)
	 *	-  r8: -EINVAL if p10 is true
	 *	- r12: points to kernel stack
	 *	- r13: points to current task
934
935
	 *	- r14: preserved (same as on entry)
	 *	- p13: preserved
Linus Torvalds's avatar
Linus Torvalds committed
936
937
	 *	- p15: TRUE if interrupts need to be re-enabled
	 *	- ar.fpsr: set to kernel settings
938
	 *	-  b6: preserved (same as on entry)
Linus Torvalds's avatar
Linus Torvalds committed
939
	 */
940
#ifdef __IA64_ASM_PARAVIRTUALIZED_NATIVE
Linus Torvalds's avatar
Linus Torvalds committed
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
GLOBAL_ENTRY(ia64_syscall_setup)
#if PT(B6) != 0
# error This code assumes that b6 is the first field in pt_regs.
#endif
	st8 [r1]=r19				// save b6
	add r16=PT(CR_IPSR),r1			// initialize first base pointer
	add r17=PT(R11),r1			// initialize second base pointer
	;;
	alloc r19=ar.pfs,8,0,0,0		// ensure in0-in7 are writable
	st8 [r16]=r29,PT(AR_PFS)-PT(CR_IPSR)	// save cr.ipsr
	tnat.nz p8,p0=in0

	st8.spill [r17]=r11,PT(CR_IIP)-PT(R11)	// save r11
	tnat.nz p9,p0=in1
(pKStk)	mov r18=r0				// make sure r18 isn't NaT
	;;

	st8 [r16]=r26,PT(CR_IFS)-PT(AR_PFS)	// save ar.pfs
	st8 [r17]=r28,PT(AR_UNAT)-PT(CR_IIP)	// save cr.iip
	mov r28=b0				// save b0 (2 cyc)
	;;

	st8 [r17]=r25,PT(AR_RSC)-PT(AR_UNAT)	// save ar.unat
	dep r19=0,r19,38,26			// clear all bits but 0..37 [I0]
(p8)	mov in0=-1
	;;

	st8 [r16]=r19,PT(AR_RNAT)-PT(CR_IFS)	// store ar.pfs.pfm in cr.ifs
	extr.u r11=r19,7,7	// I0		// get sol of ar.pfs
	and r8=0x7f,r19		// A		// get sof of ar.pfs

	st8 [r17]=r27,PT(AR_BSPSTORE)-PT(AR_RSC)// save ar.rsc
	tbit.nz p15,p0=r29,IA64_PSR_I_BIT // I0
(p9)	mov in1=-1
	;;

(pUStk) sub r18=r18,r22				// r18=RSE.ndirty*8
	tnat.nz p10,p0=in2
	add r11=8,r11
	;;
(pKStk) adds r16=PT(PR)-PT(AR_RNAT),r16		// skip over ar_rnat field
(pKStk) adds r17=PT(B0)-PT(AR_BSPSTORE),r17	// skip over ar_bspstore field
	tnat.nz p11,p0=in3
	;;
(p10)	mov in2=-1
	tnat.nz p12,p0=in4				// [I0]
(p11)	mov in3=-1
	;;
(pUStk) st8 [r16]=r24,PT(PR)-PT(AR_RNAT)	// save ar.rnat
(pUStk) st8 [r17]=r23,PT(B0)-PT(AR_BSPSTORE)	// save ar.bspstore
	shl r18=r18,16				// compute ar.rsc to be used for "loadrs"
	;;
	st8 [r16]=r31,PT(LOADRS)-PT(PR)		// save predicates
	st8 [r17]=r28,PT(R1)-PT(B0)		// save b0
	tnat.nz p13,p0=in5				// [I0]
	;;
	st8 [r16]=r18,PT(R12)-PT(LOADRS)	// save ar.rsc value for "loadrs"
	st8.spill [r17]=r20,PT(R13)-PT(R1)	// save original r1
(p12)	mov in4=-1
	;;

.mem.offset 0,0; st8.spill [r16]=r12,PT(AR_FPSR)-PT(R12)	// save r12
.mem.offset 8,0; st8.spill [r17]=r13,PT(R15)-PT(R13)		// save r13
(p13)	mov in5=-1
	;;
	st8 [r16]=r21,PT(R8)-PT(AR_FPSR)	// save ar.fpsr
1007
	tnat.nz p13,p0=in6
Linus Torvalds's avatar
Linus Torvalds committed
1008
1009
	cmp.lt p10,p9=r11,r8	// frame size can't be more than local+8
	;;
1010
	mov r8=1
Linus Torvalds's avatar
Linus Torvalds committed
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
(p9)	tnat.nz p10,p0=r15
	adds r12=-16,r1		// switch to kernel memory stack (with 16 bytes of scratch)

	st8.spill [r17]=r15			// save r15
	tnat.nz p8,p0=in7
	nop.i 0

	mov r13=r2				// establish `current'
	movl r1=__gp				// establish kernel global pointer
	;;
1021
	st8 [r16]=r8		// ensure pt_regs.r8 != 0 (see handle_syscall_error)
1022
(p13)	mov in6=-1
Linus Torvalds's avatar
Linus Torvalds committed
1023
1024
1025
1026
1027
1028
1029
1030
1031
(p8)	mov in7=-1

	cmp.eq pSys,pNonSys=r0,r0		// set pSys=1, pNonSys=0
	movl r17=FPSR_DEFAULT
	;;
	mov.m ar.fpsr=r17			// set ar.fpsr to kernel default value
(p10)	mov r8=-EINVAL
	br.ret.sptk.many b7
END(ia64_syscall_setup)
1032
#endif /* __IA64_ASM_PARAVIRTUALIZED_NATIVE */
Linus Torvalds's avatar
Linus Torvalds committed
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045

	.org ia64_ivt+0x3c00
/////////////////////////////////////////////////////////////////////////////////////////
// 0x3c00 Entry 15 (size 64 bundles) Reserved
	DBG_FAULT(15)
	FAULT(15)

	.org ia64_ivt+0x4000
/////////////////////////////////////////////////////////////////////////////////////////
// 0x4000 Entry 16 (size 64 bundles) Reserved
	DBG_FAULT(16)
	FAULT(16)

1046
#if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(__IA64_ASM_PARAVIRTUALIZED_NATIVE)
1047
1048
1049
1050
1051
1052
1053
1054
1055
	/*
	 * There is no particular reason for this code to be here, other than
	 * that there happens to be space here that would go unused otherwise.
	 * If this fault ever gets "unreserved", simply moved the following
	 * code to a more suitable spot...
	 *
	 * account_sys_enter is called from SAVE_MIN* macros if accounting is
	 * enabled and if the macro is entered from user mode.
	 */
1056
GLOBAL_ENTRY(account_sys_enter)
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
	// mov.m r20=ar.itc is called in advance, and r13 is current
	add r16=TI_AC_STAMP+IA64_TASK_SIZE,r13
	add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r13
	;;
	ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP	// time at last check in kernel
	ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE	// time at left from kernel
        ;;
	ld8 r23=[r16],TI_AC_STAMP-TI_AC_STIME	// cumulated stime
	ld8 r21=[r17]				// cumulated utime
	sub r22=r19,r18				// stime before leave kernel
	;;
	st8 [r16]=r20,TI_AC_STIME-TI_AC_STAMP	// update stamp
	sub r18=r20,r19				// elapsed time in user mode
	;;
	add r23=r23,r22				// sum stime
	add r21=r21,r18				// sum utime
	;;
	st8 [r16]=r23				// update stime
	st8 [r17]=r21				// update utime
	;;
	br.ret.sptk.many rp
END(account_sys_enter)
#endif

Linus Torvalds's avatar
Linus Torvalds committed
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
	.org ia64_ivt+0x4400
/////////////////////////////////////////////////////////////////////////////////////////
// 0x4400 Entry 17 (size 64 bundles) Reserved
	DBG_FAULT(17)
	FAULT(17)

	.org ia64_ivt+0x4800
/////////////////////////////////////////////////////////////////////////////////////////
// 0x4800 Entry 18 (size 64 bundles) Reserved
	DBG_FAULT(18)
	FAULT(18)

	.org ia64_ivt+0x4c00
/////////////////////////////////////////////////////////////////////////////////////////
// 0x4c00 Entry 19 (size 64 bundles) Reserved
	DBG_FAULT(19)
	FAULT(19)

//
// --- End of long entries, Beginning of short entries
//

	.org ia64_ivt+0x5000
/////////////////////////////////////////////////////////////////////////////////////////
// 0x5000 Entry 20 (size 16 bundles) Page Not Present (10,22,49)
ENTRY(page_not_present)
	DBG_FAULT(20)
1108
1109
	MOV_FROM_IFA(r16)
	RSM_PSR_DT
Linus Torvalds's avatar
Linus Torvalds committed
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
	/*
	 * The Linux page fault handler doesn't expect non-present pages to be in
	 * the TLB.  Flush the existing entry now, so we meet that expectation.
	 */
	mov r17=PAGE_SHIFT<<2
	;;
	ptc.l r16,r17
	;;
	mov r31=pr
	srlz.d
	br.sptk.many page_fault
END(page_not_present)

	.org ia64_ivt+0x5100
/////////////////////////////////////////////////////////////////////////////////////////
// 0x5100 Entry 21 (size 16 bundles) Key Permission (13,25,52)
ENTRY(key_permission)
	DBG_FAULT(21)
1128
1129
	MOV_FROM_IFA(r16)
	RSM_PSR_DT
Linus Torvalds's avatar
Linus Torvalds committed
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
	mov r31=pr
	;;
	srlz.d
	br.sptk.many page_fault
END(key_permission)

	.org ia64_ivt+0x5200
/////////////////////////////////////////////////////////////////////////////////////////
// 0x5200 Entry 22 (size 16 bundles) Instruction Access Rights (26)
ENTRY(iaccess_rights)
	DBG_FAULT(22)
1141
1142
	MOV_FROM_IFA(r16)
	RSM_PSR_DT
Linus Torvalds's avatar
Linus Torvalds committed
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
	mov r31=pr
	;;
	srlz.d
	br.sptk.many page_fault
END(iaccess_rights)

	.org ia64_ivt+0x5300
/////////////////////////////////////////////////////////////////////////////////////////
// 0x5300 Entry 23 (size 16 bundles) Data Access Rights (14,53)
ENTRY(daccess_rights)
	DBG_FAULT(23)
1154
1155
	MOV_FROM_IFA(r16)
	RSM_PSR_DT
Linus Torvalds's avatar
Linus Torvalds committed
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
	mov r31=pr
	;;
	srlz.d
	br.sptk.many page_fault
END(daccess_rights)

	.org ia64_ivt+0x5400
/////////////////////////////////////////////////////////////////////////////////////////
// 0x5400 Entry 24 (size 16 bundles) General Exception (5,32,34,36,38,39)
ENTRY(general_exception)
	DBG_FAULT(24)
1167
	MOV_FROM_ISR(r16)
Linus Torvalds's avatar
Linus Torvalds committed
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
	mov r31=pr
	;;
	cmp4.eq p6,p0=0,r16
(p6)	br.sptk.many dispatch_illegal_op_fault
	;;
	mov r19=24		// fault number
	br.sptk.many dispatch_to_fault_handler
END(general_exception)

	.org ia64_ivt+0x5500
/////////////////////////////////////////////////////////////////////////////////////////
// 0x5500 Entry 25 (size 16 bundles) Disabled FP-Register (35)
ENTRY(disabled_fp_reg)
	DBG_FAULT(25)
	rsm psr.dfh		// ensure we can access fph
	;;
	srlz.d
	mov r31=pr
	mov r19=25
	br.sptk.many dispatch_to_fault_handler
END(disabled_fp_reg)

	.org ia64_ivt+0x5600
/////////////////////////////////////////////////////////////////////////////////////////
// 0x5600 Entry 26 (size 16 bundles) Nat Consumption (11,23,37,50)
ENTRY(nat_consumption)
	DBG_FAULT(26)
1195

1196
1197
	MOV_FROM_IPSR(p0, r16)
	MOV_FROM_ISR(r17)
1198
1199
1200
1201
1202
1203
1204
1205
1206
	mov r31=pr				// save PR
	;;
	and r18=0xf,r17				// r18 = cr.ipsr.code{3:0}
	tbit.z p6,p0=r17,IA64_ISR_NA_BIT
	;;
	cmp.ne.or p6,p0=IA64_ISR_CODE_LFETCH,r18
	dep r16=-1,r16,IA64_PSR_ED_BIT,1
(p6)	br.cond.spnt 1f		// branch if (cr.ispr.na == 0 || cr.ipsr.code{3:0} != LFETCH)
	;;
1207
	MOV_TO_IPSR(p0, r16, r18)
1208
1209
	mov pr=r31,-1
	;;
1210
	RFI
1211
1212
1213

1:	mov pr=r31,-1
	;;
Linus Torvalds's avatar
Linus Torvalds committed
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
	FAULT(26)
END(nat_consumption)

	.org ia64_ivt+0x5700
/////////////////////////////////////////////////////////////////////////////////////////
// 0x5700 Entry 27 (size 16 bundles) Speculation (40)
ENTRY(speculation_vector)
	DBG_FAULT(27)
	/*
	 * A [f]chk.[as] instruction needs to take the branch to the recovery code but
	 * this part of the architecture is not implemented in hardware on some CPUs, such
	 * as Itanium.  Thus, in general we need to emulate the behavior.  IIM contains
	 * the relative target (not yet sign extended).  So after sign extending it we
	 * simply add it to IIP.  We also need to reset the EI field of the IPSR to zero,
	 * i.e., the slot to restart into.
	 *
	 * cr.imm contains zero_ext(imm21)
	 */
1232
	MOV_FROM_IIM(r18)
Linus Torvalds's avatar
Linus Torvalds committed
1233
	;;
1234
	MOV_FROM_IIP(r17)
Linus Torvalds's avatar
Linus Torvalds committed
1235
1236
1237
	shl r18=r18,43			// put sign bit in position (43=64-21)
	;;

1238
	MOV_FROM_IPSR(p0, r16)
Linus Torvalds's avatar
Linus Torvalds committed
1239
1240
1241
1242
1243
	shr r18=r18,39			// sign extend (39=43-4)
	;;

	add r17=r17,r18			// now add the offset
	;;
1244
	MOV_TO_IIP(r17, r19)
Linus Torvalds's avatar
Linus Torvalds committed
1245
1246
1247
	dep r16=0,r16,41,2		// clear EI
	;;

1248
	MOV_TO_IPSR(p0, r16, r19)
Linus Torvalds's avatar
Linus Torvalds committed
1249
1250
	;;

1251
	RFI
Linus Torvalds's avatar
Linus Torvalds committed
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397