spinlock.h 11.9 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
6
 * Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org)
Linus Torvalds's avatar
Linus Torvalds committed
7
8
9
10
11
 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
 */
#ifndef _ASM_SPINLOCK_H
#define _ASM_SPINLOCK_H

12
13
#include <linux/compiler.h>

14
#include <asm/barrier.h>
Linus Torvalds's avatar
Linus Torvalds committed
15
16
17
18
#include <asm/war.h>

/*
 * Your basic SMP spinlocks, allowing only a single CPU anywhere
19
20
21
22
23
24
25
 *
 * Simple spin lock operations.  There are two variants, one clears IRQ's
 * on the local processor, one does not.
 *
 * These are fair FIFO ticket locks
 *
 * (the type definitions are in asm/spinlock_types.h)
Linus Torvalds's avatar
Linus Torvalds committed
26
27
28
29
 */


/*
30
31
32
33
34
 * Ticket locks are conceptually two parts, one indicating the current head of
 * the queue, and the other indicating the current tail. The lock is acquired
 * by atomically noting the tail and incrementing it by one (thus adding
 * ourself to the queue and noting our position), then waiting until the head
 * becomes equal to the the initial value of the tail.
Linus Torvalds's avatar
Linus Torvalds committed
35
36
 */

37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
static inline int __raw_spin_is_locked(raw_spinlock_t *lock)
{
	unsigned int counters = ACCESS_ONCE(lock->lock);

	return ((counters >> 14) ^ counters) & 0x1fff;
}

#define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock)
#define __raw_spin_unlock_wait(x) \
	while (__raw_spin_is_locked(x)) { cpu_relax(); }

static inline int __raw_spin_is_contended(raw_spinlock_t *lock)
{
	unsigned int counters = ACCESS_ONCE(lock->lock);

	return (((counters >> 14) - counters) & 0x1fff) > 1;
}

Ingo Molnar's avatar
Ingo Molnar committed
55
static inline void __raw_spin_lock(raw_spinlock_t *lock)
Linus Torvalds's avatar
Linus Torvalds committed
56
{
57
58
	int my_ticket;
	int tmp;
Linus Torvalds's avatar
Linus Torvalds committed
59
60

	if (R10000_LLSC_WAR) {
61
62
63
64
65
66
67
68
		__asm__ __volatile__ (
		"	.set push		# __raw_spin_lock	\n"
		"	.set noreorder					\n"
		"							\n"
		"1:	ll	%[ticket], %[ticket_ptr]		\n"
		"	addiu	%[my_ticket], %[ticket], 0x4000		\n"
		"	sc	%[my_ticket], %[ticket_ptr]		\n"
		"	beqzl	%[my_ticket], 1b			\n"
Linus Torvalds's avatar
Linus Torvalds committed
69
		"	 nop						\n"
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
		"	srl	%[my_ticket], %[ticket], 14		\n"
		"	andi	%[my_ticket], %[my_ticket], 0x1fff	\n"
		"	andi	%[ticket], %[ticket], 0x1fff		\n"
		"	bne	%[ticket], %[my_ticket], 4f		\n"
		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
		"2:							\n"
		"	.subsection 2					\n"
		"4:	andi	%[ticket], %[ticket], 0x1fff		\n"
		"5:	sll	%[ticket], 5				\n"
		"							\n"
		"6:	bnez	%[ticket], 6b				\n"
		"	 subu	%[ticket], 1				\n"
		"							\n"
		"	lw	%[ticket], %[ticket_ptr]		\n"
		"	andi	%[ticket], %[ticket], 0x1fff		\n"
		"	beq	%[ticket], %[my_ticket], 2b		\n"
		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
		"	b	5b					\n"
		"	 subu	%[ticket], %[ticket], 1			\n"
		"	.previous					\n"
		"	.set pop					\n"
		: [ticket_ptr] "+m" (lock->lock),
		  [ticket] "=&r" (tmp),
		  [my_ticket] "=&r" (my_ticket));
Linus Torvalds's avatar
Linus Torvalds committed
94
	} else {
95
96
97
98
99
100
101
102
		__asm__ __volatile__ (
		"	.set push		# __raw_spin_lock	\n"
		"	.set noreorder					\n"
		"							\n"
		"	ll	%[ticket], %[ticket_ptr]		\n"
		"1:	addiu	%[my_ticket], %[ticket], 0x4000		\n"
		"	sc	%[my_ticket], %[ticket_ptr]		\n"
		"	beqz	%[my_ticket], 3f			\n"
103
		"	 nop						\n"
104
105
106
107
108
109
		"	srl	%[my_ticket], %[ticket], 14		\n"
		"	andi	%[my_ticket], %[my_ticket], 0x1fff	\n"
		"	andi	%[ticket], %[ticket], 0x1fff		\n"
		"	bne	%[ticket], %[my_ticket], 4f		\n"
		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
		"2:							\n"
110
		"	.subsection 2					\n"
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
		"3:	b	1b					\n"
		"	 ll	%[ticket], %[ticket_ptr]		\n"
		"							\n"
		"4:	andi	%[ticket], %[ticket], 0x1fff		\n"
		"5:	sll	%[ticket], 5				\n"
		"							\n"
		"6:	bnez	%[ticket], 6b				\n"
		"	 subu	%[ticket], 1				\n"
		"							\n"
		"	lw	%[ticket], %[ticket_ptr]		\n"
		"	andi	%[ticket], %[ticket], 0x1fff		\n"
		"	beq	%[ticket], %[my_ticket], 2b		\n"
		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
		"	b	5b					\n"
		"	 subu	%[ticket], %[ticket], 1			\n"
126
		"	.previous					\n"
127
128
129
130
		"	.set pop					\n"
		: [ticket_ptr] "+m" (lock->lock),
		  [ticket] "=&r" (tmp),
		  [my_ticket] "=&r" (my_ticket));
Linus Torvalds's avatar
Linus Torvalds committed
131
	}
132

133
	smp_llsc_mb();
Linus Torvalds's avatar
Linus Torvalds committed
134
135
}

Ingo Molnar's avatar
Ingo Molnar committed
136
static inline void __raw_spin_unlock(raw_spinlock_t *lock)
Linus Torvalds's avatar
Linus Torvalds committed
137
{
138
139
140
141
142
143
144
145
146
147
148
149
	int tmp;

	smp_llsc_mb();

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__ (
		"				# __raw_spin_unlock	\n"
		"1:	ll	%[ticket], %[ticket_ptr]		\n"
		"	addiu	%[ticket], %[ticket], 1			\n"
		"	ori	%[ticket], %[ticket], 0x2000		\n"
		"	xori	%[ticket], %[ticket], 0x2000		\n"
		"	sc	%[ticket], %[ticket_ptr]		\n"
150
		"	beqzl	%[ticket], 1b				\n"
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
		: [ticket_ptr] "+m" (lock->lock),
		  [ticket] "=&r" (tmp));
	} else {
		__asm__ __volatile__ (
		"	.set push		# __raw_spin_unlock	\n"
		"	.set noreorder					\n"
		"							\n"
		"	ll	%[ticket], %[ticket_ptr]		\n"
		"1:	addiu	%[ticket], %[ticket], 1			\n"
		"	ori	%[ticket], %[ticket], 0x2000		\n"
		"	xori	%[ticket], %[ticket], 0x2000		\n"
		"	sc	%[ticket], %[ticket_ptr]		\n"
		"	beqz	%[ticket], 2f				\n"
		"	 nop						\n"
		"							\n"
		"	.subsection 2					\n"
		"2:	b	1b					\n"
		"	 ll	%[ticket], %[ticket_ptr]		\n"
		"	.previous					\n"
		"	.set pop					\n"
		: [ticket_ptr] "+m" (lock->lock),
		  [ticket] "=&r" (tmp));
	}
Linus Torvalds's avatar
Linus Torvalds committed
174
175
}

Ingo Molnar's avatar
Ingo Molnar committed
176
static inline unsigned int __raw_spin_trylock(raw_spinlock_t *lock)
Linus Torvalds's avatar
Linus Torvalds committed
177
{
178
	int tmp, tmp2, tmp3;
Linus Torvalds's avatar
Linus Torvalds committed
179
180

	if (R10000_LLSC_WAR) {
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
		__asm__ __volatile__ (
		"	.set push		# __raw_spin_trylock	\n"
		"	.set noreorder					\n"
		"							\n"
		"1:	ll	%[ticket], %[ticket_ptr]		\n"
		"	srl	%[my_ticket], %[ticket], 14		\n"
		"	andi	%[my_ticket], %[my_ticket], 0x1fff	\n"
		"	andi	%[now_serving], %[ticket], 0x1fff	\n"
		"	bne	%[my_ticket], %[now_serving], 3f	\n"
		"	 addiu	%[ticket], %[ticket], 0x4000		\n"
		"	sc	%[ticket], %[ticket_ptr]		\n"
		"	beqzl	%[ticket], 1b				\n"
		"	 li	%[ticket], 1				\n"
		"2:							\n"
		"	.subsection 2					\n"
		"3:	b	2b					\n"
		"	 li	%[ticket], 0				\n"
		"	.previous					\n"
		"	.set pop					\n"
		: [ticket_ptr] "+m" (lock->lock),
		  [ticket] "=&r" (tmp),
		  [my_ticket] "=&r" (tmp2),
		  [now_serving] "=&r" (tmp3));
Linus Torvalds's avatar
Linus Torvalds committed
204
	} else {
205
206
207
208
209
210
211
212
213
214
215
216
217
218
		__asm__ __volatile__ (
		"	.set push		# __raw_spin_trylock	\n"
		"	.set noreorder					\n"
		"							\n"
		"	ll	%[ticket], %[ticket_ptr]		\n"
		"1:	srl	%[my_ticket], %[ticket], 14		\n"
		"	andi	%[my_ticket], %[my_ticket], 0x1fff	\n"
		"	andi	%[now_serving], %[ticket], 0x1fff	\n"
		"	bne	%[my_ticket], %[now_serving], 3f	\n"
		"	 addiu	%[ticket], %[ticket], 0x4000		\n"
		"	sc	%[ticket], %[ticket_ptr]		\n"
		"	beqz	%[ticket], 4f				\n"
		"	 li	%[ticket], 1				\n"
		"2:							\n"
219
		"	.subsection 2					\n"
220
221
222
223
		"3:	b	2b					\n"
		"	 li	%[ticket], 0				\n"
		"4:	b	1b					\n"
		"	 ll	%[ticket], %[ticket_ptr]		\n"
224
		"	.previous					\n"
225
226
227
228
229
		"	.set pop					\n"
		: [ticket_ptr] "+m" (lock->lock),
		  [ticket] "=&r" (tmp),
		  [my_ticket] "=&r" (tmp2),
		  [now_serving] "=&r" (tmp3));
Linus Torvalds's avatar
Linus Torvalds committed
230
231
	}

232
	smp_llsc_mb();
233

234
	return tmp;
Linus Torvalds's avatar
Linus Torvalds committed
235
236
237
238
239
240
241
242
243
244
245
}

/*
 * Read-write spinlocks, allowing multiple readers but only one writer.
 *
 * NOTE! it is quite common to have readers in interrupts but no interrupt
 * writers. For those circumstances we can "mix" irq-safe locks - any writer
 * needs to get a irq-safe write-lock, but readers can get non-irqsafe
 * read-locks.
 */

246
247
248
249
250
251
252
253
254
255
256
257
/*
 * read_can_lock - would read_trylock() succeed?
 * @lock: the rwlock in question.
 */
#define __raw_read_can_lock(rw)	((rw)->lock >= 0)

/*
 * write_can_lock - would write_trylock() succeed?
 * @lock: the rwlock in question.
 */
#define __raw_write_can_lock(rw)	(!(rw)->lock)

Ingo Molnar's avatar
Ingo Molnar committed
258
static inline void __raw_read_lock(raw_rwlock_t *rw)
Linus Torvalds's avatar
Linus Torvalds committed
259
260
261
262
263
{
	unsigned int tmp;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__(
Ingo Molnar's avatar
Ingo Molnar committed
264
		"	.set	noreorder	# __raw_read_lock	\n"
Linus Torvalds's avatar
Linus Torvalds committed
265
266
267
268
269
270
271
272
273
274
275
276
		"1:	ll	%1, %2					\n"
		"	bltz	%1, 1b					\n"
		"	 addu	%1, 1					\n"
		"	sc	%1, %0					\n"
		"	beqzl	%1, 1b					\n"
		"	 nop						\n"
		"	.set	reorder					\n"
		: "=m" (rw->lock), "=&r" (tmp)
		: "m" (rw->lock)
		: "memory");
	} else {
		__asm__ __volatile__(
Ingo Molnar's avatar
Ingo Molnar committed
277
		"	.set	noreorder	# __raw_read_lock	\n"
Linus Torvalds's avatar
Linus Torvalds committed
278
		"1:	ll	%1, %2					\n"
279
		"	bltz	%1, 2f					\n"
Linus Torvalds's avatar
Linus Torvalds committed
280
281
282
		"	 addu	%1, 1					\n"
		"	sc	%1, %0					\n"
		"	beqz	%1, 1b					\n"
283
		"	 nop						\n"
284
285
286
287
288
289
290
		"	.subsection 2					\n"
		"2:	ll	%1, %2					\n"
		"	bltz	%1, 2b					\n"
		"	 addu	%1, 1					\n"
		"	b	1b					\n"
		"	 nop						\n"
		"	.previous					\n"
Linus Torvalds's avatar
Linus Torvalds committed
291
292
293
294
295
		"	.set	reorder					\n"
		: "=m" (rw->lock), "=&r" (tmp)
		: "m" (rw->lock)
		: "memory");
	}
296

297
	smp_llsc_mb();
Linus Torvalds's avatar
Linus Torvalds committed
298
299
300
301
302
}

/* Note the use of sub, not subu which will make the kernel die with an
   overflow exception if we ever try to unlock an rwlock that is already
   unlocked or is being held by a writer.  */
Ingo Molnar's avatar
Ingo Molnar committed
303
static inline void __raw_read_unlock(raw_rwlock_t *rw)
Linus Torvalds's avatar
Linus Torvalds committed
304
305
306
{
	unsigned int tmp;

307
	smp_llsc_mb();
308

Linus Torvalds's avatar
Linus Torvalds committed
309
310
	if (R10000_LLSC_WAR) {
		__asm__ __volatile__(
Ingo Molnar's avatar
Ingo Molnar committed
311
		"1:	ll	%1, %2		# __raw_read_unlock	\n"
Linus Torvalds's avatar
Linus Torvalds committed
312
313
314
315
316
317
318
319
		"	sub	%1, 1					\n"
		"	sc	%1, %0					\n"
		"	beqzl	%1, 1b					\n"
		: "=m" (rw->lock), "=&r" (tmp)
		: "m" (rw->lock)
		: "memory");
	} else {
		__asm__ __volatile__(
Ingo Molnar's avatar
Ingo Molnar committed
320
		"	.set	noreorder	# __raw_read_unlock	\n"
Linus Torvalds's avatar
Linus Torvalds committed
321
322
323
		"1:	ll	%1, %2					\n"
		"	sub	%1, 1					\n"
		"	sc	%1, %0					\n"
324
325
326
327
		"	beqz	%1, 2f					\n"
		"	 nop						\n"
		"	.subsection 2					\n"
		"2:	b	1b					\n"
328
		"	 nop						\n"
329
		"	.previous					\n"
Linus Torvalds's avatar
Linus Torvalds committed
330
331
332
333
334
335
336
		"	.set	reorder					\n"
		: "=m" (rw->lock), "=&r" (tmp)
		: "m" (rw->lock)
		: "memory");
	}
}

Ingo Molnar's avatar
Ingo Molnar committed
337
static inline void __raw_write_lock(raw_rwlock_t *rw)
Linus Torvalds's avatar
Linus Torvalds committed
338
339
340
341
342
{
	unsigned int tmp;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__(
Ingo Molnar's avatar
Ingo Molnar committed
343
		"	.set	noreorder	# __raw_write_lock	\n"
Linus Torvalds's avatar
Linus Torvalds committed
344
345
346
347
348
		"1:	ll	%1, %2					\n"
		"	bnez	%1, 1b					\n"
		"	 lui	%1, 0x8000				\n"
		"	sc	%1, %0					\n"
		"	beqzl	%1, 1b					\n"
349
		"	 nop						\n"
Linus Torvalds's avatar
Linus Torvalds committed
350
351
352
353
354
355
		"	.set	reorder					\n"
		: "=m" (rw->lock), "=&r" (tmp)
		: "m" (rw->lock)
		: "memory");
	} else {
		__asm__ __volatile__(
Ingo Molnar's avatar
Ingo Molnar committed
356
		"	.set	noreorder	# __raw_write_lock	\n"
Linus Torvalds's avatar
Linus Torvalds committed
357
		"1:	ll	%1, %2					\n"
358
		"	bnez	%1, 2f					\n"
Linus Torvalds's avatar
Linus Torvalds committed
359
360
		"	 lui	%1, 0x8000				\n"
		"	sc	%1, %0					\n"
361
362
363
364
365
366
367
		"	beqz	%1, 2f					\n"
		"	 nop						\n"
		"	.subsection 2					\n"
		"2:	ll	%1, %2					\n"
		"	bnez	%1, 2b					\n"
		"	 lui	%1, 0x8000				\n"
		"	b	1b					\n"
368
		"	 nop						\n"
369
		"	.previous					\n"
Linus Torvalds's avatar
Linus Torvalds committed
370
371
372
373
374
		"	.set	reorder					\n"
		: "=m" (rw->lock), "=&r" (tmp)
		: "m" (rw->lock)
		: "memory");
	}
375

376
	smp_llsc_mb();
Linus Torvalds's avatar
Linus Torvalds committed
377
378
}

Ingo Molnar's avatar
Ingo Molnar committed
379
static inline void __raw_write_unlock(raw_rwlock_t *rw)
Linus Torvalds's avatar
Linus Torvalds committed
380
{
381
382
	smp_mb();

Linus Torvalds's avatar
Linus Torvalds committed
383
	__asm__ __volatile__(
384
	"				# __raw_write_unlock	\n"
Linus Torvalds's avatar
Linus Torvalds committed
385
386
387
388
389
390
	"	sw	$0, %0					\n"
	: "=m" (rw->lock)
	: "m" (rw->lock)
	: "memory");
}

391
392
393
394
395
396
397
398
399
400
static inline int __raw_read_trylock(raw_rwlock_t *rw)
{
	unsigned int tmp;
	int ret;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__(
		"	.set	noreorder	# __raw_read_trylock	\n"
		"	li	%2, 0					\n"
		"1:	ll	%1, %3					\n"
401
		"	bltz	%1, 2f					\n"
402
403
404
		"	 addu	%1, 1					\n"
		"	sc	%1, %0					\n"
		"	.set	reorder					\n"
405
406
		"	beqzl	%1, 1b					\n"
		"	 nop						\n"
407
		__WEAK_LLSC_MB
408
409
410
411
412
413
414
415
416
417
		"	li	%2, 1					\n"
		"2:							\n"
		: "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
		: "m" (rw->lock)
		: "memory");
	} else {
		__asm__ __volatile__(
		"	.set	noreorder	# __raw_read_trylock	\n"
		"	li	%2, 0					\n"
		"1:	ll	%1, %3					\n"
418
		"	bltz	%1, 2f					\n"
419
420
421
		"	 addu	%1, 1					\n"
		"	sc	%1, %0					\n"
		"	beqz	%1, 1b					\n"
422
		"	 nop						\n"
423
		"	.set	reorder					\n"
424
		__WEAK_LLSC_MB
425
426
427
428
429
430
431
432
433
		"	li	%2, 1					\n"
		"2:							\n"
		: "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
		: "m" (rw->lock)
		: "memory");
	}

	return ret;
}
Linus Torvalds's avatar
Linus Torvalds committed
434

Ingo Molnar's avatar
Ingo Molnar committed
435
static inline int __raw_write_trylock(raw_rwlock_t *rw)
Linus Torvalds's avatar
Linus Torvalds committed
436
437
438
439
440
441
{
	unsigned int tmp;
	int ret;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__(
Ingo Molnar's avatar
Ingo Molnar committed
442
		"	.set	noreorder	# __raw_write_trylock	\n"
Linus Torvalds's avatar
Linus Torvalds committed
443
444
445
446
447
448
		"	li	%2, 0					\n"
		"1:	ll	%1, %3					\n"
		"	bnez	%1, 2f					\n"
		"	 lui	%1, 0x8000				\n"
		"	sc	%1, %0					\n"
		"	beqzl	%1, 1b					\n"
449
		"	 nop						\n"
450
		__WEAK_LLSC_MB
Linus Torvalds's avatar
Linus Torvalds committed
451
452
453
454
455
456
457
458
		"	li	%2, 1					\n"
		"	.set	reorder					\n"
		"2:							\n"
		: "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
		: "m" (rw->lock)
		: "memory");
	} else {
		__asm__ __volatile__(
Ingo Molnar's avatar
Ingo Molnar committed
459
		"	.set	noreorder	# __raw_write_trylock	\n"
Linus Torvalds's avatar
Linus Torvalds committed
460
461
462
463
464
		"	li	%2, 0					\n"
		"1:	ll	%1, %3					\n"
		"	bnez	%1, 2f					\n"
		"	lui	%1, 0x8000				\n"
		"	sc	%1, %0					\n"
465
466
467
		"	beqz	%1, 3f					\n"
		"	 li	%2, 1					\n"
		"2:							\n"
468
		__WEAK_LLSC_MB
469
470
471
472
		"	.subsection 2					\n"
		"3:	b	1b					\n"
		"	 li	%2, 0					\n"
		"	.previous					\n"
Linus Torvalds's avatar
Linus Torvalds committed
473
474
475
476
477
478
479
480
481
		"	.set	reorder					\n"
		: "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
		: "m" (rw->lock)
		: "memory");
	}

	return ret;
}

482

483
484
485
486
#define _raw_spin_relax(lock)	cpu_relax()
#define _raw_read_relax(lock)	cpu_relax()
#define _raw_write_relax(lock)	cpu_relax()

Linus Torvalds's avatar
Linus Torvalds committed
487
#endif /* _ASM_SPINLOCK_H */