cmpxchg.h 3.93 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
/*
 * Based on arch/arm/include/asm/cmpxchg.h
 *
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#ifndef __ASM_CMPXCHG_H
#define __ASM_CMPXCHG_H

#include <linux/bug.h>

#include <asm/barrier.h>

static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
{
	unsigned long ret, tmp;

	switch (size) {
	case 1:
		asm volatile("//	__xchg1\n"
32
33
		"1:	ldaxrb	%w0, %2\n"
		"	stlxrb	%w1, %w3, %2\n"
34
		"	cbnz	%w1, 1b\n"
35
36
37
			: "=&r" (ret), "=&r" (tmp), "+Q" (*(u8 *)ptr)
			: "r" (x)
			: "cc", "memory");
38
39
40
		break;
	case 2:
		asm volatile("//	__xchg2\n"
41
42
		"1:	ldaxrh	%w0, %2\n"
		"	stlxrh	%w1, %w3, %2\n"
43
		"	cbnz	%w1, 1b\n"
44
45
46
			: "=&r" (ret), "=&r" (tmp), "+Q" (*(u16 *)ptr)
			: "r" (x)
			: "cc", "memory");
47
48
49
		break;
	case 4:
		asm volatile("//	__xchg4\n"
50
51
		"1:	ldaxr	%w0, %2\n"
		"	stlxr	%w1, %w3, %2\n"
52
		"	cbnz	%w1, 1b\n"
53
54
55
			: "=&r" (ret), "=&r" (tmp), "+Q" (*(u32 *)ptr)
			: "r" (x)
			: "cc", "memory");
56
57
58
		break;
	case 8:
		asm volatile("//	__xchg8\n"
59
60
		"1:	ldaxr	%0, %2\n"
		"	stlxr	%w1, %3, %2\n"
61
		"	cbnz	%w1, 1b\n"
62
63
64
			: "=&r" (ret), "=&r" (tmp), "+Q" (*(u64 *)ptr)
			: "r" (x)
			: "cc", "memory");
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
		break;
	default:
		BUILD_BUG();
	}

	return ret;
}

#define xchg(ptr,x) \
	((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr))))

static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
				      unsigned long new, int size)
{
	unsigned long oldval = 0, res;

	switch (size) {
	case 1:
		do {
			asm volatile("// __cmpxchg1\n"
85
			"	ldxrb	%w1, %2\n"
86
87
88
			"	mov	%w0, #0\n"
			"	cmp	%w1, %w3\n"
			"	b.ne	1f\n"
89
			"	stxrb	%w0, %w4, %2\n"
90
			"1:\n"
91
92
				: "=&r" (res), "=&r" (oldval), "+Q" (*(u8 *)ptr)
				: "Ir" (old), "r" (new)
93
94
95
96
97
98
99
				: "cc");
		} while (res);
		break;

	case 2:
		do {
			asm volatile("// __cmpxchg2\n"
100
			"	ldxrh	%w1, %2\n"
101
102
103
			"	mov	%w0, #0\n"
			"	cmp	%w1, %w3\n"
			"	b.ne	1f\n"
104
			"	stxrh	%w0, %w4, %2\n"
105
			"1:\n"
106
107
108
				: "=&r" (res), "=&r" (oldval), "+Q" (*(u16 *)ptr)
				: "Ir" (old), "r" (new)
				: "cc");
109
110
111
112
113
114
		} while (res);
		break;

	case 4:
		do {
			asm volatile("// __cmpxchg4\n"
115
			"	ldxr	%w1, %2\n"
116
117
118
			"	mov	%w0, #0\n"
			"	cmp	%w1, %w3\n"
			"	b.ne	1f\n"
119
			"	stxr	%w0, %w4, %2\n"
120
			"1:\n"
121
122
				: "=&r" (res), "=&r" (oldval), "+Q" (*(u32 *)ptr)
				: "Ir" (old), "r" (new)
123
124
125
126
127
128
129
				: "cc");
		} while (res);
		break;

	case 8:
		do {
			asm volatile("// __cmpxchg8\n"
130
			"	ldxr	%1, %2\n"
131
132
133
			"	mov	%w0, #0\n"
			"	cmp	%1, %3\n"
			"	b.ne	1f\n"
134
			"	stxr	%w0, %4, %2\n"
135
			"1:\n"
136
137
				: "=&r" (res), "=&r" (oldval), "+Q" (*(u64 *)ptr)
				: "Ir" (old), "r" (new)
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
				: "cc");
		} while (res);
		break;

	default:
		BUILD_BUG();
	}

	return oldval;
}

static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
					 unsigned long new, int size)
{
	unsigned long ret;

	smp_mb();
	ret = __cmpxchg(ptr, old, new, size);
	smp_mb();

	return ret;
}

161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
#define cmpxchg(ptr, o, n) \
({ \
	__typeof__(*(ptr)) __ret; \
	__ret = (__typeof__(*(ptr))) \
		__cmpxchg_mb((ptr), (unsigned long)(o), (unsigned long)(n), \
			     sizeof(*(ptr))); \
	__ret; \
})

#define cmpxchg_local(ptr, o, n) \
({ \
	__typeof__(*(ptr)) __ret; \
	__ret = (__typeof__(*(ptr))) \
		__cmpxchg((ptr), (unsigned long)(o), \
			  (unsigned long)(n), sizeof(*(ptr))); \
	__ret; \
})
178

179
180
181
#define cmpxchg64(ptr,o,n)		cmpxchg((ptr),(o),(n))
#define cmpxchg64_local(ptr,o,n)	cmpxchg_local((ptr),(o),(n))

182
183
#define cmpxchg64_relaxed(ptr,o,n)	cmpxchg_local((ptr),(o),(n))

184
#endif	/* __ASM_CMPXCHG_H */