0
|
1
|
|
2 /*
|
|
3 * Copyright (C) Igor Sysoev
|
|
4 */
|
|
5
|
|
6
|
|
7 #ifndef _NGX_ATOMIC_H_INCLUDED_
|
|
8 #define _NGX_ATOMIC_H_INCLUDED_
|
|
9
|
|
10
|
|
11 #include <ngx_config.h>
|
|
12 #include <ngx_core.h>
|
|
13
|
|
14
|
96
|
15 #if ( __i386__ || __i386 )
|
0
|
16
|
|
17 #define NGX_HAVE_ATOMIC_OPS 1
|
|
18
|
44
|
19 typedef int32_t ngx_atomic_int_t;
|
|
20 typedef uint32_t ngx_atomic_uint_t;
|
|
21 typedef volatile ngx_atomic_uint_t ngx_atomic_t;
|
42
|
22 #define NGX_ATOMIC_T_LEN sizeof("-2147483648") - 1
|
|
23
|
|
24
|
|
25 #if (NGX_SMP)
|
|
26 #define NGX_SMP_LOCK "lock;"
|
|
27 #else
|
|
28 #define NGX_SMP_LOCK
|
|
29 #endif
|
|
30
|
|
31 /*
|
|
32 * the "=q" is any of the %eax, %ebx, %ecx, or %edx registers.
|
|
33 * the '"0" (1)' parameter preloads 1 into %0.
|
|
34 * the "cc" means that flags were changed.
|
44
|
35 *
|
|
36 * "xadd r, [m]":
|
|
37 *
|
|
38 * temp = [m];
|
|
39 * [m] += r;
|
|
40 * r = temp;
|
42
|
41 */
|
|
42
|
44
|
43 static ngx_inline ngx_atomic_uint_t
|
42
|
44 ngx_atomic_inc(ngx_atomic_t *value)
|
|
45 {
|
44
|
46 ngx_atomic_uint_t old;
|
42
|
47
|
|
48 __asm__ volatile (
|
|
49
|
|
50 NGX_SMP_LOCK
|
|
51 " xaddl %0, %2; "
|
|
52 " incl %0; "
|
|
53
|
|
54 : "=q" (old) : "0" (1), "m" (*value) : "cc", "memory");
|
|
55
|
|
56 return old;
|
|
57 }
|
|
58
|
|
59
|
44
|
60 static ngx_inline ngx_atomic_uint_t
|
42
|
61 ngx_atomic_dec(ngx_atomic_t *value)
|
|
62 {
|
44
|
63 ngx_atomic_uint_t old;
|
42
|
64
|
|
65 __asm__ volatile (
|
|
66
|
|
67 NGX_SMP_LOCK
|
|
68 " xaddl %0, %2; "
|
|
69 " decl %0; "
|
|
70
|
|
71 : "=q" (old) : "0" (-1), "m" (*value) : "cc", "memory");
|
|
72
|
|
73 return old;
|
|
74 }
|
|
75
|
|
76
|
|
77 /*
|
|
78 * the "q" is any of the %eax, %ebx, %ecx, or %edx registers.
|
|
79 * the "=a" and "a" are the %eax register. Although we can return result
|
|
80 * in any register, we use %eax because it is used in cmpxchg anyway.
|
|
81 *
|
|
82 * "cmpxchg r, [m]":
|
|
83 *
|
44
|
84 * if (eax == [m]) {
|
|
85 * zf = 1;
|
|
86 * [m] = r;
|
|
87 * } else {
|
|
88 * zf = 0;
|
|
89 * eax = [m];
|
|
90 * }
|
42
|
91 */
|
|
92
|
44
|
93 static ngx_inline ngx_atomic_uint_t
|
|
94 ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old,
|
|
95 ngx_atomic_uint_t set)
|
42
|
96 {
|
44
|
97 ngx_atomic_uint_t res;
|
42
|
98
|
|
99 __asm__ volatile (
|
|
100
|
|
101 NGX_SMP_LOCK
|
|
102 " cmpxchgl %3, %1; "
|
|
103 " setz %b0; "
|
|
104 " movzbl %b0, %0; "
|
|
105
|
|
106 : "=a" (res) : "m" (*lock), "a" (old), "q" (set) : "cc", "memory");
|
|
107
|
|
108 return res;
|
|
109 }
|
|
110
|
|
111
|
96
|
112 #elif ( __amd64__ || __amd64 )
|
42
|
113
|
|
114 #define NGX_HAVE_ATOMIC_OPS 1
|
|
115
|
|
116 typedef int64_t ngx_atomic_int_t;
|
44
|
117 typedef uint64_t ngx_atomic_uint_t;
|
|
118 typedef volatile ngx_atomic_uint_t ngx_atomic_t;
|
42
|
119 #define NGX_ATOMIC_T_LEN sizeof("-9223372036854775808") - 1
|
|
120
|
0
|
121
|
|
122 #if (NGX_SMP)
|
|
123 #define NGX_SMP_LOCK "lock;"
|
|
124 #else
|
|
125 #define NGX_SMP_LOCK
|
|
126 #endif
|
|
127
|
|
128
|
44
|
129 static ngx_inline ngx_atomic_uint_t
|
42
|
130 ngx_atomic_inc(ngx_atomic_t *value)
|
0
|
131 {
|
44
|
132 ngx_atomic_uint_t old;
|
0
|
133
|
|
134 __asm__ volatile (
|
|
135
|
42
|
136 NGX_SMP_LOCK
|
|
137 " xaddq %0, %2; "
|
|
138 " incq %0; "
|
0
|
139
|
42
|
140 : "=r" (old) : "0" (1), "m" (*value) : "cc", "memory");
|
0
|
141
|
|
142 return old;
|
|
143 }
|
|
144
|
|
145
|
42
|
146 /* the '"0" (-1LL)' parameter preloads -1 into the 64-bit %0 register */
|
0
|
147
|
44
|
148 static ngx_inline ngx_atomic_uint_t
|
42
|
149 ngx_atomic_dec(ngx_atomic_t *value)
|
0
|
150 {
|
44
|
151 ngx_atomic_uint_t old;
|
0
|
152
|
|
153 __asm__ volatile (
|
|
154
|
42
|
155 NGX_SMP_LOCK
|
|
156 " xaddq %0, %2; "
|
|
157 " decq %0; "
|
0
|
158
|
42
|
159 : "=r" (old) : "0" (-1LL), "m" (*value) : "cc", "memory");
|
0
|
160
|
|
161 return old;
|
|
162 }
|
|
163
|
|
164
|
42
|
165 /* the "=a" and "a" are the %rax register. */
|
0
|
166
|
44
|
167 static ngx_inline ngx_atomic_uint_t
|
|
168 ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old,
|
|
169 ngx_atomic_uint_t set)
|
0
|
170 {
|
44
|
171 ngx_atomic_uint_t res;
|
0
|
172
|
|
173 __asm__ volatile (
|
|
174
|
42
|
175 NGX_SMP_LOCK
|
|
176 " cmpxchgq %3, %1; "
|
|
177 " setz %b0; "
|
|
178 " movzbq %b0, %0; "
|
0
|
179
|
42
|
180 : "=a" (res) : "m" (*lock), "a" (old), "r" (set) : "cc", "memory");
|
0
|
181
|
|
182 return res;
|
|
183 }
|
|
184
|
|
185
|
96
|
186 #elif ( __sparc__ || __sparcv9 )
|
0
|
187
|
|
188 #define NGX_HAVE_ATOMIC_OPS 1
|
|
189
|
42
|
190 #if (NGX_PTR_SIZE == 8)
|
44
|
191 typedef int64_t ngx_atomic_int_t;
|
|
192 typedef uint64_t ngx_atomic_uint_t;
|
42
|
193 #define NGX_ATOMIC_T_LEN sizeof("-9223372036854775808") - 1
|
|
194 #define NGX_CASXA "casxa"
|
|
195 #else
|
44
|
196 typedef int32_t ngx_atomic_int_t;
|
|
197 typedef uint32_t ngx_atomic_uint_t;
|
42
|
198 #define NGX_ATOMIC_T_LEN sizeof("-2147483648") - 1
|
|
199 #define NGX_CASXA "casa"
|
|
200 #endif
|
|
201
|
44
|
202 typedef volatile ngx_atomic_uint_t ngx_atomic_t;
|
0
|
203
|
|
204
|
42
|
205 /*
|
|
206 * the "+r" means the general register used for both input and output.
|
|
207 *
|
|
208 * "casa [r1] 0x80, r2, r0" and
|
|
209 * "casxa [r1] 0x80, r2, r0" do the following:
|
|
210 *
|
|
211 * if ([r1] == r2) {
|
|
212 * swap(r0, [r1]);
|
|
213 * } else {
|
|
214 * r0 = [r1];
|
|
215 * }
|
|
216 *
|
|
217 * so "r0 == r2" means that the operation was successfull.
|
|
218 */
|
|
219
|
44
|
220 static ngx_inline ngx_atomic_uint_t
|
42
|
221 ngx_atomic_inc(ngx_atomic_t *value)
|
0
|
222 {
|
44
|
223 ngx_atomic_uint_t old, new, res;
|
0
|
224
|
|
225 old = *value;
|
|
226
|
|
227 for ( ;; ) {
|
|
228
|
|
229 new = old + 1;
|
|
230 res = new;
|
|
231
|
|
232 __asm__ volatile (
|
|
233
|
42
|
234 NGX_CASXA " [%1] 0x80, %2, %0"
|
|
235
|
|
236 : "+r" (res) : "r" (value), "r" (old) : "memory");
|
|
237
|
|
238 if (res == old) {
|
|
239 return new;
|
|
240 }
|
|
241
|
|
242 old = res;
|
|
243 }
|
|
244 }
|
|
245
|
0
|
246
|
44
|
247 static ngx_inline ngx_atomic_uint_t
|
42
|
248 ngx_atomic_dec(ngx_atomic_t *value)
|
|
249 {
|
44
|
250 ngx_atomic_uint_t old, new, res;
|
42
|
251
|
|
252 old = *value;
|
|
253
|
|
254 for ( ;; ) {
|
|
255
|
|
256 new = old - 1;
|
|
257 res = new;
|
|
258
|
|
259 __asm__ volatile (
|
|
260
|
|
261 NGX_CASXA " [%1] 0x80, %2, %0"
|
|
262
|
|
263 : "+r" (res) : "r" (value), "r" (old) : "memory");
|
0
|
264
|
|
265 if (res == old) {
|
|
266 return new;
|
|
267 }
|
|
268
|
|
269 old = res;
|
|
270 }
|
|
271 }
|
|
272
|
|
273
|
44
|
274 static ngx_inline ngx_atomic_uint_t
|
|
275 ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old,
|
|
276 ngx_atomic_uint_t set)
|
0
|
277 {
|
42
|
278 __asm__ volatile (
|
|
279
|
|
280 NGX_CASXA " [%1] 0x80, %2, %0"
|
|
281
|
|
282 : "+r" (set) : "r" (lock), "r" (old) : "memory");
|
|
283
|
|
284 return (set == old);
|
|
285 }
|
|
286
|
|
287
|
74
|
288 #elif ( __ppc__ || __powerpc__ )
|
42
|
289
|
|
290 #define NGX_HAVE_ATOMIC_OPS 1
|
|
291
|
|
292 #if (NGX_PTR_SIZE == 8)
|
44
|
293 typedef int64_t ngx_atomic_int_t;
|
|
294 typedef uint64_t ngx_atomic_uint_t;
|
42
|
295 #define NGX_ATOMIC_T_LEN sizeof("-9223372036854775808") - 1
|
|
296 #else
|
44
|
297 typedef int32_t ngx_atomic_int_t;
|
|
298 typedef uint32_t ngx_atomic_uint_t;
|
42
|
299 #define NGX_ATOMIC_T_LEN sizeof("-2147483648") - 1
|
|
300 #endif
|
|
301
|
44
|
302 typedef volatile ngx_atomic_uint_t ngx_atomic_t;
|
42
|
303
|
|
304
|
|
305 /*
|
|
306 * the ppc assembler treats ";" as comment, so we have to use "\n".
|
|
307 * the minus in "bne-" is a hint for the branch prediction unit that
|
|
308 * this branch is unlikely to be taken.
|
|
309 *
|
|
310 * the "=&r" means that no input registers can be used.
|
100
|
311 * the "=&b" means that the base registers can be used only, i.e.
|
|
312 * any register except r0. the r0 register always has a zero value and
|
|
313 * could not be used in "addi r0, r0, 1".
|
42
|
314 * the "1b" means the nearest backward label "1" and the "1f" means
|
|
315 * the nearest forward label "1".
|
|
316 */
|
|
317
|
44
|
318 static ngx_inline ngx_atomic_uint_t
|
42
|
319 ngx_atomic_inc(ngx_atomic_t *value)
|
|
320 {
|
44
|
321 ngx_atomic_uint_t res;
|
0
|
322
|
|
323 __asm__ volatile (
|
|
324
|
42
|
325 "1: lwarx %0, 0, %1 \n" /* load from [value] into "res" */
|
|
326 /* and store reservation */
|
|
327 " addi %0, %0, 1 \n" /* add "1" to "res" */
|
|
328 " stwcx. %0, 0, %1 \n" /* store "res" into [value] if reservation */
|
|
329 /* is not cleared */
|
|
330 " bne- 1b \n" /* try again if reservation was cleared */
|
|
331
|
|
332 : "=&b" (res) : "r" (value) : "cc", "memory");
|
|
333
|
|
334 return res;
|
|
335 }
|
|
336
|
|
337
|
44
|
338 static ngx_inline ngx_atomic_uint_t
|
42
|
339 ngx_atomic_dec(ngx_atomic_t *value)
|
|
340 {
|
44
|
341 ngx_atomic_uint_t res;
|
42
|
342
|
|
343 __asm__ volatile (
|
|
344
|
|
345 "1: lwarx %0, 0, %1 \n" /* load from [value] into "res" */
|
|
346 /* and store reservation */
|
|
347 " addi %0, %0, -1 \n" /* sub "1" from "res" */
|
|
348 " stwcx. %0, 0, %1 \n" /* store "res" into [value] if reservation */
|
|
349 /* is not cleared */
|
|
350 " bne- 1b \n" /* try again if reservation was cleared */
|
|
351
|
|
352 : "=&b" (res) : "r" (value) : "cc", "memory");
|
0
|
353
|
42
|
354 return res;
|
|
355 }
|
|
356
|
|
357
|
44
|
358 static ngx_inline ngx_atomic_uint_t
|
|
359 ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old,
|
|
360 ngx_atomic_uint_t set)
|
42
|
361 {
|
44
|
362 ngx_atomic_uint_t res, temp;
|
42
|
363
|
|
364 __asm__ volatile (
|
0
|
365
|
42
|
366 " li %0, 0 \n" /* preset "0" to "res" */
|
|
367 " lwarx %1, 0, %2 \n" /* load from [lock] into "temp" */
|
|
368 /* and store reservation */
|
|
369 " cmpw %1, %3 \n" /* compare "temp" and "old" */
|
|
370 " bne- 1f \n" /* not equal */
|
|
371 " stwcx. %4, 0, %2 \n" /* store "set" into [lock] if reservation */
|
|
372 /* is not cleared */
|
|
373 " bne- 1f \n" /* the reservation was cleared */
|
|
374 " li %0, 1 \n" /* set "1" to "res" */
|
|
375 "1: \n"
|
|
376
|
|
377 : "=&r" (res), "=&r" (temp)
|
|
378 : "r" (lock), "r" (old), "r" (set)
|
|
379 : "cc", "memory");
|
|
380
|
|
381 return res;
|
0
|
382 }
|
|
383
|
42
|
384
|
0
|
385 #else
|
|
386
|
|
387 #define NGX_HAVE_ATOMIC_OPS 0
|
|
388
|
44
|
389 typedef int32_t ngx_atomic_int_t;
|
|
390 typedef uint32_t ngx_atomic_uint_t;
|
|
391 typedef volatile ngx_atomic_uint_t ngx_atomic_t;
|
46
|
392 #define NGX_ATOMIC_T_LEN sizeof("-2147483648") - 1
|
0
|
393
|
42
|
394 #define ngx_atomic_inc(x) ++(*(x))
|
|
395 #define ngx_atomic_dec(x) --(*(x))
|
0
|
396
|
44
|
397 static ngx_inline ngx_atomic_uint_t
|
|
398 ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old,
|
|
399 ngx_atomic_uint_t set)
|
0
|
400 {
|
42
|
401 *lock = set;
|
0
|
402 return 1;
|
|
403 }
|
|
404
|
|
405 #endif
|
|
406
|
|
407
|
|
408 void ngx_spinlock(ngx_atomic_t *lock, ngx_uint_t spin);
|
|
409
|
|
410 #define ngx_trylock(lock) (*(lock) == 0 && ngx_atomic_cmp_set(lock, 0, 1))
|
|
411 #define ngx_unlock(lock) *(lock) = 0
|
|
412
|
|
413
|
|
414 #endif /* _NGX_ATOMIC_H_INCLUDED_ */
|