|
0
|
1
|
|
|
2 /*
|
|
|
3 * Copyright (C) Igor Sysoev
|
|
|
4 */
|
|
|
5
|
|
|
6
|
|
|
7 #ifndef _NGX_ATOMIC_H_INCLUDED_
|
|
|
8 #define _NGX_ATOMIC_H_INCLUDED_
|
|
|
9
|
|
|
10
|
|
|
11 #include <ngx_config.h>
|
|
|
12 #include <ngx_core.h>
|
|
|
13
|
|
|
14
|
|
42
|
15 #if ( __i386__ )
|
|
0
|
16
|
|
|
17 #define NGX_HAVE_ATOMIC_OPS 1
|
|
|
18
|
|
44
|
19 typedef int32_t ngx_atomic_int_t;
|
|
|
20 typedef uint32_t ngx_atomic_uint_t;
|
|
|
21 typedef volatile ngx_atomic_uint_t ngx_atomic_t;
|
|
42
|
22 #define NGX_ATOMIC_T_LEN sizeof("-2147483648") - 1
|
|
|
23
|
|
|
24
|
|
|
25 #if (NGX_SMP)
|
|
|
26 #define NGX_SMP_LOCK "lock;"
|
|
|
27 #else
|
|
|
28 #define NGX_SMP_LOCK
|
|
|
29 #endif
|
|
|
30
|
|
|
31 /*
|
|
|
32 * the "=q" is any of the %eax, %ebx, %ecx, or %edx registers.
|
|
|
33 * the '"0" (1)' parameter preloads 1 into %0.
|
|
|
34 * the "cc" means that flags were changed.
|
|
44
|
35 *
|
|
|
36 * "xadd r, [m]":
|
|
|
37 *
|
|
|
38 * temp = [m];
|
|
|
39 * [m] += r;
|
|
|
40 * r = temp;
|
|
42
|
41 */
|
|
|
42
|
|
44
|
43 static ngx_inline ngx_atomic_uint_t
|
|
42
|
44 ngx_atomic_inc(ngx_atomic_t *value)
|
|
|
45 {
|
|
44
|
46 ngx_atomic_uint_t old;
|
|
42
|
47
|
|
|
48 __asm__ volatile (
|
|
|
49
|
|
|
50 NGX_SMP_LOCK
|
|
|
51 " xaddl %0, %2; "
|
|
|
52 " incl %0; "
|
|
|
53
|
|
|
54 : "=q" (old) : "0" (1), "m" (*value) : "cc", "memory");
|
|
|
55
|
|
|
56 return old;
|
|
|
57 }
|
|
|
58
|
|
|
59
|
|
44
|
60 static ngx_inline ngx_atomic_uint_t
|
|
42
|
61 ngx_atomic_dec(ngx_atomic_t *value)
|
|
|
62 {
|
|
44
|
63 ngx_atomic_uint_t old;
|
|
42
|
64
|
|
|
65 __asm__ volatile (
|
|
|
66
|
|
|
67 NGX_SMP_LOCK
|
|
|
68 " xaddl %0, %2; "
|
|
|
69 " decl %0; "
|
|
|
70
|
|
|
71 : "=q" (old) : "0" (-1), "m" (*value) : "cc", "memory");
|
|
|
72
|
|
|
73 return old;
|
|
|
74 }
|
|
|
75
|
|
|
76
|
|
|
77 /*
|
|
|
78 * the "q" is any of the %eax, %ebx, %ecx, or %edx registers.
|
|
|
79 * the "=a" and "a" are the %eax register. Although we can return result
|
|
|
80 * in any register, we use %eax because it is used in cmpxchg anyway.
|
|
|
81 *
|
|
|
82 * "cmpxchg r, [m]":
|
|
|
83 *
|
|
44
|
84 * if (eax == [m]) {
|
|
|
85 * zf = 1;
|
|
|
86 * [m] = r;
|
|
|
87 * } else {
|
|
|
88 * zf = 0;
|
|
|
89 * eax = [m];
|
|
|
90 * }
|
|
42
|
91 */
|
|
|
92
|
|
44
|
93 static ngx_inline ngx_atomic_uint_t
|
|
|
94 ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old,
|
|
|
95 ngx_atomic_uint_t set)
|
|
42
|
96 {
|
|
44
|
97 ngx_atomic_uint_t res;
|
|
42
|
98
|
|
|
99 __asm__ volatile (
|
|
|
100
|
|
|
101 NGX_SMP_LOCK
|
|
|
102 " cmpxchgl %3, %1; "
|
|
|
103 " setz %b0; "
|
|
|
104 " movzbl %b0, %0; "
|
|
|
105
|
|
|
106 : "=a" (res) : "m" (*lock), "a" (old), "q" (set) : "cc", "memory");
|
|
|
107
|
|
|
108 return res;
|
|
|
109 }
|
|
|
110
|
|
|
111
|
|
|
112 #elif ( __amd64__ )
|
|
|
113
|
|
|
114 #define NGX_HAVE_ATOMIC_OPS 1
|
|
|
115
|
|
|
116 typedef int64_t ngx_atomic_int_t;
|
|
44
|
117 typedef uint64_t ngx_atomic_uint_t;
|
|
|
118 typedef volatile ngx_atomic_uint_t ngx_atomic_t;
|
|
42
|
119 #define NGX_ATOMIC_T_LEN sizeof("-9223372036854775808") - 1
|
|
|
120
|
|
0
|
121
|
|
|
122 #if (NGX_SMP)
|
|
|
123 #define NGX_SMP_LOCK "lock;"
|
|
|
124 #else
|
|
|
125 #define NGX_SMP_LOCK
|
|
|
126 #endif
|
|
|
127
|
|
|
128
|
|
44
|
129 static ngx_inline ngx_atomic_uint_t
|
|
42
|
130 ngx_atomic_inc(ngx_atomic_t *value)
|
|
0
|
131 {
|
|
44
|
132 ngx_atomic_uint_t old;
|
|
0
|
133
|
|
|
134 __asm__ volatile (
|
|
|
135
|
|
42
|
136 NGX_SMP_LOCK
|
|
|
137 " xaddq %0, %2; "
|
|
|
138 " incq %0; "
|
|
0
|
139
|
|
42
|
140 : "=r" (old) : "0" (1), "m" (*value) : "cc", "memory");
|
|
0
|
141
|
|
|
142 return old;
|
|
|
143 }
|
|
|
144
|
|
|
145
|
|
42
|
146 /* the '"0" (-1LL)' parameter preloads -1 into the 64-bit %0 register */
|
|
0
|
147
|
|
44
|
148 static ngx_inline ngx_atomic_uint_t
|
|
42
|
149 ngx_atomic_dec(ngx_atomic_t *value)
|
|
0
|
150 {
|
|
44
|
151 ngx_atomic_uint_t old;
|
|
0
|
152
|
|
|
153 __asm__ volatile (
|
|
|
154
|
|
42
|
155 NGX_SMP_LOCK
|
|
|
156 " xaddq %0, %2; "
|
|
|
157 " decq %0; "
|
|
0
|
158
|
|
42
|
159 : "=r" (old) : "0" (-1LL), "m" (*value) : "cc", "memory");
|
|
0
|
160
|
|
|
161 return old;
|
|
|
162 }
|
|
|
163
|
|
|
164
|
|
42
|
165 /* the "=a" and "a" are the %rax register. */
|
|
0
|
166
|
|
44
|
167 static ngx_inline ngx_atomic_uint_t
|
|
|
168 ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old,
|
|
|
169 ngx_atomic_uint_t set)
|
|
0
|
170 {
|
|
44
|
171 ngx_atomic_uint_t res;
|
|
0
|
172
|
|
|
173 __asm__ volatile (
|
|
|
174
|
|
42
|
175 NGX_SMP_LOCK
|
|
|
176 " cmpxchgq %3, %1; "
|
|
|
177 " setz %b0; "
|
|
|
178 " movzbq %b0, %0; "
|
|
0
|
179
|
|
42
|
180 : "=a" (res) : "m" (*lock), "a" (old), "r" (set) : "cc", "memory");
|
|
0
|
181
|
|
|
182 return res;
|
|
|
183 }
|
|
|
184
|
|
|
185
|
|
|
186 #elif ( __sparc__ )
|
|
|
187
|
|
|
188 #define NGX_HAVE_ATOMIC_OPS 1
|
|
|
189
|
|
42
|
190 #if (NGX_PTR_SIZE == 8)
|
|
44
|
191 typedef int64_t ngx_atomic_int_t;
|
|
|
192 typedef uint64_t ngx_atomic_uint_t;
|
|
42
|
193 #define NGX_ATOMIC_T_LEN sizeof("-9223372036854775808") - 1
|
|
|
194 #define NGX_CASXA "casxa"
|
|
|
195 #else
|
|
44
|
196 typedef int32_t ngx_atomic_int_t;
|
|
|
197 typedef uint32_t ngx_atomic_uint_t;
|
|
42
|
198 #define NGX_ATOMIC_T_LEN sizeof("-2147483648") - 1
|
|
|
199 #define NGX_CASXA "casa"
|
|
|
200 #endif
|
|
|
201
|
|
44
|
202 typedef volatile ngx_atomic_uint_t ngx_atomic_t;
|
|
0
|
203
|
|
|
204
|
|
42
|
205 /*
|
|
|
206 * the "+r" means the general register used for both input and output.
|
|
|
207 *
|
|
|
208 * "casa [r1] 0x80, r2, r0" and
|
|
|
209 * "casxa [r1] 0x80, r2, r0" do the following:
|
|
|
210 *
|
|
|
211 * if ([r1] == r2) {
|
|
|
212 * swap(r0, [r1]);
|
|
|
213 * } else {
|
|
|
214 * r0 = [r1];
|
|
|
215 * }
|
|
|
216 *
|
|
|
217 * so "r0 == r2" means that the operation was successfull.
|
|
|
218 */
|
|
|
219
|
|
44
|
220 static ngx_inline ngx_atomic_uint_t
|
|
42
|
221 ngx_atomic_inc(ngx_atomic_t *value)
|
|
0
|
222 {
|
|
44
|
223 ngx_atomic_uint_t old, new, res;
|
|
0
|
224
|
|
|
225 old = *value;
|
|
|
226
|
|
|
227 for ( ;; ) {
|
|
|
228
|
|
|
229 new = old + 1;
|
|
|
230 res = new;
|
|
|
231
|
|
|
232 __asm__ volatile (
|
|
|
233
|
|
42
|
234 NGX_CASXA " [%1] 0x80, %2, %0"
|
|
|
235
|
|
|
236 : "+r" (res) : "r" (value), "r" (old) : "memory");
|
|
|
237
|
|
|
238 if (res == old) {
|
|
|
239 return new;
|
|
|
240 }
|
|
|
241
|
|
|
242 old = res;
|
|
|
243 }
|
|
|
244 }
|
|
|
245
|
|
0
|
246
|
|
44
|
247 static ngx_inline ngx_atomic_uint_t
|
|
42
|
248 ngx_atomic_dec(ngx_atomic_t *value)
|
|
|
249 {
|
|
44
|
250 ngx_atomic_uint_t old, new, res;
|
|
42
|
251
|
|
|
252 old = *value;
|
|
|
253
|
|
|
254 for ( ;; ) {
|
|
|
255
|
|
|
256 new = old - 1;
|
|
|
257 res = new;
|
|
|
258
|
|
|
259 __asm__ volatile (
|
|
|
260
|
|
|
261 NGX_CASXA " [%1] 0x80, %2, %0"
|
|
|
262
|
|
|
263 : "+r" (res) : "r" (value), "r" (old) : "memory");
|
|
0
|
264
|
|
|
265 if (res == old) {
|
|
|
266 return new;
|
|
|
267 }
|
|
|
268
|
|
|
269 old = res;
|
|
|
270 }
|
|
|
271 }
|
|
|
272
|
|
|
273
|
|
44
|
274 static ngx_inline ngx_atomic_uint_t
|
|
|
275 ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old,
|
|
|
276 ngx_atomic_uint_t set)
|
|
0
|
277 {
|
|
42
|
278 __asm__ volatile (
|
|
|
279
|
|
|
280 NGX_CASXA " [%1] 0x80, %2, %0"
|
|
|
281
|
|
|
282 : "+r" (set) : "r" (lock), "r" (old) : "memory");
|
|
|
283
|
|
|
284 return (set == old);
|
|
|
285 }
|
|
|
286
|
|
|
287
|
|
74
|
288 #elif ( __ppc__ || __powerpc__ )
|
|
42
|
289
|
|
|
290 #define NGX_HAVE_ATOMIC_OPS 1
|
|
|
291
|
|
|
292 #if (NGX_PTR_SIZE == 8)
|
|
44
|
293 typedef int64_t ngx_atomic_int_t;
|
|
|
294 typedef uint64_t ngx_atomic_uint_t;
|
|
42
|
295 #define NGX_ATOMIC_T_LEN sizeof("-9223372036854775808") - 1
|
|
|
296 #else
|
|
44
|
297 typedef int32_t ngx_atomic_int_t;
|
|
|
298 typedef uint32_t ngx_atomic_uint_t;
|
|
42
|
299 #define NGX_ATOMIC_T_LEN sizeof("-2147483648") - 1
|
|
|
300 #endif
|
|
|
301
|
|
44
|
302 typedef volatile ngx_atomic_uint_t ngx_atomic_t;
|
|
42
|
303
|
|
|
304
|
|
|
305 /*
|
|
|
306 * the ppc assembler treats ";" as comment, so we have to use "\n".
|
|
|
307 * the minus in "bne-" is a hint for the branch prediction unit that
|
|
|
308 * this branch is unlikely to be taken.
|
|
|
309 *
|
|
|
310 * the "=&r" means that no input registers can be used.
|
|
|
311 * the "=&b" means that the base registers can be used only, i.e. any register
|
|
|
312 * except r0. the r0 register can not be used in "addi r0, r0, 1".
|
|
|
313 * the "1b" means the nearest backward label "1" and the "1f" means
|
|
|
314 * the nearest forward label "1".
|
|
|
315 */
|
|
|
316
|
|
44
|
317 static ngx_inline ngx_atomic_uint_t
|
|
42
|
318 ngx_atomic_inc(ngx_atomic_t *value)
|
|
|
319 {
|
|
44
|
320 ngx_atomic_uint_t res;
|
|
0
|
321
|
|
|
322 __asm__ volatile (
|
|
|
323
|
|
42
|
324 "1: lwarx %0, 0, %1 \n" /* load from [value] into "res" */
|
|
|
325 /* and store reservation */
|
|
|
326 " addi %0, %0, 1 \n" /* add "1" to "res" */
|
|
|
327 " stwcx. %0, 0, %1 \n" /* store "res" into [value] if reservation */
|
|
|
328 /* is not cleared */
|
|
|
329 " bne- 1b \n" /* try again if reservation was cleared */
|
|
|
330
|
|
|
331 : "=&b" (res) : "r" (value) : "cc", "memory");
|
|
|
332
|
|
|
333 return res;
|
|
|
334 }
|
|
|
335
|
|
|
336
|
|
44
|
337 static ngx_inline ngx_atomic_uint_t
|
|
42
|
338 ngx_atomic_dec(ngx_atomic_t *value)
|
|
|
339 {
|
|
44
|
340 ngx_atomic_uint_t res;
|
|
42
|
341
|
|
|
342 __asm__ volatile (
|
|
|
343
|
|
|
344 "1: lwarx %0, 0, %1 \n" /* load from [value] into "res" */
|
|
|
345 /* and store reservation */
|
|
|
346 " addi %0, %0, -1 \n" /* sub "1" from "res" */
|
|
|
347 " stwcx. %0, 0, %1 \n" /* store "res" into [value] if reservation */
|
|
|
348 /* is not cleared */
|
|
|
349 " bne- 1b \n" /* try again if reservation was cleared */
|
|
|
350
|
|
|
351 : "=&b" (res) : "r" (value) : "cc", "memory");
|
|
0
|
352
|
|
42
|
353 return res;
|
|
|
354 }
|
|
|
355
|
|
|
356
|
|
44
|
357 static ngx_inline ngx_atomic_uint_t
|
|
|
358 ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old,
|
|
|
359 ngx_atomic_uint_t set)
|
|
42
|
360 {
|
|
44
|
361 ngx_atomic_uint_t res, temp;
|
|
42
|
362
|
|
|
363 __asm__ volatile (
|
|
0
|
364
|
|
42
|
365 " li %0, 0 \n" /* preset "0" to "res" */
|
|
|
366 " lwarx %1, 0, %2 \n" /* load from [lock] into "temp" */
|
|
|
367 /* and store reservation */
|
|
|
368 " cmpw %1, %3 \n" /* compare "temp" and "old" */
|
|
|
369 " bne- 1f \n" /* not equal */
|
|
|
370 " stwcx. %4, 0, %2 \n" /* store "set" into [lock] if reservation */
|
|
|
371 /* is not cleared */
|
|
|
372 " bne- 1f \n" /* the reservation was cleared */
|
|
|
373 " li %0, 1 \n" /* set "1" to "res" */
|
|
|
374 "1: \n"
|
|
|
375
|
|
|
376 : "=&r" (res), "=&r" (temp)
|
|
|
377 : "r" (lock), "r" (old), "r" (set)
|
|
|
378 : "cc", "memory");
|
|
|
379
|
|
|
380 return res;
|
|
0
|
381 }
|
|
|
382
|
|
42
|
383
|
|
0
|
384 #else
|
|
|
385
|
|
|
386 #define NGX_HAVE_ATOMIC_OPS 0
|
|
|
387
|
|
44
|
388 typedef int32_t ngx_atomic_int_t;
|
|
|
389 typedef uint32_t ngx_atomic_uint_t;
|
|
|
390 typedef volatile ngx_atomic_uint_t ngx_atomic_t;
|
|
46
|
391 #define NGX_ATOMIC_T_LEN sizeof("-2147483648") - 1
|
|
0
|
392
|
|
42
|
393 #define ngx_atomic_inc(x) ++(*(x))
|
|
|
394 #define ngx_atomic_dec(x) --(*(x))
|
|
0
|
395
|
|
44
|
396 static ngx_inline ngx_atomic_uint_t
|
|
|
397 ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old,
|
|
|
398 ngx_atomic_uint_t set)
|
|
0
|
399 {
|
|
42
|
400 *lock = set;
|
|
0
|
401 return 1;
|
|
|
402 }
|
|
|
403
|
|
|
404 #endif
|
|
|
405
|
|
|
406
|
|
|
407 void ngx_spinlock(ngx_atomic_t *lock, ngx_uint_t spin);
|
|
|
408
|
|
|
409 #define ngx_trylock(lock) (*(lock) == 0 && ngx_atomic_cmp_set(lock, 0, 1))
|
|
|
410 #define ngx_unlock(lock) *(lock) = 0
|
|
|
411
|
|
|
412
|
|
|
413 #endif /* _NGX_ATOMIC_H_INCLUDED_ */
|
