|
0
|
1
|
|
|
2 /*
|
|
|
3 * Copyright (C) Igor Sysoev
|
|
|
4 */
|
|
|
5
|
|
|
6
|
|
|
7 #ifndef _NGX_ATOMIC_H_INCLUDED_
|
|
|
8 #define _NGX_ATOMIC_H_INCLUDED_
|
|
|
9
|
|
|
10
|
|
|
11 #include <ngx_config.h>
|
|
|
12 #include <ngx_core.h>
|
|
|
13
|
|
|
14
|
|
42
|
15 #if ( __i386__ )
|
|
0
|
16
|
|
|
17 #define NGX_HAVE_ATOMIC_OPS 1
|
|
|
18
|
|
42
|
19 typedef uint32_t ngx_atomic_int_t;
|
|
|
20 typedef volatile ngx_atomic_int_t ngx_atomic_t;
|
|
|
21 #define NGX_ATOMIC_T_LEN sizeof("-2147483648") - 1
|
|
|
22
|
|
|
23
|
|
|
24 #if (NGX_SMP)
|
|
|
25 #define NGX_SMP_LOCK "lock;"
|
|
|
26 #else
|
|
|
27 #define NGX_SMP_LOCK
|
|
|
28 #endif
|
|
|
29
|
|
|
30 /*
|
|
|
31 * the "=q" is any of the %eax, %ebx, %ecx, or %edx registers.
|
|
|
32 * the '"0" (1)' parameter preloads 1 into %0.
|
|
|
33 * the "cc" means that flags were changed.
|
|
|
34 */
|
|
|
35
|
|
|
36 static ngx_inline ngx_atomic_int_t
|
|
|
37 ngx_atomic_inc(ngx_atomic_t *value)
|
|
|
38 {
|
|
|
39 ngx_atomic_int_t old;
|
|
|
40
|
|
|
41 __asm__ volatile (
|
|
|
42
|
|
|
43 NGX_SMP_LOCK
|
|
|
44 " xaddl %0, %2; "
|
|
|
45 " incl %0; "
|
|
|
46
|
|
|
47 : "=q" (old) : "0" (1), "m" (*value) : "cc", "memory");
|
|
|
48
|
|
|
49 return old;
|
|
|
50 }
|
|
|
51
|
|
|
52
|
|
|
53 static ngx_inline ngx_atomic_int_t
|
|
|
54 ngx_atomic_dec(ngx_atomic_t *value)
|
|
|
55 {
|
|
|
56 ngx_atomic_int_t old;
|
|
|
57
|
|
|
58 __asm__ volatile (
|
|
|
59
|
|
|
60 NGX_SMP_LOCK
|
|
|
61 " xaddl %0, %2; "
|
|
|
62 " decl %0; "
|
|
|
63
|
|
|
64 : "=q" (old) : "0" (-1), "m" (*value) : "cc", "memory");
|
|
|
65
|
|
|
66 return old;
|
|
|
67 }
|
|
|
68
|
|
|
69
|
|
|
70 /*
|
|
|
71 * the "q" is any of the %eax, %ebx, %ecx, or %edx registers.
|
|
|
72 * the "=a" and "a" are the %eax register. Although we can return result
|
|
|
73 * in any register, we use %eax because it is used in cmpxchg anyway.
|
|
|
74 *
|
|
|
75 * "cmpxchg r, [m]":
|
|
|
76 *
|
|
|
77 * if (eax == [m]) {
|
|
|
78 * zf = 1;
|
|
|
79 * [m] = r;
|
|
|
80 * } else {
|
|
|
81 * zf = 0;
|
|
|
82 * eax = [m];
|
|
|
83 * }
|
|
|
84 */
|
|
|
85
|
|
|
86 static ngx_inline ngx_atomic_int_t
|
|
|
87 ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_int_t old,
|
|
|
88 ngx_atomic_int_t set)
|
|
|
89 {
|
|
|
90 ngx_atomic_int_t res;
|
|
|
91
|
|
|
92 __asm__ volatile (
|
|
|
93
|
|
|
94 NGX_SMP_LOCK
|
|
|
95 " cmpxchgl %3, %1; "
|
|
|
96 " setz %b0; "
|
|
|
97 " movzbl %b0, %0; "
|
|
|
98
|
|
|
99 : "=a" (res) : "m" (*lock), "a" (old), "q" (set) : "cc", "memory");
|
|
|
100
|
|
|
101 return res;
|
|
|
102 }
|
|
|
103
|
|
|
104
|
|
|
105 #elif ( __amd64__ )
|
|
|
106
|
|
|
107 #define NGX_HAVE_ATOMIC_OPS 1
|
|
|
108
|
|
|
109 typedef int64_t ngx_atomic_int_t;
|
|
|
110 typedef volatile ngx_atomic_int_t ngx_atomic_t;
|
|
|
111 #define NGX_ATOMIC_T_LEN sizeof("-9223372036854775808") - 1
|
|
|
112
|
|
0
|
113
|
|
|
114 #if (NGX_SMP)
|
|
|
115 #define NGX_SMP_LOCK "lock;"
|
|
|
116 #else
|
|
|
117 #define NGX_SMP_LOCK
|
|
|
118 #endif
|
|
|
119
|
|
|
120
|
|
42
|
121 static ngx_inline ngx_atomic_int_t
|
|
|
122 ngx_atomic_inc(ngx_atomic_t *value)
|
|
0
|
123 {
|
|
42
|
124 ngx_atomic_int_t old;
|
|
0
|
125
|
|
|
126 __asm__ volatile (
|
|
|
127
|
|
42
|
128 NGX_SMP_LOCK
|
|
|
129 " xaddq %0, %2; "
|
|
|
130 " incq %0; "
|
|
0
|
131
|
|
42
|
132 : "=r" (old) : "0" (1), "m" (*value) : "cc", "memory");
|
|
0
|
133
|
|
|
134 return old;
|
|
|
135 }
|
|
|
136
|
|
|
137
|
|
42
|
138 /* the '"0" (-1LL)' parameter preloads -1 into the 64-bit %0 register */
|
|
0
|
139
|
|
42
|
140 static ngx_inline ngx_atomic_int_t
|
|
|
141 ngx_atomic_dec(ngx_atomic_t *value)
|
|
0
|
142 {
|
|
42
|
143 ngx_atomic_int_t old;
|
|
0
|
144
|
|
|
145 __asm__ volatile (
|
|
|
146
|
|
42
|
147 NGX_SMP_LOCK
|
|
|
148 " xaddq %0, %2; "
|
|
|
149 " decq %0; "
|
|
0
|
150
|
|
42
|
151 : "=r" (old) : "0" (-1LL), "m" (*value) : "cc", "memory");
|
|
0
|
152
|
|
|
153 return old;
|
|
|
154 }
|
|
|
155
|
|
|
156
|
|
42
|
157 /* the "=a" and "a" are the %rax register. */
|
|
0
|
158
|
|
42
|
159 static ngx_inline ngx_atomic_int_t
|
|
|
160 ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_int_t old,
|
|
|
161 ngx_atomic_int_t set)
|
|
0
|
162 {
|
|
42
|
163 ngx_atomic_int_t res;
|
|
0
|
164
|
|
|
165 __asm__ volatile (
|
|
|
166
|
|
42
|
167 NGX_SMP_LOCK
|
|
|
168 " cmpxchgq %3, %1; "
|
|
|
169 " setz %b0; "
|
|
|
170 " movzbq %b0, %0; "
|
|
0
|
171
|
|
42
|
172 : "=a" (res) : "m" (*lock), "a" (old), "r" (set) : "cc", "memory");
|
|
0
|
173
|
|
|
174 return res;
|
|
|
175 }
|
|
|
176
|
|
|
177
|
|
|
178 #elif ( __sparc__ )
|
|
|
179
|
|
|
180 #define NGX_HAVE_ATOMIC_OPS 1
|
|
|
181
|
|
42
|
182 #if (NGX_PTR_SIZE == 8)
|
|
|
183 typedef uint64_t ngx_atomic_int_t;
|
|
|
184 #define NGX_ATOMIC_T_LEN sizeof("-9223372036854775808") - 1
|
|
|
185 #define NGX_CASXA "casxa"
|
|
|
186 #else
|
|
|
187 typedef uint32_t ngx_atomic_int_t;
|
|
|
188 #define NGX_ATOMIC_T_LEN sizeof("-2147483648") - 1
|
|
|
189 #define NGX_CASXA "casa"
|
|
|
190 #endif
|
|
|
191
|
|
|
192 typedef volatile ngx_atomic_int_t ngx_atomic_t;
|
|
0
|
193
|
|
|
194
|
|
42
|
195 /*
|
|
|
196 * the "+r" means the general register used for both input and output.
|
|
|
197 *
|
|
|
198 * "casa [r1] 0x80, r2, r0" and
|
|
|
199 * "casxa [r1] 0x80, r2, r0" do the following:
|
|
|
200 *
|
|
|
201 * if ([r1] == r2) {
|
|
|
202 * swap(r0, [r1]);
|
|
|
203 * } else {
|
|
|
204 * r0 = [r1];
|
|
|
205 * }
|
|
|
206 *
|
|
|
207 * so "r0 == r2" means that the operation was successfull.
|
|
|
208 */
|
|
|
209
|
|
|
210 static ngx_inline ngx_atomic_int_t
|
|
|
211 ngx_atomic_inc(ngx_atomic_t *value)
|
|
0
|
212 {
|
|
42
|
213 ngx_atomic_int_t old, new, res;
|
|
0
|
214
|
|
|
215 old = *value;
|
|
|
216
|
|
|
217 for ( ;; ) {
|
|
|
218
|
|
|
219 new = old + 1;
|
|
|
220 res = new;
|
|
|
221
|
|
|
222 __asm__ volatile (
|
|
|
223
|
|
42
|
224 NGX_CASXA " [%1] 0x80, %2, %0"
|
|
|
225
|
|
|
226 : "+r" (res) : "r" (value), "r" (old) : "memory");
|
|
|
227
|
|
|
228 if (res == old) {
|
|
|
229 return new;
|
|
|
230 }
|
|
|
231
|
|
|
232 old = res;
|
|
|
233 }
|
|
|
234 }
|
|
|
235
|
|
0
|
236
|
|
42
|
237 static ngx_inline ngx_atomic_int_t
|
|
|
238 ngx_atomic_dec(ngx_atomic_t *value)
|
|
|
239 {
|
|
|
240 ngx_atomic_int_t old, new, res;
|
|
|
241
|
|
|
242 old = *value;
|
|
|
243
|
|
|
244 for ( ;; ) {
|
|
|
245
|
|
|
246 new = old - 1;
|
|
|
247 res = new;
|
|
|
248
|
|
|
249 __asm__ volatile (
|
|
|
250
|
|
|
251 NGX_CASXA " [%1] 0x80, %2, %0"
|
|
|
252
|
|
|
253 : "+r" (res) : "r" (value), "r" (old) : "memory");
|
|
0
|
254
|
|
|
255 if (res == old) {
|
|
|
256 return new;
|
|
|
257 }
|
|
|
258
|
|
|
259 old = res;
|
|
|
260 }
|
|
|
261 }
|
|
|
262
|
|
|
263
|
|
42
|
264 static ngx_inline ngx_atomic_int_t
|
|
|
265 ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_int_t old, ngx_atomic_int_t set)
|
|
0
|
266 {
|
|
42
|
267 __asm__ volatile (
|
|
|
268
|
|
|
269 NGX_CASXA " [%1] 0x80, %2, %0"
|
|
|
270
|
|
|
271 : "+r" (set) : "r" (lock), "r" (old) : "memory");
|
|
|
272
|
|
|
273 return (set == old);
|
|
|
274 }
|
|
|
275
|
|
|
276
|
|
|
277 #elif ( __ppc__ )
|
|
|
278
|
|
|
279 #define NGX_HAVE_ATOMIC_OPS 1
|
|
|
280
|
|
|
281 #if (NGX_PTR_SIZE == 8)
|
|
|
282 typedef uint64_t ngx_atomic_int_t;
|
|
|
283 #define NGX_ATOMIC_T_LEN sizeof("-9223372036854775808") - 1
|
|
|
284 #else
|
|
|
285 #define NGX_ATOMIC_T_LEN sizeof("-2147483648") - 1
|
|
|
286 typedef uint32_t ngx_atomic_int_t;
|
|
|
287 #endif
|
|
|
288
|
|
|
289 typedef volatile ngx_atomic_int_t ngx_atomic_t;
|
|
|
290
|
|
|
291
|
|
|
292 /*
|
|
|
293 * the ppc assembler treats ";" as comment, so we have to use "\n".
|
|
|
294 * the minus in "bne-" is a hint for the branch prediction unit that
|
|
|
295 * this branch is unlikely to be taken.
|
|
|
296 *
|
|
|
297 * the "=&r" means that no input registers can be used.
|
|
|
298 * the "=&b" means that the base registers can be used only, i.e. any register
|
|
|
299 * except r0. the r0 register can not be used in "addi r0, r0, 1".
|
|
|
300 * the "1b" means the nearest backward label "1" and the "1f" means
|
|
|
301 * the nearest forward label "1".
|
|
|
302 */
|
|
|
303
|
|
|
304 static ngx_inline ngx_atomic_int_t
|
|
|
305 ngx_atomic_inc(ngx_atomic_t *value)
|
|
|
306 {
|
|
|
307 ngx_atomic_int_t res;
|
|
0
|
308
|
|
|
309 __asm__ volatile (
|
|
|
310
|
|
42
|
311 "1: lwarx %0, 0, %1 \n" /* load from [value] into "res" */
|
|
|
312 /* and store reservation */
|
|
|
313 " addi %0, %0, 1 \n" /* add "1" to "res" */
|
|
|
314 " stwcx. %0, 0, %1 \n" /* store "res" into [value] if reservation */
|
|
|
315 /* is not cleared */
|
|
|
316 " bne- 1b \n" /* try again if reservation was cleared */
|
|
|
317
|
|
|
318 : "=&b" (res) : "r" (value) : "cc", "memory");
|
|
|
319
|
|
|
320 return res;
|
|
|
321 }
|
|
|
322
|
|
|
323
|
|
|
324 static ngx_inline ngx_atomic_int_t
|
|
|
325 ngx_atomic_dec(ngx_atomic_t *value)
|
|
|
326 {
|
|
|
327 ngx_atomic_int_t res;
|
|
|
328
|
|
|
329 __asm__ volatile (
|
|
|
330
|
|
|
331 "1: lwarx %0, 0, %1 \n" /* load from [value] into "res" */
|
|
|
332 /* and store reservation */
|
|
|
333 " addi %0, %0, -1 \n" /* sub "1" from "res" */
|
|
|
334 " stwcx. %0, 0, %1 \n" /* store "res" into [value] if reservation */
|
|
|
335 /* is not cleared */
|
|
|
336 " bne- 1b \n" /* try again if reservation was cleared */
|
|
|
337
|
|
|
338 : "=&b" (res) : "r" (value) : "cc", "memory");
|
|
0
|
339
|
|
42
|
340 return res;
|
|
|
341 }
|
|
|
342
|
|
|
343
|
|
|
344 static ngx_inline ngx_atomic_int_t
|
|
|
345 ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_int_t old,
|
|
|
346 ngx_atomic_int_t set)
|
|
|
347 {
|
|
|
348 ngx_atomic_int_t res, temp;
|
|
|
349
|
|
|
350 __asm__ volatile (
|
|
0
|
351
|
|
42
|
352 " li %0, 0 \n" /* preset "0" to "res" */
|
|
|
353 " lwarx %1, 0, %2 \n" /* load from [lock] into "temp" */
|
|
|
354 /* and store reservation */
|
|
|
355 " cmpw %1, %3 \n" /* compare "temp" and "old" */
|
|
|
356 " bne- 1f \n" /* not equal */
|
|
|
357 " stwcx. %4, 0, %2 \n" /* store "set" into [lock] if reservation */
|
|
|
358 /* is not cleared */
|
|
|
359 " bne- 1f \n" /* the reservation was cleared */
|
|
|
360 " li %0, 1 \n" /* set "1" to "res" */
|
|
|
361 "1: \n"
|
|
|
362
|
|
|
363 : "=&r" (res), "=&r" (temp)
|
|
|
364 : "r" (lock), "r" (old), "r" (set)
|
|
|
365 : "cc", "memory");
|
|
|
366
|
|
|
367 return res;
|
|
0
|
368 }
|
|
|
369
|
|
42
|
370
|
|
0
|
371 #else
|
|
|
372
|
|
|
373 #define NGX_HAVE_ATOMIC_OPS 0
|
|
|
374
|
|
42
|
375 typedef uint32_t ngx_atomic_int_t;
|
|
|
376 typedef volatile ngx_atomic_int_t ngx_atomic_t;
|
|
0
|
377
|
|
42
|
378 #define ngx_atomic_inc(x) ++(*(x))
|
|
|
379 #define ngx_atomic_dec(x) --(*(x))
|
|
0
|
380
|
|
42
|
381 static ngx_inline ngx_atomic_int_t
|
|
|
382 ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_int_t old,
|
|
|
383 ngx_atomic_int_t set)
|
|
0
|
384 {
|
|
42
|
385 *lock = set;
|
|
0
|
386 return 1;
|
|
|
387 }
|
|
|
388
|
|
|
389 #endif
|
|
|
390
|
|
|
391
|
|
|
392 void ngx_spinlock(ngx_atomic_t *lock, ngx_uint_t spin);
|
|
|
393
|
|
|
394 #define ngx_trylock(lock) (*(lock) == 0 && ngx_atomic_cmp_set(lock, 0, 1))
|
|
|
395 #define ngx_unlock(lock) *(lock) = 0
|
|
|
396
|
|
|
397
|
|
|
398 #endif /* _NGX_ATOMIC_H_INCLUDED_ */
|
