|
110
|
1
|
|
|
2 /*
|
|
|
3 * Copyright (C) Igor Sysoev
|
|
|
4 */
|
|
|
5
|
|
|
6
|
|
|
7 /*
|
|
|
8 * The ppc assembler treats ";" as comment, so we have to use "\n".
|
|
|
9 * The minus in "bne-" is a hint for the branch prediction unit that
|
|
|
10 * this branch is unlikely to be taken.
|
|
|
11 * The "1b" means the nearest backward label "1" and the "1f" means
|
|
|
12 * the nearest forward label "1".
|
|
126
|
13 *
|
|
110
|
14 * The "b" means that the base registers can be used only, i.e.
|
|
|
15 * any register except r0. The r0 register always has a zero value and
|
|
|
16 * could not be used in "addi r0, r0, 1".
|
|
|
17 * The "=&b" means that no input registers can be used.
|
|
270
|
18 *
|
|
|
19 * "sync" read and write barriers
|
|
|
20 * "isync" read barrier, is faster than "sync"
|
|
|
21 * "eieio" write barrier, is faster than "sync"
|
|
|
22 * "lwsync" write barrier, is faster than "eieio" on ppc64
|
|
110
|
23 */
|
|
|
24
|
|
270
|
25 #if (NGX_PTR_SIZE == 8)
|
|
|
26
|
|
110
|
27 static ngx_inline ngx_atomic_uint_t
|
|
|
28 ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old,
|
|
|
29 ngx_atomic_uint_t set)
|
|
|
30 {
|
|
|
31 ngx_atomic_uint_t res, temp;
|
|
|
32
|
|
|
33 __asm__ volatile (
|
|
|
34
|
|
|
35 " li %0, 0 \n" /* preset "0" to "res" */
|
|
270
|
36 " lwsync \n" /* write barrier */
|
|
|
37 "1: \n"
|
|
|
38 " ldarx %1, 0, %2 \n" /* load from [lock] into "temp" */
|
|
110
|
39 /* and store reservation */
|
|
270
|
40 " cmpd %1, %3 \n" /* compare "temp" and "old" */
|
|
|
41 " bne- 2f \n" /* not equal */
|
|
|
42 " stdcx. %4, 0, %2 \n" /* store "set" into [lock] if reservation */
|
|
110
|
43 /* is not cleared */
|
|
270
|
44 " bne- 1b \n" /* the reservation was cleared */
|
|
|
45 " isync \n" /* read barrier */
|
|
110
|
46 " li %0, 1 \n" /* set "1" to "res" */
|
|
270
|
47 "2: \n"
|
|
110
|
48
|
|
|
49 : "=&b" (res), "=&b" (temp)
|
|
|
50 : "b" (lock), "b" (old), "b" (set)
|
|
|
51 : "cc", "memory");
|
|
|
52
|
|
|
53 return res;
|
|
|
54 }
|
|
|
55
|
|
|
56
|
|
|
57 static ngx_inline ngx_atomic_int_t
|
|
|
58 ngx_atomic_fetch_add(ngx_atomic_t *value, ngx_atomic_int_t add)
|
|
|
59 {
|
|
|
60 ngx_atomic_uint_t res, temp;
|
|
|
61
|
|
|
62 __asm__ volatile (
|
|
|
63
|
|
270
|
64 " lwsync \n" /* write barrier */
|
|
|
65 "1: ldarx %0, 0, %2 \n" /* load from [value] into "res" */
|
|
110
|
66 /* and store reservation */
|
|
|
67 " add %1, %0, %3 \n" /* "res" + "add" store in "temp" */
|
|
270
|
68 " stdcx. %1, 0, %2 \n" /* store "temp" into [value] if reservation */
|
|
110
|
69 /* is not cleared */
|
|
|
70 " bne- 1b \n" /* try again if reservation was cleared */
|
|
270
|
71 " isync \n" /* read barrier */
|
|
110
|
72
|
|
|
73 : "=&b" (res), "=&b" (temp)
|
|
|
74 : "b" (value), "b" (add)
|
|
|
75 : "cc", "memory");
|
|
|
76
|
|
|
77 return res;
|
|
|
78 }
|
|
112
|
79
|
|
|
80
|
|
|
81 #if (NGX_SMP)
|
|
270
|
82 #define ngx_memory_barrier() \
|
|
|
83 __asm__ volatile ("isync \n lwsync \n" ::: "memory")
|
|
112
|
84 #else
|
|
|
85 #define ngx_memory_barrier() __asm__ volatile ("" ::: "memory")
|
|
|
86 #endif
|
|
160
|
87
|
|
270
|
88 #else
|
|
|
89
|
|
|
90 static ngx_inline ngx_atomic_uint_t
|
|
|
91 ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old,
|
|
|
92 ngx_atomic_uint_t set)
|
|
|
93 {
|
|
|
94 ngx_atomic_uint_t res, temp;
|
|
|
95
|
|
|
96 __asm__ volatile (
|
|
|
97
|
|
|
98 " li %0, 0 \n" /* preset "0" to "res" */
|
|
|
99 " eieio \n" /* write barrier */
|
|
|
100 "1: \n"
|
|
|
101 " lwarx %1, 0, %2 \n" /* load from [lock] into "temp" */
|
|
|
102 /* and store reservation */
|
|
|
103 " cmpw %1, %3 \n" /* compare "temp" and "old" */
|
|
|
104 " bne- 2f \n" /* not equal */
|
|
|
105 " stwcx. %4, 0, %2 \n" /* store "set" into [lock] if reservation */
|
|
|
106 /* is not cleared */
|
|
|
107 " bne- 1b \n" /* the reservation was cleared */
|
|
|
108 " isync \n" /* read barrier */
|
|
|
109 " li %0, 1 \n" /* set "1" to "res" */
|
|
|
110 "2: \n"
|
|
|
111
|
|
|
112 : "=&b" (res), "=&b" (temp)
|
|
|
113 : "b" (lock), "b" (old), "b" (set)
|
|
|
114 : "cc", "memory");
|
|
|
115
|
|
|
116 return res;
|
|
|
117 }
|
|
|
118
|
|
|
119
|
|
|
120 static ngx_inline ngx_atomic_int_t
|
|
|
121 ngx_atomic_fetch_add(ngx_atomic_t *value, ngx_atomic_int_t add)
|
|
|
122 {
|
|
|
123 ngx_atomic_uint_t res, temp;
|
|
|
124
|
|
|
125 __asm__ volatile (
|
|
|
126
|
|
|
127 " eieio \n" /* write barrier */
|
|
|
128 "1: lwarx %0, 0, %2 \n" /* load from [value] into "res" */
|
|
|
129 /* and store reservation */
|
|
|
130 " add %1, %0, %3 \n" /* "res" + "add" store in "temp" */
|
|
|
131 " stwcx. %1, 0, %2 \n" /* store "temp" into [value] if reservation */
|
|
|
132 /* is not cleared */
|
|
|
133 " bne- 1b \n" /* try again if reservation was cleared */
|
|
|
134 " isync \n" /* read barrier */
|
|
|
135
|
|
|
136 : "=&b" (res), "=&b" (temp)
|
|
|
137 : "b" (value), "b" (add)
|
|
|
138 : "cc", "memory");
|
|
|
139
|
|
|
140 return res;
|
|
|
141 }
|
|
|
142
|
|
|
143
|
|
|
144 #if (NGX_SMP)
|
|
|
145 #define ngx_memory_barrier() \
|
|
|
146 __asm__ volatile ("isync \n eieio \n" ::: "memory")
|
|
|
147 #else
|
|
|
148 #define ngx_memory_barrier() __asm__ volatile ("" ::: "memory")
|
|
|
149 #endif
|
|
|
150
|
|
|
151 #endif
|
|
|
152
|
|
|
153
|
|
160
|
154 #define ngx_cpu_pause()
|
