Mercurial > hg > nginx
annotate src/os/unix/ngx_gcc_atomic_ppc.h @ 7660:d33e17499088
Version bump.
| author | Maxim Dounin <mdounin@mdounin.ru> |
|---|---|
| date | Tue, 26 May 2020 22:03:00 +0300 |
| parents | d620f497c50f |
| children |
| rev | line source |
|---|---|
| 561 | 1 |
| 2 /* | |
| 3 * Copyright (C) Igor Sysoev | |
| 4412 | 4 * Copyright (C) Nginx, Inc. |
| 561 | 5 */ |
| 6 | |
| 7 | |
| 8 /* | |
| 9 * The ppc assembler treats ";" as comment, so we have to use "\n". | |
| 10 * The minus in "bne-" is a hint for the branch prediction unit that | |
| 11 * this branch is unlikely to be taken. | |
| 12 * The "1b" means the nearest backward label "1" and the "1f" means | |
| 13 * the nearest forward label "1". | |
| 577 | 14 * |
| 561 | 15 * The "b" means that the base registers can be used only, i.e. |
| 16 * any register except r0. The r0 register always has a zero value and | |
| 17 * could not be used in "addi r0, r0, 1". | |
| 18 * The "=&b" means that no input registers can be used. | |
| 938 | 19 * |
| 20 * "sync" read and write barriers | |
| 21 * "isync" read barrier, is faster than "sync" | |
| 22 * "eieio" write barrier, is faster than "sync" | |
| 23 * "lwsync" write barrier, is faster than "eieio" on ppc64 | |
| 561 | 24 */ |
| 25 | |
|
934
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
26 #if (NGX_PTR_SIZE == 8) |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
27 |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
28 static ngx_inline ngx_atomic_uint_t |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
29 ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old, |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
30 ngx_atomic_uint_t set) |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
31 { |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
32 ngx_atomic_uint_t res, temp; |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
33 |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
34 __asm__ volatile ( |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
35 |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
36 " li %0, 0 \n" /* preset "0" to "res" */ |
| 938 | 37 " lwsync \n" /* write barrier */ |
|
937
fc1358d3d23a
larx/stcx. should cycle if the reservation was cleared
Igor Sysoev <igor@sysoev.ru>
parents:
935
diff
changeset
|
38 "1: \n" |
|
934
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
39 " ldarx %1, 0, %2 \n" /* load from [lock] into "temp" */ |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
40 /* and store reservation */ |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
41 " cmpd %1, %3 \n" /* compare "temp" and "old" */ |
|
937
fc1358d3d23a
larx/stcx. should cycle if the reservation was cleared
Igor Sysoev <igor@sysoev.ru>
parents:
935
diff
changeset
|
42 " bne- 2f \n" /* not equal */ |
|
934
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
43 " stdcx. %4, 0, %2 \n" /* store "set" into [lock] if reservation */ |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
44 /* is not cleared */ |
|
937
fc1358d3d23a
larx/stcx. should cycle if the reservation was cleared
Igor Sysoev <igor@sysoev.ru>
parents:
935
diff
changeset
|
45 " bne- 1b \n" /* the reservation was cleared */ |
| 938 | 46 " isync \n" /* read barrier */ |
|
934
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
47 " li %0, 1 \n" /* set "1" to "res" */ |
|
937
fc1358d3d23a
larx/stcx. should cycle if the reservation was cleared
Igor Sysoev <igor@sysoev.ru>
parents:
935
diff
changeset
|
48 "2: \n" |
|
934
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
49 |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
50 : "=&b" (res), "=&b" (temp) |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
51 : "b" (lock), "b" (old), "b" (set) |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
52 : "cc", "memory"); |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
53 |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
54 return res; |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
55 } |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
56 |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
57 |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
58 static ngx_inline ngx_atomic_int_t |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
59 ngx_atomic_fetch_add(ngx_atomic_t *value, ngx_atomic_int_t add) |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
60 { |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
61 ngx_atomic_uint_t res, temp; |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
62 |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
63 __asm__ volatile ( |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
64 |
| 938 | 65 " lwsync \n" /* write barrier */ |
|
934
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
66 "1: ldarx %0, 0, %2 \n" /* load from [value] into "res" */ |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
67 /* and store reservation */ |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
68 " add %1, %0, %3 \n" /* "res" + "add" store in "temp" */ |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
69 " stdcx. %1, 0, %2 \n" /* store "temp" into [value] if reservation */ |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
70 /* is not cleared */ |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
71 " bne- 1b \n" /* try again if reservation was cleared */ |
| 938 | 72 " isync \n" /* read barrier */ |
|
934
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
73 |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
74 : "=&b" (res), "=&b" (temp) |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
75 : "b" (value), "b" (add) |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
76 : "cc", "memory"); |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
77 |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
78 return res; |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
79 } |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
80 |
|
935
db8e718447f1
use light-weight sync on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
934
diff
changeset
|
81 |
|
db8e718447f1
use light-weight sync on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
934
diff
changeset
|
82 #if (NGX_SMP) |
| 938 | 83 #define ngx_memory_barrier() \ |
| 84 __asm__ volatile ("isync \n lwsync \n" ::: "memory") | |
|
935
db8e718447f1
use light-weight sync on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
934
diff
changeset
|
85 #else |
|
db8e718447f1
use light-weight sync on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
934
diff
changeset
|
86 #define ngx_memory_barrier() __asm__ volatile ("" ::: "memory") |
|
db8e718447f1
use light-weight sync on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
934
diff
changeset
|
87 #endif |
|
db8e718447f1
use light-weight sync on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
934
diff
changeset
|
88 |
|
934
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
89 #else |
|
4745e72044fb
fix atomic operations on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
611
diff
changeset
|
90 |
| 561 | 91 static ngx_inline ngx_atomic_uint_t |
| 92 ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old, | |
| 93 ngx_atomic_uint_t set) | |
| 94 { | |
| 95 ngx_atomic_uint_t res, temp; | |
| 96 | |
| 97 __asm__ volatile ( | |
| 98 | |
| 99 " li %0, 0 \n" /* preset "0" to "res" */ | |
| 938 | 100 " eieio \n" /* write barrier */ |
|
937
fc1358d3d23a
larx/stcx. should cycle if the reservation was cleared
Igor Sysoev <igor@sysoev.ru>
parents:
935
diff
changeset
|
101 "1: \n" |
| 561 | 102 " lwarx %1, 0, %2 \n" /* load from [lock] into "temp" */ |
| 103 /* and store reservation */ | |
| 104 " cmpw %1, %3 \n" /* compare "temp" and "old" */ | |
|
937
fc1358d3d23a
larx/stcx. should cycle if the reservation was cleared
Igor Sysoev <igor@sysoev.ru>
parents:
935
diff
changeset
|
105 " bne- 2f \n" /* not equal */ |
| 561 | 106 " stwcx. %4, 0, %2 \n" /* store "set" into [lock] if reservation */ |
| 107 /* is not cleared */ | |
|
937
fc1358d3d23a
larx/stcx. should cycle if the reservation was cleared
Igor Sysoev <igor@sysoev.ru>
parents:
935
diff
changeset
|
108 " bne- 1b \n" /* the reservation was cleared */ |
| 938 | 109 " isync \n" /* read barrier */ |
| 561 | 110 " li %0, 1 \n" /* set "1" to "res" */ |
|
937
fc1358d3d23a
larx/stcx. should cycle if the reservation was cleared
Igor Sysoev <igor@sysoev.ru>
parents:
935
diff
changeset
|
111 "2: \n" |
| 561 | 112 |
| 113 : "=&b" (res), "=&b" (temp) | |
| 114 : "b" (lock), "b" (old), "b" (set) | |
| 115 : "cc", "memory"); | |
| 116 | |
| 117 return res; | |
| 118 } | |
| 119 | |
| 120 | |
| 121 static ngx_inline ngx_atomic_int_t | |
| 122 ngx_atomic_fetch_add(ngx_atomic_t *value, ngx_atomic_int_t add) | |
| 123 { | |
| 124 ngx_atomic_uint_t res, temp; | |
| 125 | |
| 126 __asm__ volatile ( | |
| 127 | |
| 938 | 128 " eieio \n" /* write barrier */ |
| 561 | 129 "1: lwarx %0, 0, %2 \n" /* load from [value] into "res" */ |
| 130 /* and store reservation */ | |
| 131 " add %1, %0, %3 \n" /* "res" + "add" store in "temp" */ | |
| 132 " stwcx. %1, 0, %2 \n" /* store "temp" into [value] if reservation */ | |
| 133 /* is not cleared */ | |
| 134 " bne- 1b \n" /* try again if reservation was cleared */ | |
| 938 | 135 " isync \n" /* read barrier */ |
| 561 | 136 |
| 137 : "=&b" (res), "=&b" (temp) | |
| 138 : "b" (value), "b" (add) | |
| 139 : "cc", "memory"); | |
| 140 | |
| 141 return res; | |
| 142 } | |
| 563 | 143 |
| 144 | |
| 145 #if (NGX_SMP) | |
| 938 | 146 #define ngx_memory_barrier() \ |
| 147 __asm__ volatile ("isync \n eieio \n" ::: "memory") | |
| 563 | 148 #else |
| 149 #define ngx_memory_barrier() __asm__ volatile ("" ::: "memory") | |
| 150 #endif | |
| 611 | 151 |
|
935
db8e718447f1
use light-weight sync on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
934
diff
changeset
|
152 #endif |
|
db8e718447f1
use light-weight sync on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
934
diff
changeset
|
153 |
|
db8e718447f1
use light-weight sync on ppc64
Igor Sysoev <igor@sysoev.ru>
parents:
934
diff
changeset
|
154 |
| 611 | 155 #define ngx_cpu_pause() |