Mercurial > hg > nginx
diff src/os/unix/ngx_atomic.h @ 493:975f62e77f02 release-0.1.21
nginx-0.1.21-RELEASE import
*) Bugfix: the ngx_http_stub_status_module showed incorrect statistics
if "rtsig" method was used or if several worker process ran on SMP.
*) Bugfix: nginx could not be built by the icc compiler on Linux or if
the zlib-1.2.x library was building from sources.
*) Bugfix: nginx could not be built on NetBSD 2.0.
author | Igor Sysoev <igor@sysoev.ru> |
---|---|
date | Tue, 22 Feb 2005 14:40:13 +0000 |
parents | 42d11f017717 |
children | fc9909c369b2 |
line wrap: on
line diff
--- a/src/os/unix/ngx_atomic.h +++ b/src/os/unix/ngx_atomic.h @@ -12,11 +12,104 @@ #include <ngx_core.h> -#if ( __i386__ || __amd64__ ) +#if ( __i386__ ) #define NGX_HAVE_ATOMIC_OPS 1 -typedef volatile uint32_t ngx_atomic_t; +typedef uint32_t ngx_atomic_int_t; +typedef volatile ngx_atomic_int_t ngx_atomic_t; +#define NGX_ATOMIC_T_LEN sizeof("-2147483648") - 1 + + +#if (NGX_SMP) +#define NGX_SMP_LOCK "lock;" +#else +#define NGX_SMP_LOCK +#endif + +/* + * the "=q" is any of the %eax, %ebx, %ecx, or %edx registers. + * the '"0" (1)' parameter preloads 1 into %0. + * the "cc" means that flags were changed. + */ + +static ngx_inline ngx_atomic_int_t +ngx_atomic_inc(ngx_atomic_t *value) +{ + ngx_atomic_int_t old; + + __asm__ volatile ( + + NGX_SMP_LOCK + " xaddl %0, %2; " + " incl %0; " + + : "=q" (old) : "0" (1), "m" (*value) : "cc", "memory"); + + return old; +} + + +static ngx_inline ngx_atomic_int_t +ngx_atomic_dec(ngx_atomic_t *value) +{ + ngx_atomic_int_t old; + + __asm__ volatile ( + + NGX_SMP_LOCK + " xaddl %0, %2; " + " decl %0; " + + : "=q" (old) : "0" (-1), "m" (*value) : "cc", "memory"); + + return old; +} + + +/* + * the "q" is any of the %eax, %ebx, %ecx, or %edx registers. + * the "=a" and "a" are the %eax register. Although we can return result + * in any register, we use %eax because it is used in cmpxchg anyway. + * + * "cmpxchg r, [m]": + * + * if (eax == [m]) { + * zf = 1; + * [m] = r; + * } else { + * zf = 0; + * eax = [m]; + * } + */ + +static ngx_inline ngx_atomic_int_t +ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_int_t old, + ngx_atomic_int_t set) +{ + ngx_atomic_int_t res; + + __asm__ volatile ( + + NGX_SMP_LOCK + " cmpxchgl %3, %1; " + " setz %b0; " + " movzbl %b0, %0; " + + : "=a" (res) : "m" (*lock), "a" (old), "q" (set) : "cc", "memory"); + + return res; +} + + +#elif ( __amd64__ ) + +#define NGX_HAVE_ATOMIC_OPS 1 + +typedef int64_t ngx_atomic_int_t; +typedef volatile ngx_atomic_int_t ngx_atomic_t; +#define NGX_ATOMIC_T_LEN sizeof("-9223372036854775808") - 1 + #if (NGX_SMP) #define NGX_SMP_LOCK "lock;" @@ -25,56 +118,58 @@ typedef volatile uint32_t ngx_atomic_t; #endif -static ngx_inline uint32_t ngx_atomic_inc(ngx_atomic_t *value) +static ngx_inline ngx_atomic_int_t +ngx_atomic_inc(ngx_atomic_t *value) { - uint32_t old; + ngx_atomic_int_t old; __asm__ volatile ( - NGX_SMP_LOCK - " xaddl %0, %2; " - " incl %0; " + NGX_SMP_LOCK + " xaddq %0, %2; " + " incq %0; " - : "=q" (old) : "0" (1), "m" (*value)); + : "=r" (old) : "0" (1), "m" (*value) : "cc", "memory"); return old; } -#if 0 +/* the '"0" (-1LL)' parameter preloads -1 into the 64-bit %0 register */ -static ngx_inline uint32_t ngx_atomic_dec(ngx_atomic_t *value) +static ngx_inline ngx_atomic_int_t +ngx_atomic_dec(ngx_atomic_t *value) { - uint32_t old; + ngx_atomic_int_t old; __asm__ volatile ( - NGX_SMP_LOCK - " xaddl %0, %1; " - " decl %0; " + NGX_SMP_LOCK + " xaddq %0, %2; " + " decq %0; " - : "=q" (old) : "0" (-1), "m" (*value)); + : "=r" (old) : "0" (-1LL), "m" (*value) : "cc", "memory"); return old; } -#endif +/* the "=a" and "a" are the %rax register. */ -static ngx_inline uint32_t ngx_atomic_cmp_set(ngx_atomic_t *lock, - ngx_atomic_t old, - ngx_atomic_t set) +static ngx_inline ngx_atomic_int_t +ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_int_t old, + ngx_atomic_int_t set) { - uint32_t res; + ngx_atomic_int_t res; __asm__ volatile ( - NGX_SMP_LOCK - " cmpxchgl %3, %1; " - " setz %%al; " - " movzbl %%al, %0; " + NGX_SMP_LOCK + " cmpxchgq %3, %1; " + " setz %b0; " + " movzbq %b0, %0; " - : "=a" (res) : "m" (*lock), "a" (old), "q" (set)); + : "=a" (res) : "m" (*lock), "a" (old), "r" (set) : "cc", "memory"); return res; } @@ -84,12 +179,38 @@ static ngx_inline uint32_t ngx_atomic_cm #define NGX_HAVE_ATOMIC_OPS 1 -typedef volatile uint32_t ngx_atomic_t; +#if (NGX_PTR_SIZE == 8) +typedef uint64_t ngx_atomic_int_t; +#define NGX_ATOMIC_T_LEN sizeof("-9223372036854775808") - 1 +#define NGX_CASXA "casxa" +#else +typedef uint32_t ngx_atomic_int_t; +#define NGX_ATOMIC_T_LEN sizeof("-2147483648") - 1 +#define NGX_CASXA "casa" +#endif + +typedef volatile ngx_atomic_int_t ngx_atomic_t; -static ngx_inline uint32_t ngx_atomic_inc(ngx_atomic_t *value) +/* + * the "+r" means the general register used for both input and output. + * + * "casa [r1] 0x80, r2, r0" and + * "casxa [r1] 0x80, r2, r0" do the following: + * + * if ([r1] == r2) { + * swap(r0, [r1]); + * } else { + * r0 = [r1]; + * } + * + * so "r0 == r2" means that the operation was successfull. + */ + +static ngx_inline ngx_atomic_int_t +ngx_atomic_inc(ngx_atomic_t *value) { - uint32_t old, new, res; + ngx_atomic_int_t old, new, res; old = *value; @@ -100,9 +221,36 @@ static ngx_inline uint32_t ngx_atomic_in __asm__ volatile ( - "casa [%1] 0x80, %2, %0" + NGX_CASXA " [%1] 0x80, %2, %0" + + : "+r" (res) : "r" (value), "r" (old) : "memory"); + + if (res == old) { + return new; + } + + old = res; + } +} + - : "+r" (res) : "r" (value), "r" (old)); +static ngx_inline ngx_atomic_int_t +ngx_atomic_dec(ngx_atomic_t *value) +{ + ngx_atomic_int_t old, new, res; + + old = *value; + + for ( ;; ) { + + new = old - 1; + res = new; + + __asm__ volatile ( + + NGX_CASXA " [%1] 0x80, %2, %0" + + : "+r" (res) : "r" (value), "r" (old) : "memory"); if (res == old) { return new; @@ -113,33 +261,128 @@ static ngx_inline uint32_t ngx_atomic_in } -static ngx_inline uint32_t ngx_atomic_cmp_set(ngx_atomic_t *lock, - ngx_atomic_t old, - ngx_atomic_t set) +static ngx_inline ngx_atomic_int_t +ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_int_t old, ngx_atomic_int_t set) { - uint32_t res = (uint32_t) set; + __asm__ volatile ( + + NGX_CASXA " [%1] 0x80, %2, %0" + + : "+r" (set) : "r" (lock), "r" (old) : "memory"); + + return (set == old); +} + + +#elif ( __ppc__ ) + +#define NGX_HAVE_ATOMIC_OPS 1 + +#if (NGX_PTR_SIZE == 8) +typedef uint64_t ngx_atomic_int_t; +#define NGX_ATOMIC_T_LEN sizeof("-9223372036854775808") - 1 +#else +#define NGX_ATOMIC_T_LEN sizeof("-2147483648") - 1 +typedef uint32_t ngx_atomic_int_t; +#endif + +typedef volatile ngx_atomic_int_t ngx_atomic_t; + + +/* + * the ppc assembler treats ";" as comment, so we have to use "\n". + * the minus in "bne-" is a hint for the branch prediction unit that + * this branch is unlikely to be taken. + * + * the "=&r" means that no input registers can be used. + * the "=&b" means that the base registers can be used only, i.e. any register + * except r0. the r0 register can not be used in "addi r0, r0, 1". + * the "1b" means the nearest backward label "1" and the "1f" means + * the nearest forward label "1". + */ + +static ngx_inline ngx_atomic_int_t +ngx_atomic_inc(ngx_atomic_t *value) +{ + ngx_atomic_int_t res; __asm__ volatile ( - "casa [%1] 0x80, %2, %0" + "1: lwarx %0, 0, %1 \n" /* load from [value] into "res" */ + /* and store reservation */ + " addi %0, %0, 1 \n" /* add "1" to "res" */ + " stwcx. %0, 0, %1 \n" /* store "res" into [value] if reservation */ + /* is not cleared */ + " bne- 1b \n" /* try again if reservation was cleared */ + + : "=&b" (res) : "r" (value) : "cc", "memory"); + + return res; +} + + +static ngx_inline ngx_atomic_int_t +ngx_atomic_dec(ngx_atomic_t *value) +{ + ngx_atomic_int_t res; + + __asm__ volatile ( + + "1: lwarx %0, 0, %1 \n" /* load from [value] into "res" */ + /* and store reservation */ + " addi %0, %0, -1 \n" /* sub "1" from "res" */ + " stwcx. %0, 0, %1 \n" /* store "res" into [value] if reservation */ + /* is not cleared */ + " bne- 1b \n" /* try again if reservation was cleared */ + + : "=&b" (res) : "r" (value) : "cc", "memory"); - : "+r" (res) : "r" (lock), "r" (old)); + return res; +} + + +static ngx_inline ngx_atomic_int_t +ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_int_t old, + ngx_atomic_int_t set) +{ + ngx_atomic_int_t res, temp; + + __asm__ volatile ( - return (res == old); + " li %0, 0 \n" /* preset "0" to "res" */ + " lwarx %1, 0, %2 \n" /* load from [lock] into "temp" */ + /* and store reservation */ + " cmpw %1, %3 \n" /* compare "temp" and "old" */ + " bne- 1f \n" /* not equal */ + " stwcx. %4, 0, %2 \n" /* store "set" into [lock] if reservation */ + /* is not cleared */ + " bne- 1f \n" /* the reservation was cleared */ + " li %0, 1 \n" /* set "1" to "res" */ + "1: \n" + + : "=&r" (res), "=&r" (temp) + : "r" (lock), "r" (old), "r" (set) + : "cc", "memory"); + + return res; } + #else #define NGX_HAVE_ATOMIC_OPS 0 -typedef volatile uint32_t ngx_atomic_t; +typedef uint32_t ngx_atomic_int_t; +typedef volatile ngx_atomic_int_t ngx_atomic_t; -#define ngx_atomic_inc(x) ++(*(x)); +#define ngx_atomic_inc(x) ++(*(x)) +#define ngx_atomic_dec(x) --(*(x)) -static ngx_inline uint32_t ngx_atomic_cmp_set(ngx_atomic_t *lock, - ngx_atomic_t old, - ngx_atomic_t set) +static ngx_inline ngx_atomic_int_t +ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_int_t old, + ngx_atomic_int_t set) { + *lock = set; return 1; }