annotate src/core/ngx_rwlock.c @ 8018:5119c8150478

Fixed runtime handling of systems without EPOLLRDHUP support. In 7583:efd71d49bde0 (nginx 1.17.5) along with introduction of the ioctl(FIONREAD) support proper handling of systems without EPOLLRDHUP support in the kernel (but with EPOLLRDHUP in headers) was broken. Before the change, rev->available was never set to 0 unless ngx_use_epoll_rdhup was also set (that is, runtime test for EPOLLRDHUP introduced in 6536:f7849bfb6d21 succeeded). After the change, rev->available might reach 0 on systems without runtime EPOLLRDHUP support, stopping further reading in ngx_readv_chain() and ngx_unix_recv(). And, if EOF happened to be already reported along with the last event, it is not reported again by epoll_wait(), leading to connection hangs and timeouts on such systems. This affects Linux kernels before 2.6.17 if nginx was compiled with newer headers, and, more importantly, emulation layers, such as DigitalOcean's App Platform's / gVisor's epoll emulation layer. Fix is to explicitly check ngx_use_epoll_rdhup before the corresponding rev->pending_eof tests in ngx_readv_chain() and ngx_unix_recv().
author Marcus Ball <marcus.ball@live.com>
date Mon, 30 May 2022 02:38:07 +0300
parents 7752d8523066
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
6101
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
1
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
2 /*
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
3 * Copyright (C) Ruslan Ermilov
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
4 * Copyright (C) Nginx, Inc.
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
5 */
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
6
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
7
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
8 #include <ngx_config.h>
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
9 #include <ngx_core.h>
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
10
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
11
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
12 #if (NGX_HAVE_ATOMIC_OPS)
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
13
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
14
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
15 #define NGX_RWLOCK_SPIN 2048
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
16 #define NGX_RWLOCK_WLOCK ((ngx_atomic_uint_t) -1)
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
17
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
18
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
19 void
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
20 ngx_rwlock_wlock(ngx_atomic_t *lock)
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
21 {
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
22 ngx_uint_t i, n;
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
23
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
24 for ( ;; ) {
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
25
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
26 if (*lock == 0 && ngx_atomic_cmp_set(lock, 0, NGX_RWLOCK_WLOCK)) {
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
27 return;
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
28 }
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
29
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
30 if (ngx_ncpu > 1) {
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
31
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
32 for (n = 1; n < NGX_RWLOCK_SPIN; n <<= 1) {
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
33
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
34 for (i = 0; i < n; i++) {
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
35 ngx_cpu_pause();
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
36 }
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
37
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
38 if (*lock == 0
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
39 && ngx_atomic_cmp_set(lock, 0, NGX_RWLOCK_WLOCK))
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
40 {
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
41 return;
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
42 }
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
43 }
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
44 }
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
45
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
46 ngx_sched_yield();
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
47 }
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
48 }
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
49
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
50
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
51 void
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
52 ngx_rwlock_rlock(ngx_atomic_t *lock)
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
53 {
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
54 ngx_uint_t i, n;
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
55 ngx_atomic_uint_t readers;
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
56
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
57 for ( ;; ) {
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
58 readers = *lock;
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
59
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
60 if (readers != NGX_RWLOCK_WLOCK
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
61 && ngx_atomic_cmp_set(lock, readers, readers + 1))
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
62 {
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
63 return;
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
64 }
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
65
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
66 if (ngx_ncpu > 1) {
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
67
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
68 for (n = 1; n < NGX_RWLOCK_SPIN; n <<= 1) {
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
69
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
70 for (i = 0; i < n; i++) {
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
71 ngx_cpu_pause();
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
72 }
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
73
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
74 readers = *lock;
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
75
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
76 if (readers != NGX_RWLOCK_WLOCK
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
77 && ngx_atomic_cmp_set(lock, readers, readers + 1))
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
78 {
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
79 return;
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
80 }
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
81 }
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
82 }
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
83
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
84 ngx_sched_yield();
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
85 }
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
86 }
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
87
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
88
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
89 void
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
90 ngx_rwlock_unlock(ngx_atomic_t *lock)
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
91 {
7995
7752d8523066 Core: simplify reader lock release.
Pavel Pautov <p.pautov@f5.com>
parents: 7038
diff changeset
92 if (*lock == NGX_RWLOCK_WLOCK) {
7037
12efcdcb8a4b Added memory barrier semantics to ngx_rwlock_unlock().
Ruslan Ermilov <ru@nginx.com>
parents: 6270
diff changeset
93 (void) ngx_atomic_cmp_set(lock, NGX_RWLOCK_WLOCK, 0);
7995
7752d8523066 Core: simplify reader lock release.
Pavel Pautov <p.pautov@f5.com>
parents: 7038
diff changeset
94 } else {
7752d8523066 Core: simplify reader lock release.
Pavel Pautov <p.pautov@f5.com>
parents: 7038
diff changeset
95 (void) ngx_atomic_fetch_add(lock, -1);
6101
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
96 }
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
97 }
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
98
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
99
7038
d1816a2696de Introduced ngx_rwlock_downgrade().
Ruslan Ermilov <ru@nginx.com>
parents: 7037
diff changeset
100 void
d1816a2696de Introduced ngx_rwlock_downgrade().
Ruslan Ermilov <ru@nginx.com>
parents: 7037
diff changeset
101 ngx_rwlock_downgrade(ngx_atomic_t *lock)
d1816a2696de Introduced ngx_rwlock_downgrade().
Ruslan Ermilov <ru@nginx.com>
parents: 7037
diff changeset
102 {
d1816a2696de Introduced ngx_rwlock_downgrade().
Ruslan Ermilov <ru@nginx.com>
parents: 7037
diff changeset
103 if (*lock == NGX_RWLOCK_WLOCK) {
d1816a2696de Introduced ngx_rwlock_downgrade().
Ruslan Ermilov <ru@nginx.com>
parents: 7037
diff changeset
104 *lock = 1;
d1816a2696de Introduced ngx_rwlock_downgrade().
Ruslan Ermilov <ru@nginx.com>
parents: 7037
diff changeset
105 }
d1816a2696de Introduced ngx_rwlock_downgrade().
Ruslan Ermilov <ru@nginx.com>
parents: 7037
diff changeset
106 }
d1816a2696de Introduced ngx_rwlock_downgrade().
Ruslan Ermilov <ru@nginx.com>
parents: 7037
diff changeset
107
d1816a2696de Introduced ngx_rwlock_downgrade().
Ruslan Ermilov <ru@nginx.com>
parents: 7037
diff changeset
108
6103
79ddb0bdb273 Upstream: the "zone" directive.
Ruslan Ermilov <ru@nginx.com>
parents: 6101
diff changeset
109 #else
79ddb0bdb273 Upstream: the "zone" directive.
Ruslan Ermilov <ru@nginx.com>
parents: 6101
diff changeset
110
6270
e769cc88f996 Core: read/write locks are also required by the Stream module.
Piotr Sikora <piotrsikora@google.com>
parents: 6103
diff changeset
111 #if (NGX_HTTP_UPSTREAM_ZONE || NGX_STREAM_UPSTREAM_ZONE)
6103
79ddb0bdb273 Upstream: the "zone" directive.
Ruslan Ermilov <ru@nginx.com>
parents: 6101
diff changeset
112
79ddb0bdb273 Upstream: the "zone" directive.
Ruslan Ermilov <ru@nginx.com>
parents: 6101
diff changeset
113 #error ngx_atomic_cmp_set() is not defined!
79ddb0bdb273 Upstream: the "zone" directive.
Ruslan Ermilov <ru@nginx.com>
parents: 6101
diff changeset
114
6101
682d8222c6b1 Core: read/write locks.
Ruslan Ermilov <ru@nginx.com>
parents:
diff changeset
115 #endif
6103
79ddb0bdb273 Upstream: the "zone" directive.
Ruslan Ermilov <ru@nginx.com>
parents: 6101
diff changeset
116
79ddb0bdb273 Upstream: the "zone" directive.
Ruslan Ermilov <ru@nginx.com>
parents: 6101
diff changeset
117 #endif