Mercurial > hg > nginx
annotate src/http/modules/ngx_http_upstream_ip_hash_module.c @ 4207:4fc91bae6f83
Better recheck of dead upstream servers.
Previously nginx used to mark backend again as live as soon as fail_timeout
passes (10s by default) since last failure. On the other hand, detecting
dead backend takes up to 60s (proxy_connect_timeout) in typical situation
"backend is down and doesn't respond to any packets". This resulted in
suboptimal behaviour in the above situation (up to 23% of requests were
directed to dead backend with default settings).
More detailed description of the problem may be found here (in Russian):
http://mailman.nginx.org/pipermail/nginx-ru/2011-August/042172.html
Fix is to only allow one request after fail_timeout passes, and
mark backend as "live" only if this request succeeds.
Note that with new code backend will not be marked "live" unless "check"
request is completed, and this may take a while in some specific workloads
(e.g. streaming). This is believed to be acceptable.
| author | Maxim Dounin <mdounin@mdounin.ru> |
|---|---|
| date | Wed, 12 Oct 2011 14:22:48 +0000 |
| parents | 2e91aecb9e57 |
| children | d620f497c50f |
| rev | line source |
|---|---|
| 884 | 1 |
| 2 /* | |
| 3 * Copyright (C) Igor Sysoev | |
| 4 */ | |
| 5 | |
| 6 | |
| 7 #include <ngx_config.h> | |
| 8 #include <ngx_core.h> | |
| 9 #include <ngx_http.h> | |
| 10 | |
| 11 | |
| 12 typedef struct { | |
| 13 /* the round robin data must be first */ | |
| 14 ngx_http_upstream_rr_peer_data_t rrp; | |
| 15 | |
| 16 ngx_uint_t hash; | |
| 17 | |
| 18 u_char addr[3]; | |
| 19 | |
| 20 u_char tries; | |
| 21 | |
| 22 ngx_event_get_peer_pt get_rr_peer; | |
| 23 } ngx_http_upstream_ip_hash_peer_data_t; | |
| 24 | |
| 25 | |
| 26 static ngx_int_t ngx_http_upstream_init_ip_hash_peer(ngx_http_request_t *r, | |
| 27 ngx_http_upstream_srv_conf_t *us); | |
| 28 static ngx_int_t ngx_http_upstream_get_ip_hash_peer(ngx_peer_connection_t *pc, | |
| 29 void *data); | |
| 30 static char *ngx_http_upstream_ip_hash(ngx_conf_t *cf, ngx_command_t *cmd, | |
| 31 void *conf); | |
| 32 | |
| 33 | |
| 34 static ngx_command_t ngx_http_upstream_ip_hash_commands[] = { | |
| 35 | |
| 36 { ngx_string("ip_hash"), | |
| 37 NGX_HTTP_UPS_CONF|NGX_CONF_NOARGS, | |
| 38 ngx_http_upstream_ip_hash, | |
| 39 0, | |
| 40 0, | |
| 41 NULL }, | |
| 42 | |
| 43 ngx_null_command | |
| 44 }; | |
| 45 | |
| 46 | |
| 47 static ngx_http_module_t ngx_http_upstream_ip_hash_module_ctx = { | |
| 48 NULL, /* preconfiguration */ | |
| 49 NULL, /* postconfiguration */ | |
| 50 | |
| 51 NULL, /* create main configuration */ | |
| 52 NULL, /* init main configuration */ | |
| 53 | |
| 54 NULL, /* create server configuration */ | |
| 55 NULL, /* merge server configuration */ | |
| 56 | |
| 57 NULL, /* create location configuration */ | |
| 58 NULL /* merge location configuration */ | |
| 59 }; | |
| 60 | |
| 61 | |
| 62 ngx_module_t ngx_http_upstream_ip_hash_module = { | |
| 63 NGX_MODULE_V1, | |
| 64 &ngx_http_upstream_ip_hash_module_ctx, /* module context */ | |
| 65 ngx_http_upstream_ip_hash_commands, /* module directives */ | |
| 66 NGX_HTTP_MODULE, /* module type */ | |
| 67 NULL, /* init master */ | |
| 68 NULL, /* init module */ | |
| 69 NULL, /* init process */ | |
| 70 NULL, /* init thread */ | |
| 71 NULL, /* exit thread */ | |
| 72 NULL, /* exit process */ | |
| 73 NULL, /* exit master */ | |
| 74 NGX_MODULE_V1_PADDING | |
| 75 }; | |
| 76 | |
| 77 | |
| 78 ngx_int_t | |
| 79 ngx_http_upstream_init_ip_hash(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) | |
| 80 { | |
| 81 if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) { | |
| 82 return NGX_ERROR; | |
| 83 } | |
| 84 | |
| 85 us->peer.init = ngx_http_upstream_init_ip_hash_peer; | |
| 86 | |
| 87 return NGX_OK; | |
| 88 } | |
| 89 | |
| 90 | |
| 91 static ngx_int_t | |
| 92 ngx_http_upstream_init_ip_hash_peer(ngx_http_request_t *r, | |
| 93 ngx_http_upstream_srv_conf_t *us) | |
| 94 { | |
|
1144
324a5a3fe27f
ip_hash used wrong byte order
Igor Sysoev <igor@sysoev.ru>
parents:
1129
diff
changeset
|
95 u_char *p; |
| 884 | 96 struct sockaddr_in *sin; |
| 97 ngx_http_upstream_ip_hash_peer_data_t *iphp; | |
| 98 | |
| 99 iphp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_ip_hash_peer_data_t)); | |
| 100 if (iphp == NULL) { | |
| 101 return NGX_ERROR; | |
| 102 } | |
| 103 | |
| 104 r->upstream->peer.data = &iphp->rrp; | |
| 105 | |
| 106 if (ngx_http_upstream_init_round_robin_peer(r, us) != NGX_OK) { | |
| 107 return NGX_ERROR; | |
| 108 } | |
| 109 | |
| 110 r->upstream->peer.get = ngx_http_upstream_get_ip_hash_peer; | |
| 111 | |
| 112 /* AF_INET only */ | |
|
2512
2e91aecb9e57
a prelimiary IPv6 support, HTTP listen
Igor Sysoev <igor@sysoev.ru>
parents:
1418
diff
changeset
|
113 |
|
2e91aecb9e57
a prelimiary IPv6 support, HTTP listen
Igor Sysoev <igor@sysoev.ru>
parents:
1418
diff
changeset
|
114 if (r->connection->sockaddr->sa_family == AF_INET) { |
|
2e91aecb9e57
a prelimiary IPv6 support, HTTP listen
Igor Sysoev <igor@sysoev.ru>
parents:
1418
diff
changeset
|
115 |
|
2e91aecb9e57
a prelimiary IPv6 support, HTTP listen
Igor Sysoev <igor@sysoev.ru>
parents:
1418
diff
changeset
|
116 sin = (struct sockaddr_in *) r->connection->sockaddr; |
|
2e91aecb9e57
a prelimiary IPv6 support, HTTP listen
Igor Sysoev <igor@sysoev.ru>
parents:
1418
diff
changeset
|
117 p = (u_char *) &sin->sin_addr.s_addr; |
|
2e91aecb9e57
a prelimiary IPv6 support, HTTP listen
Igor Sysoev <igor@sysoev.ru>
parents:
1418
diff
changeset
|
118 iphp->addr[0] = p[0]; |
|
2e91aecb9e57
a prelimiary IPv6 support, HTTP listen
Igor Sysoev <igor@sysoev.ru>
parents:
1418
diff
changeset
|
119 iphp->addr[1] = p[1]; |
|
2e91aecb9e57
a prelimiary IPv6 support, HTTP listen
Igor Sysoev <igor@sysoev.ru>
parents:
1418
diff
changeset
|
120 iphp->addr[2] = p[2]; |
|
2e91aecb9e57
a prelimiary IPv6 support, HTTP listen
Igor Sysoev <igor@sysoev.ru>
parents:
1418
diff
changeset
|
121 |
|
2e91aecb9e57
a prelimiary IPv6 support, HTTP listen
Igor Sysoev <igor@sysoev.ru>
parents:
1418
diff
changeset
|
122 } else { |
|
2e91aecb9e57
a prelimiary IPv6 support, HTTP listen
Igor Sysoev <igor@sysoev.ru>
parents:
1418
diff
changeset
|
123 iphp->addr[0] = 0; |
|
2e91aecb9e57
a prelimiary IPv6 support, HTTP listen
Igor Sysoev <igor@sysoev.ru>
parents:
1418
diff
changeset
|
124 iphp->addr[1] = 0; |
|
2e91aecb9e57
a prelimiary IPv6 support, HTTP listen
Igor Sysoev <igor@sysoev.ru>
parents:
1418
diff
changeset
|
125 iphp->addr[2] = 0; |
|
2e91aecb9e57
a prelimiary IPv6 support, HTTP listen
Igor Sysoev <igor@sysoev.ru>
parents:
1418
diff
changeset
|
126 } |
| 884 | 127 |
| 128 iphp->hash = 89; | |
| 129 iphp->tries = 0; | |
| 130 iphp->get_rr_peer = ngx_http_upstream_get_round_robin_peer; | |
| 131 | |
| 132 return NGX_OK; | |
| 133 } | |
| 134 | |
| 135 | |
| 136 static ngx_int_t | |
| 137 ngx_http_upstream_get_ip_hash_peer(ngx_peer_connection_t *pc, void *data) | |
| 138 { | |
| 139 ngx_http_upstream_ip_hash_peer_data_t *iphp = data; | |
| 140 | |
| 141 time_t now; | |
| 142 uintptr_t m; | |
| 143 ngx_uint_t i, n, p, hash; | |
| 144 ngx_http_upstream_rr_peer_t *peer; | |
| 145 | |
| 146 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, | |
| 147 "get ip hash peer, try: %ui", pc->tries); | |
| 148 | |
| 149 /* TODO: cached */ | |
| 150 | |
|
1418
acb1f441e7b2
update ip_hash to "backup" option
Igor Sysoev <igor@sysoev.ru>
parents:
1417
diff
changeset
|
151 if (iphp->tries > 20 || iphp->rrp.peers->single) { |
| 884 | 152 return iphp->get_rr_peer(pc, &iphp->rrp); |
| 153 } | |
| 154 | |
| 155 now = ngx_time(); | |
| 156 | |
| 157 pc->cached = 0; | |
| 158 pc->connection = NULL; | |
| 159 | |
| 160 hash = iphp->hash; | |
| 161 | |
| 162 for ( ;; ) { | |
| 163 | |
| 164 for (i = 0; i < 3; i++) { | |
| 165 hash = (hash * 113 + iphp->addr[i]) % 6271; | |
| 166 } | |
| 167 | |
| 168 p = hash % iphp->rrp.peers->number; | |
| 169 | |
| 170 n = p / (8 * sizeof(uintptr_t)); | |
|
1416
ad2311c943a3
fix ip_hash on 64-bit platform
Igor Sysoev <igor@sysoev.ru>
parents:
1284
diff
changeset
|
171 m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t)); |
| 884 | 172 |
| 173 if (!(iphp->rrp.tried[n] & m)) { | |
| 174 | |
| 175 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0, | |
| 176 "get ip hash peer, hash: %ui %04XA", p, m); | |
| 177 | |
| 178 peer = &iphp->rrp.peers->peer[p]; | |
| 179 | |
| 180 /* ngx_lock_mutex(iphp->rrp.peers->mutex); */ | |
| 181 | |
| 182 if (!peer->down) { | |
| 183 | |
| 1025 | 184 if (peer->max_fails == 0 || peer->fails < peer->max_fails) { |
| 185 break; | |
| 186 } | |
| 884 | 187 |
|
4207
4fc91bae6f83
Better recheck of dead upstream servers.
Maxim Dounin <mdounin@mdounin.ru>
parents:
2512
diff
changeset
|
188 if (now - peer->checked > peer->fail_timeout) { |
|
4fc91bae6f83
Better recheck of dead upstream servers.
Maxim Dounin <mdounin@mdounin.ru>
parents:
2512
diff
changeset
|
189 peer->checked = now; |
| 1025 | 190 break; |
| 191 } | |
| 1129 | 192 } |
| 884 | 193 |
| 1129 | 194 iphp->rrp.tried[n] |= m; |
| 884 | 195 |
| 196 /* ngx_unlock_mutex(iphp->rrp.peers->mutex); */ | |
| 197 | |
| 198 pc->tries--; | |
| 199 } | |
| 200 | |
| 201 if (++iphp->tries >= 20) { | |
| 202 return iphp->get_rr_peer(pc, &iphp->rrp); | |
| 203 } | |
| 204 } | |
| 205 | |
|
1417
b23a80f9a7b8
set current peer to use it in ngx_http_upstream_free_round_robin_peer()
Igor Sysoev <igor@sysoev.ru>
parents:
1416
diff
changeset
|
206 iphp->rrp.current = p; |
|
b23a80f9a7b8
set current peer to use it in ngx_http_upstream_free_round_robin_peer()
Igor Sysoev <igor@sysoev.ru>
parents:
1416
diff
changeset
|
207 |
| 884 | 208 pc->sockaddr = peer->sockaddr; |
| 209 pc->socklen = peer->socklen; | |
| 210 pc->name = &peer->name; | |
| 211 | |
| 212 /* ngx_unlock_mutex(iphp->rrp.peers->mutex); */ | |
| 213 | |
| 214 iphp->rrp.tried[n] |= m; | |
| 215 iphp->hash = hash; | |
| 216 | |
| 217 return NGX_OK; | |
| 218 } | |
| 219 | |
| 220 | |
| 221 static char * | |
| 222 ngx_http_upstream_ip_hash(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) | |
| 223 { | |
| 224 ngx_http_upstream_srv_conf_t *uscf; | |
| 225 | |
| 226 uscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_upstream_module); | |
| 227 | |
| 228 uscf->peer.init_upstream = ngx_http_upstream_init_ip_hash; | |
| 229 | |
| 230 uscf->flags = NGX_HTTP_UPSTREAM_CREATE | |
| 231 |NGX_HTTP_UPSTREAM_MAX_FAILS | |
| 232 |NGX_HTTP_UPSTREAM_FAIL_TIMEOUT | |
| 233 |NGX_HTTP_UPSTREAM_DOWN; | |
| 234 | |
| 235 return NGX_CONF_OK; | |
| 236 } |