Mercurial > hg > nginx
annotate src/http/modules/ngx_http_upstream_keepalive_module.c @ 4694:5b5c07dee156
Upstream keepalive: "single" parameter deprecated.
The original idea was to optimize edge cases in case of interchangeable
backends, i.e. don't establish a new connection if we have any one
cached. This causes more harm than good though, as it screws up
underlying balancer's idea about backends used and may result in
various unexpected problems.
| author | Maxim Dounin <mdounin@mdounin.ru> |
|---|---|
| date | Mon, 18 Jun 2012 14:23:42 +0000 |
| parents | d620f497c50f |
| children | 82336a9ce26d |
| rev | line source |
|---|---|
| 4127 | 1 |
| 2 /* | |
| 3 * Copyright (C) Maxim Dounin | |
| 4412 | 4 * Copyright (C) Nginx, Inc. |
| 4127 | 5 */ |
| 6 | |
| 7 | |
| 8 #include <ngx_config.h> | |
| 9 #include <ngx_core.h> | |
| 10 #include <ngx_http.h> | |
| 11 | |
| 12 | |
| 13 typedef struct { | |
| 14 ngx_uint_t max_cached; | |
| 15 | |
| 16 ngx_queue_t cache; | |
| 17 ngx_queue_t free; | |
| 18 | |
| 19 ngx_http_upstream_init_pt original_init_upstream; | |
| 20 ngx_http_upstream_init_peer_pt original_init_peer; | |
| 21 | |
| 22 } ngx_http_upstream_keepalive_srv_conf_t; | |
| 23 | |
| 24 | |
| 25 typedef struct { | |
| 26 ngx_http_upstream_keepalive_srv_conf_t *conf; | |
| 27 | |
| 28 ngx_http_upstream_t *upstream; | |
| 29 | |
| 30 void *data; | |
| 31 | |
| 32 ngx_event_get_peer_pt original_get_peer; | |
| 33 ngx_event_free_peer_pt original_free_peer; | |
| 34 | |
| 35 #if (NGX_HTTP_SSL) | |
| 36 ngx_event_set_peer_session_pt original_set_session; | |
| 37 ngx_event_save_peer_session_pt original_save_session; | |
| 38 #endif | |
| 39 | |
| 40 ngx_uint_t failed; /* unsigned:1 */ | |
| 41 | |
| 42 } ngx_http_upstream_keepalive_peer_data_t; | |
| 43 | |
| 44 | |
| 45 typedef struct { | |
| 46 ngx_http_upstream_keepalive_srv_conf_t *conf; | |
| 47 | |
| 48 ngx_queue_t queue; | |
| 49 ngx_connection_t *connection; | |
| 50 | |
| 51 socklen_t socklen; | |
| 52 u_char sockaddr[NGX_SOCKADDRLEN]; | |
| 53 | |
| 54 } ngx_http_upstream_keepalive_cache_t; | |
| 55 | |
| 56 | |
| 57 static ngx_int_t ngx_http_upstream_init_keepalive_peer(ngx_http_request_t *r, | |
| 58 ngx_http_upstream_srv_conf_t *us); | |
| 59 static ngx_int_t ngx_http_upstream_get_keepalive_peer(ngx_peer_connection_t *pc, | |
| 60 void *data); | |
| 61 static void ngx_http_upstream_free_keepalive_peer(ngx_peer_connection_t *pc, | |
| 62 void *data, ngx_uint_t state); | |
| 63 | |
| 64 static void ngx_http_upstream_keepalive_dummy_handler(ngx_event_t *ev); | |
| 65 static void ngx_http_upstream_keepalive_close_handler(ngx_event_t *ev); | |
| 66 static void ngx_http_upstream_keepalive_close(ngx_connection_t *c); | |
| 67 | |
| 68 | |
| 69 #if (NGX_HTTP_SSL) | |
| 70 static ngx_int_t ngx_http_upstream_keepalive_set_session( | |
| 71 ngx_peer_connection_t *pc, void *data); | |
| 72 static void ngx_http_upstream_keepalive_save_session(ngx_peer_connection_t *pc, | |
| 73 void *data); | |
| 74 #endif | |
| 75 | |
| 76 static void *ngx_http_upstream_keepalive_create_conf(ngx_conf_t *cf); | |
| 77 static char *ngx_http_upstream_keepalive(ngx_conf_t *cf, ngx_command_t *cmd, | |
| 78 void *conf); | |
| 79 | |
| 80 | |
| 81 static ngx_command_t ngx_http_upstream_keepalive_commands[] = { | |
| 82 | |
| 83 { ngx_string("keepalive"), | |
| 84 NGX_HTTP_UPS_CONF|NGX_CONF_TAKE12, | |
| 85 ngx_http_upstream_keepalive, | |
| 86 0, | |
| 87 0, | |
| 88 NULL }, | |
| 89 | |
| 90 ngx_null_command | |
| 91 }; | |
| 92 | |
| 93 | |
| 94 static ngx_http_module_t ngx_http_upstream_keepalive_module_ctx = { | |
| 95 NULL, /* preconfiguration */ | |
| 96 NULL, /* postconfiguration */ | |
| 97 | |
| 98 NULL, /* create main configuration */ | |
| 99 NULL, /* init main configuration */ | |
| 100 | |
| 101 ngx_http_upstream_keepalive_create_conf, /* create server configuration */ | |
| 102 NULL, /* merge server configuration */ | |
| 103 | |
| 104 NULL, /* create location configuration */ | |
| 105 NULL /* merge location configuration */ | |
| 106 }; | |
| 107 | |
| 108 | |
| 109 ngx_module_t ngx_http_upstream_keepalive_module = { | |
| 110 NGX_MODULE_V1, | |
| 111 &ngx_http_upstream_keepalive_module_ctx, /* module context */ | |
| 112 ngx_http_upstream_keepalive_commands, /* module directives */ | |
| 113 NGX_HTTP_MODULE, /* module type */ | |
| 114 NULL, /* init master */ | |
| 115 NULL, /* init module */ | |
| 116 NULL, /* init process */ | |
| 117 NULL, /* init thread */ | |
| 118 NULL, /* exit thread */ | |
| 119 NULL, /* exit process */ | |
| 120 NULL, /* exit master */ | |
| 121 NGX_MODULE_V1_PADDING | |
| 122 }; | |
| 123 | |
| 124 | |
| 125 static ngx_int_t | |
| 126 ngx_http_upstream_init_keepalive(ngx_conf_t *cf, | |
| 127 ngx_http_upstream_srv_conf_t *us) | |
| 128 { | |
| 129 ngx_uint_t i; | |
| 130 ngx_http_upstream_keepalive_srv_conf_t *kcf; | |
| 131 ngx_http_upstream_keepalive_cache_t *cached; | |
| 132 | |
| 133 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, cf->log, 0, | |
| 134 "init keepalive"); | |
| 135 | |
| 136 kcf = ngx_http_conf_upstream_srv_conf(us, | |
| 137 ngx_http_upstream_keepalive_module); | |
| 138 | |
| 139 if (kcf->original_init_upstream(cf, us) != NGX_OK) { | |
| 140 return NGX_ERROR; | |
| 141 } | |
| 142 | |
| 143 kcf->original_init_peer = us->peer.init; | |
| 144 | |
| 145 us->peer.init = ngx_http_upstream_init_keepalive_peer; | |
| 146 | |
| 147 /* allocate cache items and add to free queue */ | |
| 148 | |
| 149 cached = ngx_pcalloc(cf->pool, | |
| 150 sizeof(ngx_http_upstream_keepalive_cache_t) * kcf->max_cached); | |
| 151 if (cached == NULL) { | |
| 152 return NGX_ERROR; | |
| 153 } | |
| 154 | |
| 155 ngx_queue_init(&kcf->cache); | |
| 156 ngx_queue_init(&kcf->free); | |
| 157 | |
| 158 for (i = 0; i < kcf->max_cached; i++) { | |
| 159 ngx_queue_insert_head(&kcf->free, &cached[i].queue); | |
| 160 cached[i].conf = kcf; | |
| 161 } | |
| 162 | |
| 163 return NGX_OK; | |
| 164 } | |
| 165 | |
| 166 | |
| 167 static ngx_int_t | |
| 168 ngx_http_upstream_init_keepalive_peer(ngx_http_request_t *r, | |
| 169 ngx_http_upstream_srv_conf_t *us) | |
| 170 { | |
| 171 ngx_http_upstream_keepalive_peer_data_t *kp; | |
| 172 ngx_http_upstream_keepalive_srv_conf_t *kcf; | |
| 173 | |
| 174 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, | |
| 175 "init keepalive peer"); | |
| 176 | |
| 177 kcf = ngx_http_conf_upstream_srv_conf(us, | |
| 178 ngx_http_upstream_keepalive_module); | |
| 179 | |
| 180 kp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_keepalive_peer_data_t)); | |
| 181 if (kp == NULL) { | |
| 182 return NGX_ERROR; | |
| 183 } | |
| 184 | |
| 185 if (kcf->original_init_peer(r, us) != NGX_OK) { | |
| 186 return NGX_ERROR; | |
| 187 } | |
| 188 | |
| 189 kp->conf = kcf; | |
| 190 kp->upstream = r->upstream; | |
| 191 kp->data = r->upstream->peer.data; | |
| 192 kp->original_get_peer = r->upstream->peer.get; | |
| 193 kp->original_free_peer = r->upstream->peer.free; | |
| 194 | |
| 195 r->upstream->peer.data = kp; | |
| 196 r->upstream->peer.get = ngx_http_upstream_get_keepalive_peer; | |
| 197 r->upstream->peer.free = ngx_http_upstream_free_keepalive_peer; | |
| 198 | |
| 199 #if (NGX_HTTP_SSL) | |
| 200 kp->original_set_session = r->upstream->peer.set_session; | |
| 201 kp->original_save_session = r->upstream->peer.save_session; | |
| 202 r->upstream->peer.set_session = ngx_http_upstream_keepalive_set_session; | |
| 203 r->upstream->peer.save_session = ngx_http_upstream_keepalive_save_session; | |
| 204 #endif | |
| 205 | |
| 206 return NGX_OK; | |
| 207 } | |
| 208 | |
| 209 | |
| 210 static ngx_int_t | |
| 211 ngx_http_upstream_get_keepalive_peer(ngx_peer_connection_t *pc, void *data) | |
| 212 { | |
| 213 ngx_http_upstream_keepalive_peer_data_t *kp = data; | |
| 214 ngx_http_upstream_keepalive_cache_t *item; | |
| 215 | |
| 216 ngx_int_t rc; | |
| 217 ngx_queue_t *q, *cache; | |
| 218 ngx_connection_t *c; | |
| 219 | |
| 220 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, | |
| 221 "get keepalive peer"); | |
| 222 | |
| 223 kp->failed = 0; | |
| 224 | |
|
4694
5b5c07dee156
Upstream keepalive: "single" parameter deprecated.
Maxim Dounin <mdounin@mdounin.ru>
parents:
4412
diff
changeset
|
225 /* ask balancer */ |
| 4127 | 226 |
| 227 rc = kp->original_get_peer(pc, kp->data); | |
| 228 | |
|
4694
5b5c07dee156
Upstream keepalive: "single" parameter deprecated.
Maxim Dounin <mdounin@mdounin.ru>
parents:
4412
diff
changeset
|
229 if (rc != NGX_OK) { |
| 4127 | 230 return rc; |
| 231 } | |
| 232 | |
| 233 /* search cache for suitable connection */ | |
| 234 | |
| 235 cache = &kp->conf->cache; | |
| 236 | |
| 237 for (q = ngx_queue_head(cache); | |
| 238 q != ngx_queue_sentinel(cache); | |
| 239 q = ngx_queue_next(q)) | |
| 240 { | |
| 241 item = ngx_queue_data(q, ngx_http_upstream_keepalive_cache_t, queue); | |
| 242 c = item->connection; | |
| 243 | |
| 244 if (ngx_memn2cmp((u_char *) &item->sockaddr, (u_char *) pc->sockaddr, | |
| 245 item->socklen, pc->socklen) | |
| 246 == 0) | |
| 247 { | |
| 248 ngx_queue_remove(q); | |
| 249 ngx_queue_insert_head(&kp->conf->free, q); | |
| 250 | |
| 251 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, | |
| 252 "get keepalive peer: using connection %p", c); | |
| 253 | |
| 254 c->idle = 0; | |
| 255 c->log = pc->log; | |
| 256 c->read->log = pc->log; | |
| 257 c->write->log = pc->log; | |
| 258 c->pool->log = pc->log; | |
| 259 | |
| 260 pc->connection = c; | |
| 261 pc->cached = 1; | |
| 262 | |
| 263 return NGX_DONE; | |
| 264 } | |
| 265 } | |
| 266 | |
| 267 return NGX_OK; | |
| 268 } | |
| 269 | |
| 270 | |
| 271 static void | |
| 272 ngx_http_upstream_free_keepalive_peer(ngx_peer_connection_t *pc, void *data, | |
| 273 ngx_uint_t state) | |
| 274 { | |
| 275 ngx_http_upstream_keepalive_peer_data_t *kp = data; | |
| 276 ngx_http_upstream_keepalive_cache_t *item; | |
| 277 | |
| 278 ngx_queue_t *q; | |
| 279 ngx_connection_t *c; | |
| 280 ngx_http_upstream_t *u; | |
| 281 | |
| 282 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, | |
| 283 "free keepalive peer"); | |
| 284 | |
| 285 /* remember failed state - peer.free() may be called more than once */ | |
| 286 | |
| 287 if (state & NGX_PEER_FAILED) { | |
| 288 kp->failed = 1; | |
| 289 } | |
| 290 | |
| 291 /* cache valid connections */ | |
| 292 | |
| 293 u = kp->upstream; | |
| 294 c = pc->connection; | |
| 295 | |
| 296 if (kp->failed | |
| 297 || c == NULL | |
| 298 || c->read->eof | |
| 299 || c->read->error | |
| 300 || c->read->timedout | |
| 301 || c->write->error | |
| 302 || c->write->timedout) | |
| 303 { | |
| 304 goto invalid; | |
| 305 } | |
| 306 | |
| 307 if (!u->keepalive) { | |
| 308 goto invalid; | |
| 309 } | |
| 310 | |
| 311 if (ngx_handle_read_event(c->read, 0) != NGX_OK) { | |
| 312 goto invalid; | |
| 313 } | |
| 314 | |
| 315 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, | |
| 316 "free keepalive peer: saving connection %p", c); | |
| 317 | |
| 318 if (ngx_queue_empty(&kp->conf->free)) { | |
| 319 | |
| 320 q = ngx_queue_last(&kp->conf->cache); | |
| 321 ngx_queue_remove(q); | |
| 322 | |
| 323 item = ngx_queue_data(q, ngx_http_upstream_keepalive_cache_t, queue); | |
| 324 | |
| 325 ngx_http_upstream_keepalive_close(item->connection); | |
| 326 | |
| 327 } else { | |
| 328 q = ngx_queue_head(&kp->conf->free); | |
| 329 ngx_queue_remove(q); | |
| 330 | |
| 331 item = ngx_queue_data(q, ngx_http_upstream_keepalive_cache_t, queue); | |
| 332 } | |
| 333 | |
| 334 item->connection = c; | |
| 335 ngx_queue_insert_head(&kp->conf->cache, q); | |
| 336 | |
| 337 pc->connection = NULL; | |
| 338 | |
| 339 if (c->read->timer_set) { | |
| 340 ngx_del_timer(c->read); | |
| 341 } | |
| 342 if (c->write->timer_set) { | |
| 343 ngx_del_timer(c->write); | |
| 344 } | |
| 345 | |
| 346 c->write->handler = ngx_http_upstream_keepalive_dummy_handler; | |
| 347 c->read->handler = ngx_http_upstream_keepalive_close_handler; | |
| 348 | |
| 349 c->data = item; | |
| 350 c->idle = 1; | |
| 351 c->log = ngx_cycle->log; | |
| 352 c->read->log = ngx_cycle->log; | |
| 353 c->write->log = ngx_cycle->log; | |
| 354 c->pool->log = ngx_cycle->log; | |
| 355 | |
| 356 item->socklen = pc->socklen; | |
| 357 ngx_memcpy(&item->sockaddr, pc->sockaddr, pc->socklen); | |
| 358 | |
| 359 if (c->read->ready) { | |
| 360 ngx_http_upstream_keepalive_close_handler(c->read); | |
| 361 } | |
| 362 | |
| 363 invalid: | |
| 364 | |
| 365 kp->original_free_peer(pc, kp->data, state); | |
| 366 } | |
| 367 | |
| 368 | |
| 369 static void | |
| 370 ngx_http_upstream_keepalive_dummy_handler(ngx_event_t *ev) | |
| 371 { | |
| 372 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ev->log, 0, | |
| 373 "keepalive dummy handler"); | |
| 374 } | |
| 375 | |
| 376 | |
| 377 static void | |
| 378 ngx_http_upstream_keepalive_close_handler(ngx_event_t *ev) | |
| 379 { | |
| 380 ngx_http_upstream_keepalive_srv_conf_t *conf; | |
| 381 ngx_http_upstream_keepalive_cache_t *item; | |
| 382 | |
| 383 int n; | |
| 384 char buf[1]; | |
| 385 ngx_connection_t *c; | |
| 386 | |
| 387 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ev->log, 0, | |
| 388 "keepalive close handler"); | |
| 389 | |
| 390 c = ev->data; | |
| 391 | |
| 392 if (c->close) { | |
| 393 goto close; | |
| 394 } | |
| 395 | |
| 396 n = recv(c->fd, buf, 1, MSG_PEEK); | |
| 397 | |
| 398 if (n == -1 && ngx_socket_errno == NGX_EAGAIN) { | |
| 399 /* stale event */ | |
| 400 | |
| 401 if (ngx_handle_read_event(c->read, 0) != NGX_OK) { | |
| 402 goto close; | |
| 403 } | |
| 404 | |
| 405 return; | |
| 406 } | |
| 407 | |
| 408 close: | |
| 409 | |
| 410 item = c->data; | |
| 411 conf = item->conf; | |
| 412 | |
| 413 ngx_http_upstream_keepalive_close(c); | |
| 414 | |
| 415 ngx_queue_remove(&item->queue); | |
| 416 ngx_queue_insert_head(&conf->free, &item->queue); | |
| 417 } | |
| 418 | |
| 419 | |
| 420 static void | |
| 421 ngx_http_upstream_keepalive_close(ngx_connection_t *c) | |
| 422 { | |
| 423 | |
| 424 #if (NGX_HTTP_SSL) | |
| 425 | |
| 426 if (c->ssl) { | |
| 427 c->ssl->no_wait_shutdown = 1; | |
| 428 c->ssl->no_send_shutdown = 1; | |
| 429 | |
| 430 if (ngx_ssl_shutdown(c) == NGX_AGAIN) { | |
| 431 c->ssl->handler = ngx_http_upstream_keepalive_close; | |
| 432 return; | |
| 433 } | |
| 434 } | |
| 435 | |
| 436 #endif | |
| 437 | |
| 438 ngx_destroy_pool(c->pool); | |
| 439 ngx_close_connection(c); | |
| 440 } | |
| 441 | |
| 442 | |
| 443 #if (NGX_HTTP_SSL) | |
| 444 | |
| 445 static ngx_int_t | |
| 446 ngx_http_upstream_keepalive_set_session(ngx_peer_connection_t *pc, void *data) | |
| 447 { | |
| 448 ngx_http_upstream_keepalive_peer_data_t *kp = data; | |
| 449 | |
| 450 return kp->original_set_session(pc, kp->data); | |
| 451 } | |
| 452 | |
| 453 | |
| 454 static void | |
| 455 ngx_http_upstream_keepalive_save_session(ngx_peer_connection_t *pc, void *data) | |
| 456 { | |
| 457 ngx_http_upstream_keepalive_peer_data_t *kp = data; | |
| 458 | |
| 459 kp->original_save_session(pc, kp->data); | |
| 460 return; | |
| 461 } | |
| 462 | |
| 463 #endif | |
| 464 | |
| 465 | |
| 466 static void * | |
| 467 ngx_http_upstream_keepalive_create_conf(ngx_conf_t *cf) | |
| 468 { | |
| 469 ngx_http_upstream_keepalive_srv_conf_t *conf; | |
| 470 | |
| 471 conf = ngx_pcalloc(cf->pool, | |
| 472 sizeof(ngx_http_upstream_keepalive_srv_conf_t)); | |
| 473 if (conf == NULL) { | |
| 474 return NULL; | |
| 475 } | |
| 476 | |
| 477 /* | |
| 478 * set by ngx_pcalloc(): | |
| 479 * | |
| 480 * conf->original_init_upstream = NULL; | |
| 481 * conf->original_init_peer = NULL; | |
| 482 */ | |
| 483 | |
| 484 conf->max_cached = 1; | |
| 485 | |
| 486 return conf; | |
| 487 } | |
| 488 | |
| 489 | |
| 490 static char * | |
| 491 ngx_http_upstream_keepalive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) | |
| 492 { | |
| 493 ngx_http_upstream_srv_conf_t *uscf; | |
| 494 ngx_http_upstream_keepalive_srv_conf_t *kcf; | |
| 495 | |
| 496 ngx_int_t n; | |
| 497 ngx_str_t *value; | |
| 498 ngx_uint_t i; | |
| 499 | |
| 500 uscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_upstream_module); | |
| 501 | |
| 502 kcf = ngx_http_conf_upstream_srv_conf(uscf, | |
| 503 ngx_http_upstream_keepalive_module); | |
| 504 | |
| 505 kcf->original_init_upstream = uscf->peer.init_upstream | |
| 506 ? uscf->peer.init_upstream | |
| 507 : ngx_http_upstream_init_round_robin; | |
| 508 | |
| 509 uscf->peer.init_upstream = ngx_http_upstream_init_keepalive; | |
| 510 | |
| 511 /* read options */ | |
| 512 | |
| 513 value = cf->args->elts; | |
| 514 | |
| 515 n = ngx_atoi(value[1].data, value[1].len); | |
| 516 | |
| 517 if (n == NGX_ERROR || n == 0) { | |
| 518 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, | |
| 519 "invalid value \"%V\" in \"%V\" directive", | |
| 520 &value[1], &cmd->name); | |
| 521 return NGX_CONF_ERROR; | |
| 522 } | |
| 523 | |
| 524 kcf->max_cached = n; | |
| 525 | |
| 526 for (i = 2; i < cf->args->nelts; i++) { | |
| 527 | |
| 528 if (ngx_strcmp(value[i].data, "single") == 0) { | |
|
4694
5b5c07dee156
Upstream keepalive: "single" parameter deprecated.
Maxim Dounin <mdounin@mdounin.ru>
parents:
4412
diff
changeset
|
529 ngx_conf_log_error(NGX_LOG_WARN, cf, 0, |
|
5b5c07dee156
Upstream keepalive: "single" parameter deprecated.
Maxim Dounin <mdounin@mdounin.ru>
parents:
4412
diff
changeset
|
530 "the \"single\" parameter is deprecated"); |
| 4127 | 531 continue; |
| 532 } | |
| 533 | |
| 534 goto invalid; | |
| 535 } | |
| 536 | |
| 537 return NGX_CONF_OK; | |
| 538 | |
| 539 invalid: | |
| 540 | |
| 541 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, | |
| 542 "invalid parameter \"%V\"", &value[i]); | |
| 543 | |
| 544 return NGX_CONF_ERROR; | |
| 545 } |