# HG changeset patch # User Ruslan Ermilov # Date 1428660983 -10800 # Node ID 6ff0ebd6fbf467dc1970cf82c1dad452173b609e # Parent ac34eff7e1474be6a1cf4731e7f9385392373414 Upstream: track the number of active connections to upstreams. This also simplifies the implementation of the least_conn module. diff --git a/src/http/modules/ngx_http_upstream_hash_module.c b/src/http/modules/ngx_http_upstream_hash_module.c --- a/src/http/modules/ngx_http_upstream_hash_module.c +++ b/src/http/modules/ngx_http_upstream_hash_module.c @@ -262,6 +262,8 @@ ngx_http_upstream_get_hash_peer(ngx_peer pc->socklen = peer->socklen; pc->name = &peer->name; + peer->conns++; + if (now - peer->checked > peer->fail_timeout) { peer->checked = now; } @@ -562,6 +564,8 @@ ngx_http_upstream_get_chash_peer(ngx_pee pc->socklen = best->socklen; pc->name = &best->name; + best->conns++; + return NGX_OK; } diff --git a/src/http/modules/ngx_http_upstream_ip_hash_module.c b/src/http/modules/ngx_http_upstream_ip_hash_module.c --- a/src/http/modules/ngx_http_upstream_ip_hash_module.c +++ b/src/http/modules/ngx_http_upstream_ip_hash_module.c @@ -242,6 +242,8 @@ ngx_http_upstream_get_ip_hash_peer(ngx_p pc->socklen = peer->socklen; pc->name = &peer->name; + peer->conns++; + if (now - peer->checked > peer->fail_timeout) { peer->checked = now; } diff --git a/src/http/modules/ngx_http_upstream_least_conn_module.c b/src/http/modules/ngx_http_upstream_least_conn_module.c --- a/src/http/modules/ngx_http_upstream_least_conn_module.c +++ b/src/http/modules/ngx_http_upstream_least_conn_module.c @@ -10,29 +10,10 @@ #include -typedef struct { - ngx_uint_t *conns; -} ngx_http_upstream_least_conn_conf_t; - - -typedef struct { - /* the round robin data must be first */ - ngx_http_upstream_rr_peer_data_t rrp; - - ngx_uint_t *conns; - - ngx_event_get_peer_pt get_rr_peer; - ngx_event_free_peer_pt free_rr_peer; -} ngx_http_upstream_lc_peer_data_t; - - static ngx_int_t ngx_http_upstream_init_least_conn_peer(ngx_http_request_t *r, ngx_http_upstream_srv_conf_t *us); static ngx_int_t ngx_http_upstream_get_least_conn_peer( ngx_peer_connection_t *pc, void *data); -static void ngx_http_upstream_free_least_conn_peer(ngx_peer_connection_t *pc, - void *data, ngx_uint_t state); -static void *ngx_http_upstream_least_conn_create_conf(ngx_conf_t *cf); static char *ngx_http_upstream_least_conn(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); @@ -57,7 +38,7 @@ static ngx_http_module_t ngx_http_upstr NULL, /* create main configuration */ NULL, /* init main configuration */ - ngx_http_upstream_least_conn_create_conf, /* create server configuration */ + NULL, /* create server configuration */ NULL, /* merge server configuration */ NULL, /* create location configuration */ @@ -85,10 +66,6 @@ static ngx_int_t ngx_http_upstream_init_least_conn(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) { - ngx_uint_t n; - ngx_http_upstream_rr_peers_t *peers; - ngx_http_upstream_least_conn_conf_t *lcf; - ngx_log_debug0(NGX_LOG_DEBUG_HTTP, cf->log, 0, "init least conn"); @@ -96,22 +73,6 @@ ngx_http_upstream_init_least_conn(ngx_co return NGX_ERROR; } - peers = us->peer.data; - - n = peers->number; - - if (peers->next) { - n += peers->next->number; - } - - lcf = ngx_http_conf_upstream_srv_conf(us, - ngx_http_upstream_least_conn_module); - - lcf->conns = ngx_pcalloc(cf->pool, sizeof(ngx_uint_t) * n); - if (lcf->conns == NULL) { - return NGX_ERROR; - } - us->peer.init = ngx_http_upstream_init_least_conn_peer; return NGX_OK; @@ -122,33 +83,14 @@ static ngx_int_t ngx_http_upstream_init_least_conn_peer(ngx_http_request_t *r, ngx_http_upstream_srv_conf_t *us) { - ngx_http_upstream_lc_peer_data_t *lcp; - ngx_http_upstream_least_conn_conf_t *lcf; - ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "init least conn peer"); - lcf = ngx_http_conf_upstream_srv_conf(us, - ngx_http_upstream_least_conn_module); - - lcp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_lc_peer_data_t)); - if (lcp == NULL) { - return NGX_ERROR; - } - - lcp->conns = lcf->conns; - - r->upstream->peer.data = &lcp->rrp; - if (ngx_http_upstream_init_round_robin_peer(r, us) != NGX_OK) { return NGX_ERROR; } r->upstream->peer.get = ngx_http_upstream_get_least_conn_peer; - r->upstream->peer.free = ngx_http_upstream_free_least_conn_peer; - - lcp->get_rr_peer = ngx_http_upstream_get_round_robin_peer; - lcp->free_rr_peer = ngx_http_upstream_free_round_robin_peer; return NGX_OK; } @@ -157,7 +99,7 @@ ngx_http_upstream_init_least_conn_peer(n static ngx_int_t ngx_http_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data) { - ngx_http_upstream_lc_peer_data_t *lcp = data; + ngx_http_upstream_rr_peer_data_t *rrp = data; time_t now; uintptr_t m; @@ -169,8 +111,8 @@ ngx_http_upstream_get_least_conn_peer(ng ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, "get least conn peer, try: %ui", pc->tries); - if (lcp->rrp.peers->single) { - return lcp->get_rr_peer(pc, &lcp->rrp); + if (rrp->peers->single) { + return ngx_http_upstream_get_round_robin_peer(pc, rrp); } pc->cached = 0; @@ -178,7 +120,7 @@ ngx_http_upstream_get_least_conn_peer(ng now = ngx_time(); - peers = lcp->rrp.peers; + peers = rrp->peers; best = NULL; total = 0; @@ -193,7 +135,7 @@ ngx_http_upstream_get_least_conn_peer(ng n = i / (8 * sizeof(uintptr_t)); m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t)); - if (lcp->rrp.tried[n] & m) { + if (rrp->tried[n] & m) { continue; } @@ -217,15 +159,13 @@ ngx_http_upstream_get_least_conn_peer(ng */ if (best == NULL - || lcp->conns[i] * best->weight < lcp->conns[p] * peer->weight) + || peer->conns * best->weight < best->conns * peer->weight) { best = peer; many = 0; p = i; - } else if (lcp->conns[i] * best->weight - == lcp->conns[p] * peer->weight) - { + } else if (peer->conns * best->weight == best->conns * peer->weight) { many = 1; } } @@ -246,7 +186,7 @@ ngx_http_upstream_get_least_conn_peer(ng n = i / (8 * sizeof(uintptr_t)); m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t)); - if (lcp->rrp.tried[n] & m) { + if (rrp->tried[n] & m) { continue; } @@ -256,7 +196,7 @@ ngx_http_upstream_get_least_conn_peer(ng continue; } - if (lcp->conns[i] * best->weight != lcp->conns[p] * peer->weight) { + if (peer->conns * best->weight != best->conns * peer->weight) { continue; } @@ -291,13 +231,14 @@ ngx_http_upstream_get_least_conn_peer(ng pc->socklen = best->socklen; pc->name = &best->name; - lcp->rrp.current = p; + best->conns++; + + rrp->current = p; n = p / (8 * sizeof(uintptr_t)); m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t)); - lcp->rrp.tried[n] |= m; - lcp->conns[p]++; + rrp->tried[n] |= m; return NGX_OK; @@ -307,18 +248,16 @@ failed: ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, "get least conn peer, backup servers"); - lcp->conns += peers->number; + rrp->peers = peers->next; - lcp->rrp.peers = peers->next; - - n = (lcp->rrp.peers->number + (8 * sizeof(uintptr_t) - 1)) + n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1)) / (8 * sizeof(uintptr_t)); for (i = 0; i < n; i++) { - lcp->rrp.tried[i] = 0; + rrp->tried[i] = 0; } - rc = ngx_http_upstream_get_least_conn_peer(pc, lcp); + rc = ngx_http_upstream_get_least_conn_peer(pc, rrp); if (rc != NGX_BUSY) { return rc; @@ -337,47 +276,6 @@ failed: } -static void -ngx_http_upstream_free_least_conn_peer(ngx_peer_connection_t *pc, - void *data, ngx_uint_t state) -{ - ngx_http_upstream_lc_peer_data_t *lcp = data; - - ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0, - "free least conn peer %ui %ui", pc->tries, state); - - if (lcp->rrp.peers->single) { - lcp->free_rr_peer(pc, &lcp->rrp, state); - return; - } - - lcp->conns[lcp->rrp.current]--; - - lcp->free_rr_peer(pc, &lcp->rrp, state); -} - - -static void * -ngx_http_upstream_least_conn_create_conf(ngx_conf_t *cf) -{ - ngx_http_upstream_least_conn_conf_t *conf; - - conf = ngx_pcalloc(cf->pool, - sizeof(ngx_http_upstream_least_conn_conf_t)); - if (conf == NULL) { - return NULL; - } - - /* - * set by ngx_pcalloc(): - * - * conf->conns = NULL; - */ - - return conf; -} - - static char * ngx_http_upstream_least_conn(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { diff --git a/src/http/ngx_http_upstream_round_robin.c b/src/http/ngx_http_upstream_round_robin.c --- a/src/http/ngx_http_upstream_round_robin.c +++ b/src/http/ngx_http_upstream_round_robin.c @@ -434,6 +434,8 @@ ngx_http_upstream_get_round_robin_peer(n pc->socklen = peer->socklen; pc->name = &peer->name; + peer->conns++; + /* ngx_unlock_mutex(peers->mutex); */ return NGX_OK; @@ -563,13 +565,16 @@ ngx_http_upstream_free_round_robin_peer( /* TODO: NGX_PEER_KEEPALIVE */ + peer = &rrp->peers->peer[rrp->current]; + if (rrp->peers->single) { + + peer->conns--; + pc->tries = 0; return; } - peer = &rrp->peers->peer[rrp->current]; - if (state & NGX_PEER_FAILED) { now = ngx_time(); @@ -602,6 +607,8 @@ ngx_http_upstream_free_round_robin_peer( } } + peer->conns--; + if (pc->tries) { pc->tries--; } diff --git a/src/http/ngx_http_upstream_round_robin.h b/src/http/ngx_http_upstream_round_robin.h --- a/src/http/ngx_http_upstream_round_robin.h +++ b/src/http/ngx_http_upstream_round_robin.h @@ -24,6 +24,8 @@ typedef struct { ngx_int_t effective_weight; ngx_int_t weight; + ngx_uint_t conns; + ngx_uint_t fails; time_t accessed; time_t checked;