Mercurial > hg > nginx
changeset 4724:0141b4aec0e4 stable-1.2
Merge of r4655, r4656, r4657, r4695, r4696: upstream changes.
*) Upstream: least_conn balancer module.
*) Upstream: weights and IPv6 support in ip_hash balancer.
*) Upstream keepalive: "single" parameter deprecated.
| author | Maxim Dounin <mdounin@mdounin.ru> |
|---|---|
| date | Mon, 02 Jul 2012 16:41:13 +0000 |
| parents | 68ac485abbba |
| children | 47936d1c98e9 |
| files | auto/modules auto/options auto/sources src/http/modules/ngx_http_upstream_ip_hash_module.c src/http/modules/ngx_http_upstream_keepalive_module.c src/http/modules/ngx_http_upstream_least_conn_module.c src/http/ngx_http_upstream_round_robin.c src/http/ngx_http_upstream_round_robin.h |
| diffstat | 8 files changed, 478 insertions(+), 47 deletions(-) [+] |
line wrap: on
line diff
--- a/auto/modules +++ b/auto/modules @@ -345,6 +345,11 @@ if [ $HTTP_UPSTREAM_IP_HASH = YES ]; the HTTP_SRCS="$HTTP_SRCS $HTTP_UPSTREAM_IP_HASH_SRCS" fi +if [ $HTTP_UPSTREAM_LEAST_CONN = YES ]; then + HTTP_MODULES="$HTTP_MODULES $HTTP_UPSTREAM_LEAST_CONN_MODULE" + HTTP_SRCS="$HTTP_SRCS $HTTP_UPSTREAM_LEAST_CONN_SRCS" +fi + if [ $HTTP_UPSTREAM_KEEPALIVE = YES ]; then HTTP_MODULES="$HTTP_MODULES $HTTP_UPSTREAM_KEEPALIVE_MODULE" HTTP_SRCS="$HTTP_SRCS $HTTP_UPSTREAM_KEEPALIVE_SRCS"
--- a/auto/options +++ b/auto/options @@ -96,6 +96,7 @@ HTTP_FLV=NO HTTP_MP4=NO HTTP_GZIP_STATIC=NO HTTP_UPSTREAM_IP_HASH=YES +HTTP_UPSTREAM_LEAST_CONN=YES HTTP_UPSTREAM_KEEPALIVE=YES # STUB @@ -243,6 +244,8 @@ use the \"--without-http_limit_conn_modu --without-http_empty_gif_module) HTTP_EMPTY_GIF=NO ;; --without-http_browser_module) HTTP_BROWSER=NO ;; --without-http_upstream_ip_hash_module) HTTP_UPSTREAM_IP_HASH=NO ;; + --without-http_upstream_least_conn_module) + HTTP_UPSTREAM_LEAST_CONN=NO ;; --without-http_upstream_keepalive_module) HTTP_UPSTREAM_KEEPALIVE=NO ;; --with-http_perl_module) HTTP_PERL=YES ;;
--- a/auto/sources +++ b/auto/sources @@ -479,6 +479,11 @@ HTTP_UPSTREAM_IP_HASH_MODULE=ngx_http_up HTTP_UPSTREAM_IP_HASH_SRCS=src/http/modules/ngx_http_upstream_ip_hash_module.c +HTTP_UPSTREAM_LEAST_CONN_MODULE=ngx_http_upstream_least_conn_module +HTTP_UPSTREAM_LEAST_CONN_SRCS=" \ + src/http/modules/ngx_http_upstream_least_conn_module.c" + + HTTP_UPSTREAM_KEEPALIVE_MODULE=ngx_http_upstream_keepalive_module HTTP_UPSTREAM_KEEPALIVE_SRCS=" \ src/http/modules/ngx_http_upstream_keepalive_module.c"
--- a/src/http/modules/ngx_http_upstream_ip_hash_module.c +++ b/src/http/modules/ngx_http_upstream_ip_hash_module.c @@ -16,7 +16,8 @@ typedef struct { ngx_uint_t hash; - u_char addr[3]; + u_char addrlen; + u_char *addr; u_char tries; @@ -76,7 +77,10 @@ ngx_module_t ngx_http_upstream_ip_hash_ }; -ngx_int_t +static u_char ngx_http_upstream_ip_hash_pseudo_addr[3]; + + +static ngx_int_t ngx_http_upstream_init_ip_hash(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) { if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) { @@ -93,8 +97,10 @@ static ngx_int_t ngx_http_upstream_init_ip_hash_peer(ngx_http_request_t *r, ngx_http_upstream_srv_conf_t *us) { - u_char *p; struct sockaddr_in *sin; +#if (NGX_HAVE_INET6) + struct sockaddr_in6 *sin6; +#endif ngx_http_upstream_ip_hash_peer_data_t *iphp; iphp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_ip_hash_peer_data_t)); @@ -110,20 +116,25 @@ ngx_http_upstream_init_ip_hash_peer(ngx_ r->upstream->peer.get = ngx_http_upstream_get_ip_hash_peer; - /* AF_INET only */ + switch (r->connection->sockaddr->sa_family) { - if (r->connection->sockaddr->sa_family == AF_INET) { - + case AF_INET: sin = (struct sockaddr_in *) r->connection->sockaddr; - p = (u_char *) &sin->sin_addr.s_addr; - iphp->addr[0] = p[0]; - iphp->addr[1] = p[1]; - iphp->addr[2] = p[2]; + iphp->addr = (u_char *) &sin->sin_addr.s_addr; + iphp->addrlen = 3; + break; - } else { - iphp->addr[0] = 0; - iphp->addr[1] = 0; - iphp->addr[2] = 0; +#if (NGX_HAVE_INET6) + case AF_INET6: + sin6 = (struct sockaddr_in6 *) r->connection->sockaddr; + iphp->addr = (u_char *) &sin6->sin6_addr.s6_addr; + iphp->addrlen = 16; + break; +#endif + + default: + iphp->addr = ngx_http_upstream_ip_hash_pseudo_addr; + iphp->addrlen = 3; } iphp->hash = 89; @@ -140,6 +151,7 @@ ngx_http_upstream_get_ip_hash_peer(ngx_p ngx_http_upstream_ip_hash_peer_data_t *iphp = data; time_t now; + ngx_int_t w; uintptr_t m; ngx_uint_t i, n, p, hash; ngx_http_upstream_rr_peer_t *peer; @@ -162,11 +174,25 @@ ngx_http_upstream_get_ip_hash_peer(ngx_p for ( ;; ) { - for (i = 0; i < 3; i++) { + for (i = 0; i < iphp->addrlen; i++) { hash = (hash * 113 + iphp->addr[i]) % 6271; } - p = hash % iphp->rrp.peers->number; + if (!iphp->rrp.peers->weighted) { + p = hash % iphp->rrp.peers->number; + + } else { + w = hash % iphp->rrp.peers->total_weight; + + for (i = 0; i < iphp->rrp.peers->number; i++) { + w -= iphp->rrp.peers->peer[i].weight; + if (w < 0) { + break; + } + } + + p = i; + } n = p / (8 * sizeof(uintptr_t)); m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t)); @@ -229,6 +255,7 @@ ngx_http_upstream_ip_hash(ngx_conf_t *cf uscf->peer.init_upstream = ngx_http_upstream_init_ip_hash; uscf->flags = NGX_HTTP_UPSTREAM_CREATE + |NGX_HTTP_UPSTREAM_WEIGHT |NGX_HTTP_UPSTREAM_MAX_FAILS |NGX_HTTP_UPSTREAM_FAIL_TIMEOUT |NGX_HTTP_UPSTREAM_DOWN;
--- a/src/http/modules/ngx_http_upstream_keepalive_module.c +++ b/src/http/modules/ngx_http_upstream_keepalive_module.c @@ -12,7 +12,6 @@ typedef struct { ngx_uint_t max_cached; - ngx_uint_t single; /* unsigned:1 */ ngx_queue_t cache; ngx_queue_t free; @@ -223,36 +222,11 @@ ngx_http_upstream_get_keepalive_peer(ngx kp->failed = 0; - /* single pool of cached connections */ - - if (kp->conf->single && !ngx_queue_empty(&kp->conf->cache)) { - - q = ngx_queue_head(&kp->conf->cache); - - item = ngx_queue_data(q, ngx_http_upstream_keepalive_cache_t, queue); - c = item->connection; - - ngx_queue_remove(q); - ngx_queue_insert_head(&kp->conf->free, q); - - ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, - "get keepalive peer: using connection %p", c); - - c->idle = 0; - c->log = pc->log; - c->read->log = pc->log; - c->write->log = pc->log; - c->pool->log = pc->log; - - pc->connection = c; - pc->cached = 1; - - return NGX_DONE; - } + /* ask balancer */ rc = kp->original_get_peer(pc, kp->data); - if (kp->conf->single || rc != NGX_OK) { + if (rc != NGX_OK) { return rc; } @@ -552,7 +526,8 @@ ngx_http_upstream_keepalive(ngx_conf_t * for (i = 2; i < cf->args->nelts; i++) { if (ngx_strcmp(value[i].data, "single") == 0) { - kcf->single = 1; + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "the \"single\" parameter is deprecated"); continue; }
new file mode 100644 --- /dev/null +++ b/src/http/modules/ngx_http_upstream_least_conn_module.c @@ -0,0 +1,402 @@ + +/* + * Copyright (C) Maxim Dounin + * Copyright (C) Nginx, Inc. + */ + + +#include <ngx_config.h> +#include <ngx_core.h> +#include <ngx_http.h> + + +typedef struct { + ngx_uint_t *conns; +} ngx_http_upstream_least_conn_conf_t; + + +typedef struct { + /* the round robin data must be first */ + ngx_http_upstream_rr_peer_data_t rrp; + + ngx_uint_t *conns; + + ngx_event_get_peer_pt get_rr_peer; + ngx_event_free_peer_pt free_rr_peer; +} ngx_http_upstream_lc_peer_data_t; + + +static ngx_int_t ngx_http_upstream_init_least_conn_peer(ngx_http_request_t *r, + ngx_http_upstream_srv_conf_t *us); +static ngx_int_t ngx_http_upstream_get_least_conn_peer( + ngx_peer_connection_t *pc, void *data); +static void ngx_http_upstream_free_least_conn_peer(ngx_peer_connection_t *pc, + void *data, ngx_uint_t state); +static void *ngx_http_upstream_least_conn_create_conf(ngx_conf_t *cf); +static char *ngx_http_upstream_least_conn(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); + + +static ngx_command_t ngx_http_upstream_least_conn_commands[] = { + + { ngx_string("least_conn"), + NGX_HTTP_UPS_CONF|NGX_CONF_NOARGS, + ngx_http_upstream_least_conn, + 0, + 0, + NULL }, + + ngx_null_command +}; + + +static ngx_http_module_t ngx_http_upstream_least_conn_module_ctx = { + NULL, /* preconfiguration */ + NULL, /* postconfiguration */ + + NULL, /* create main configuration */ + NULL, /* init main configuration */ + + ngx_http_upstream_least_conn_create_conf, /* create server configuration */ + NULL, /* merge server configuration */ + + NULL, /* create location configuration */ + NULL /* merge location configuration */ +}; + + +ngx_module_t ngx_http_upstream_least_conn_module = { + NGX_MODULE_V1, + &ngx_http_upstream_least_conn_module_ctx, /* module context */ + ngx_http_upstream_least_conn_commands, /* module directives */ + NGX_HTTP_MODULE, /* module type */ + NULL, /* init master */ + NULL, /* init module */ + NULL, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + NULL, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING +}; + + +ngx_int_t +ngx_http_upstream_init_least_conn(ngx_conf_t *cf, + ngx_http_upstream_srv_conf_t *us) +{ + ngx_uint_t n; + ngx_http_upstream_rr_peers_t *peers; + ngx_http_upstream_least_conn_conf_t *lcf; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, cf->log, 0, + "init least conn"); + + if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) { + return NGX_ERROR; + } + + peers = us->peer.data; + + n = peers->number; + + if (peers->next) { + n += peers->next->number; + } + + lcf = ngx_http_conf_upstream_srv_conf(us, + ngx_http_upstream_least_conn_module); + + lcf->conns = ngx_pcalloc(cf->pool, sizeof(ngx_uint_t) * n); + if (lcf->conns == NULL) { + return NGX_ERROR; + } + + us->peer.init = ngx_http_upstream_init_least_conn_peer; + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_upstream_init_least_conn_peer(ngx_http_request_t *r, + ngx_http_upstream_srv_conf_t *us) +{ + ngx_http_upstream_lc_peer_data_t *lcp; + ngx_http_upstream_least_conn_conf_t *lcf; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "init least conn peer"); + + lcf = ngx_http_conf_upstream_srv_conf(us, + ngx_http_upstream_least_conn_module); + + lcp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_lc_peer_data_t)); + if (lcp == NULL) { + return NGX_ERROR; + } + + lcp->conns = lcf->conns; + + r->upstream->peer.data = &lcp->rrp; + + if (ngx_http_upstream_init_round_robin_peer(r, us) != NGX_OK) { + return NGX_ERROR; + } + + r->upstream->peer.get = ngx_http_upstream_get_least_conn_peer; + r->upstream->peer.free = ngx_http_upstream_free_least_conn_peer; + + lcp->get_rr_peer = ngx_http_upstream_get_round_robin_peer; + lcp->free_rr_peer = ngx_http_upstream_free_round_robin_peer; + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data) +{ + ngx_http_upstream_lc_peer_data_t *lcp = data; + + time_t now; + uintptr_t m; + ngx_int_t rc, total; + ngx_uint_t i, n, p, many; + ngx_http_upstream_rr_peer_t *peer, *best; + ngx_http_upstream_rr_peers_t *peers; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "get least conn peer, try: %ui", pc->tries); + + if (lcp->rrp.peers->single) { + return lcp->get_rr_peer(pc, &lcp->rrp); + } + + pc->cached = 0; + pc->connection = NULL; + + now = ngx_time(); + + peers = lcp->rrp.peers; + + best = NULL; + total = 0; + +#if (NGX_SUPPRESS_WARN) + many = 0; + p = 0; +#endif + + for (i = 0; i < peers->number; i++) { + + n = i / (8 * sizeof(uintptr_t)); + m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t)); + + if (lcp->rrp.tried[n] & m) { + continue; + } + + peer = &peers->peer[i]; + + if (peer->down) { + continue; + } + + if (peer->max_fails + && peer->fails >= peer->max_fails + && now - peer->checked <= peer->fail_timeout) + { + continue; + } + + /* + * select peer with least number of connections; if there are + * multiple peers with the same number of connections, select + * based on round-robin + */ + + if (best == NULL + || lcp->conns[i] * best->weight < lcp->conns[p] * peer->weight) + { + best = peer; + many = 0; + p = i; + + } else if (lcp->conns[i] * best->weight + == lcp->conns[p] * peer->weight) + { + many = 1; + } + } + + if (best == NULL) { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "get least conn peer, no peer found"); + + goto failed; + } + + if (many) { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "get least conn peer, many"); + + for (i = p; i < peers->number; i++) { + + n = i / (8 * sizeof(uintptr_t)); + m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t)); + + if (lcp->rrp.tried[n] & m) { + continue; + } + + peer = &peers->peer[i]; + + if (peer->down) { + continue; + } + + if (lcp->conns[i] * best->weight != lcp->conns[p] * peer->weight) { + continue; + } + + if (peer->max_fails + && peer->fails >= peer->max_fails + && now - peer->checked <= peer->fail_timeout) + { + continue; + } + + peer->current_weight += peer->effective_weight; + total += peer->effective_weight; + + if (peer->effective_weight < peer->weight) { + peer->effective_weight++; + } + + if (peer->current_weight > best->current_weight) { + best = peer; + p = i; + } + } + } + + best->current_weight -= total; + best->checked = now; + + pc->sockaddr = best->sockaddr; + pc->socklen = best->socklen; + pc->name = &best->name; + + lcp->rrp.current = p; + + n = p / (8 * sizeof(uintptr_t)); + m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t)); + + lcp->rrp.tried[n] |= m; + lcp->conns[p]++; + + if (pc->tries == 1 && peers->next) { + pc->tries += peers->next->number; + } + + return NGX_OK; + +failed: + + if (peers->next) { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "get least conn peer, backup servers"); + + lcp->conns += peers->number; + + lcp->rrp.peers = peers->next; + pc->tries = lcp->rrp.peers->number; + + n = lcp->rrp.peers->number / (8 * sizeof(uintptr_t)) + 1; + for (i = 0; i < n; i++) { + lcp->rrp.tried[i] = 0; + } + + rc = ngx_http_upstream_get_least_conn_peer(pc, lcp); + + if (rc != NGX_BUSY) { + return rc; + } + } + + /* all peers failed, mark them as live for quick recovery */ + + for (i = 0; i < peers->number; i++) { + peers->peer[i].fails = 0; + } + + pc->name = peers->name; + + return NGX_BUSY; +} + + +static void +ngx_http_upstream_free_least_conn_peer(ngx_peer_connection_t *pc, + void *data, ngx_uint_t state) +{ + ngx_http_upstream_lc_peer_data_t *lcp = data; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "free least conn peer %ui %ui", pc->tries, state); + + if (lcp->rrp.peers->single) { + lcp->free_rr_peer(pc, &lcp->rrp, state); + return; + } + + if (state == 0 && pc->tries == 0) { + return; + } + + lcp->conns[lcp->rrp.current]--; + + lcp->free_rr_peer(pc, &lcp->rrp, state); +} + + +static void * +ngx_http_upstream_least_conn_create_conf(ngx_conf_t *cf) +{ + ngx_http_upstream_least_conn_conf_t *conf; + + conf = ngx_pcalloc(cf->pool, + sizeof(ngx_http_upstream_least_conn_conf_t)); + if (conf == NULL) { + return NULL; + } + + /* + * set by ngx_pcalloc(): + * + * conf->conns = NULL; + */ + + return conf; +} + + +static char * +ngx_http_upstream_least_conn(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_upstream_srv_conf_t *uscf; + + uscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_upstream_module); + + uscf->peer.init_upstream = ngx_http_upstream_init_least_conn; + + uscf->flags = NGX_HTTP_UPSTREAM_CREATE + |NGX_HTTP_UPSTREAM_WEIGHT + |NGX_HTTP_UPSTREAM_MAX_FAILS + |NGX_HTTP_UPSTREAM_FAIL_TIMEOUT + |NGX_HTTP_UPSTREAM_DOWN + |NGX_HTTP_UPSTREAM_BACKUP; + + return NGX_CONF_OK; +}
--- a/src/http/ngx_http_upstream_round_robin.c +++ b/src/http/ngx_http_upstream_round_robin.c @@ -30,7 +30,7 @@ ngx_http_upstream_init_round_robin(ngx_c ngx_http_upstream_srv_conf_t *us) { ngx_url_t u; - ngx_uint_t i, j, n; + ngx_uint_t i, j, n, w; ngx_http_upstream_server_t *server; ngx_http_upstream_rr_peers_t *peers, *backup; @@ -40,6 +40,7 @@ ngx_http_upstream_init_round_robin(ngx_c server = us->servers->elts; n = 0; + w = 0; for (i = 0; i < us->servers->nelts; i++) { if (server[i].backup) { @@ -47,6 +48,7 @@ ngx_http_upstream_init_round_robin(ngx_c } n += server[i].naddrs; + w += server[i].naddrs * server[i].weight; } if (n == 0) { @@ -64,6 +66,8 @@ ngx_http_upstream_init_round_robin(ngx_c peers->single = (n == 1); peers->number = n; + peers->weighted = (w != n); + peers->total_weight = w; peers->name = &us->host; n = 0; @@ -96,6 +100,7 @@ ngx_http_upstream_init_round_robin(ngx_c /* backup servers */ n = 0; + w = 0; for (i = 0; i < us->servers->nelts; i++) { if (!server[i].backup) { @@ -103,6 +108,7 @@ ngx_http_upstream_init_round_robin(ngx_c } n += server[i].naddrs; + w += server[i].naddrs * server[i].weight; } if (n == 0) { @@ -118,6 +124,8 @@ ngx_http_upstream_init_round_robin(ngx_c peers->single = 0; backup->single = 0; backup->number = n; + backup->weighted = (w != n); + backup->total_weight = w; backup->name = &us->host; n = 0; @@ -185,6 +193,8 @@ ngx_http_upstream_init_round_robin(ngx_c peers->single = (n == 1); peers->number = n; + peers->weighted = 0; + peers->total_weight = n; peers->name = &us->host; for (i = 0; i < u.naddrs; i++) {
--- a/src/http/ngx_http_upstream_round_robin.h +++ b/src/http/ngx_http_upstream_round_robin.h @@ -41,13 +41,17 @@ typedef struct { typedef struct ngx_http_upstream_rr_peers_s ngx_http_upstream_rr_peers_t; struct ngx_http_upstream_rr_peers_s { - ngx_uint_t single; /* unsigned single:1; */ ngx_uint_t number; ngx_uint_t last_cached; /* ngx_mutex_t *mutex; */ ngx_connection_t **cached; + ngx_uint_t total_weight; + + unsigned single:1; + unsigned weighted:1; + ngx_str_t *name; ngx_http_upstream_rr_peers_t *next;