Mercurial > hg > nginx
changeset 6103:79ddb0bdb273
Upstream: the "zone" directive.
Upstreams with the "zone" directive are kept in shared memory,
with a consistent view of all worker processes.
| author | Ruslan Ermilov <ru@nginx.com> |
|---|---|
| date | Tue, 14 Apr 2015 19:01:25 +0300 |
| parents | 3264b7828f72 |
| children | c1cae8b2270c |
| files | auto/modules auto/options auto/sources src/core/ngx_cycle.c src/core/ngx_cycle.h src/core/ngx_rwlock.c src/http/modules/ngx_http_upstream_zone_module.c src/http/ngx_http_upstream.h src/http/ngx_http_upstream_round_robin.c src/http/ngx_http_upstream_round_robin.h |
| diffstat | 10 files changed, 407 insertions(+), 7 deletions(-) [+] |
line wrap: on
line diff
--- a/auto/modules +++ b/auto/modules @@ -391,6 +391,12 @@ if [ $HTTP_UPSTREAM_KEEPALIVE = YES ]; t HTTP_SRCS="$HTTP_SRCS $HTTP_UPSTREAM_KEEPALIVE_SRCS" fi +if [ $HTTP_UPSTREAM_ZONE = YES ]; then + have=NGX_HTTP_UPSTREAM_ZONE . auto/have + HTTP_MODULES="$HTTP_MODULES $HTTP_UPSTREAM_ZONE_MODULE" + HTTP_SRCS="$HTTP_SRCS $HTTP_UPSTREAM_ZONE_SRCS" +fi + if [ $HTTP_STUB_STATUS = YES ]; then have=NGX_STAT_STUB . auto/have HTTP_MODULES="$HTTP_MODULES ngx_http_stub_status_module"
--- a/auto/options +++ b/auto/options @@ -103,6 +103,7 @@ HTTP_UPSTREAM_HASH=YES HTTP_UPSTREAM_IP_HASH=YES HTTP_UPSTREAM_LEAST_CONN=YES HTTP_UPSTREAM_KEEPALIVE=YES +HTTP_UPSTREAM_ZONE=YES # STUB HTTP_STUB_STATUS=NO @@ -256,6 +257,7 @@ use the \"--without-http_limit_conn_modu --without-http_upstream_least_conn_module) HTTP_UPSTREAM_LEAST_CONN=NO ;; --without-http_upstream_keepalive_module) HTTP_UPSTREAM_KEEPALIVE=NO ;; + --without-http_upstream_zone_module) HTTP_UPSTREAM_ZONE=NO ;; --with-http_perl_module) HTTP_PERL=YES ;; --with-perl_modules_path=*) NGX_PERL_MODULES="$value" ;; @@ -406,6 +408,8 @@ cat << END disable ngx_http_upstream_least_conn_module --without-http_upstream_keepalive_module disable ngx_http_upstream_keepalive_module + --without-http_upstream_zone_module + disable ngx_http_upstream_zone_module --with-http_perl_module enable ngx_http_perl_module --with-perl_modules_path=PATH set Perl modules path
--- a/auto/sources +++ b/auto/sources @@ -513,6 +513,11 @@ HTTP_UPSTREAM_KEEPALIVE_SRCS=" \ src/http/modules/ngx_http_upstream_keepalive_module.c" +HTTP_UPSTREAM_ZONE_MODULE=ngx_http_upstream_zone_module +HTTP_UPSTREAM_ZONE_SRCS=" \ + src/http/modules/ngx_http_upstream_zone_module.c" + + MAIL_INCS="src/mail" MAIL_DEPS="src/mail/ngx_mail.h"
--- a/src/core/ngx_cycle.c +++ b/src/core/ngx_cycle.c @@ -441,7 +441,8 @@ ngx_init_cycle(ngx_cycle_t *old_cycle) } if (shm_zone[i].tag == oshm_zone[n].tag - && shm_zone[i].shm.size == oshm_zone[n].shm.size) + && shm_zone[i].shm.size == oshm_zone[n].shm.size + && !shm_zone[i].noreuse) { shm_zone[i].shm.addr = oshm_zone[n].shm.addr; @@ -1234,6 +1235,7 @@ ngx_shared_memory_add(ngx_conf_t *cf, ng shm_zone->shm.exists = 0; shm_zone->init = NULL; shm_zone->tag = tag; + shm_zone->noreuse = 0; return shm_zone; }
--- a/src/core/ngx_cycle.h +++ b/src/core/ngx_cycle.h @@ -31,6 +31,7 @@ struct ngx_shm_zone_s { ngx_shm_t shm; ngx_shm_zone_init_pt init; void *tag; + ngx_uint_t noreuse; /* unsigned noreuse:1; */ };
--- a/src/core/ngx_rwlock.c +++ b/src/core/ngx_rwlock.c @@ -109,4 +109,12 @@ ngx_rwlock_unlock(ngx_atomic_t *lock) } +#else + +#if (NGX_HTTP_UPSTREAM_ZONE) + +#error ngx_atomic_cmp_set() is not defined! + #endif + +#endif
new file mode 100644 --- /dev/null +++ b/src/http/modules/ngx_http_upstream_zone_module.c @@ -0,0 +1,213 @@ + +/* + * Copyright (C) Ruslan Ermilov + * Copyright (C) Nginx, Inc. + */ + + +#include <ngx_config.h> +#include <ngx_core.h> +#include <ngx_http.h> + + +static char *ngx_http_upstream_zone(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); +static ngx_int_t ngx_http_upstream_init_zone(ngx_shm_zone_t *shm_zone, + void *data); + + +static ngx_command_t ngx_http_upstream_zone_commands[] = { + + { ngx_string("zone"), + NGX_HTTP_UPS_CONF|NGX_CONF_TAKE2, + ngx_http_upstream_zone, + 0, + 0, + NULL }, + + ngx_null_command +}; + + +static ngx_http_module_t ngx_http_upstream_zone_module_ctx = { + NULL, /* preconfiguration */ + NULL, /* postconfiguration */ + + NULL, /* create main configuration */ + NULL, /* init main configuration */ + + NULL, /* create server configuration */ + NULL, /* merge server configuration */ + + NULL, /* create location configuration */ + NULL /* merge location configuration */ +}; + + +ngx_module_t ngx_http_upstream_zone_module = { + NGX_MODULE_V1, + &ngx_http_upstream_zone_module_ctx, /* module context */ + ngx_http_upstream_zone_commands, /* module directives */ + NGX_HTTP_MODULE, /* module type */ + NULL, /* init master */ + NULL, /* init module */ + NULL, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + NULL, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING +}; + + +static char * +ngx_http_upstream_zone(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_upstream_srv_conf_t *uscf; + ssize_t size; + ngx_str_t *value; + + uscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_upstream_module); + + value = cf->args->elts; + + if (!value[1].len) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid zone name \"%V\"", &value[1]); + return NGX_CONF_ERROR; + } + + size = ngx_parse_size(&value[2]); + + if (size == NGX_ERROR) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid zone size \"%V\"", &value[2]); + return NGX_CONF_ERROR; + } + + if (size < (ssize_t) (8 * ngx_pagesize)) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "zone \"%V\" is too small", &value[1]); + return NGX_CONF_ERROR; + } + + uscf->shm_zone = ngx_shared_memory_add(cf, &value[1], size, + &ngx_http_upstream_module); + if (uscf->shm_zone == NULL) { + return NGX_CONF_ERROR; + } + + if (uscf->shm_zone->data) { + uscf = uscf->shm_zone->data; + + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "upstream \"%V\" in %s:%ui " + "is already bound to zone \"%V\"", + &uscf->host, uscf->file_name, uscf->line, + &value[1]); + return NGX_CONF_ERROR; + } + + uscf->shm_zone->init = ngx_http_upstream_init_zone; + uscf->shm_zone->data = uscf; + + uscf->shm_zone->noreuse = 1; + + return NGX_CONF_OK; +} + + +static ngx_int_t +ngx_http_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data) +{ + ngx_http_upstream_srv_conf_t *ouscf = data; + + size_t len; + ngx_slab_pool_t *shpool; + ngx_http_upstream_rr_peer_t *peer, **peerp; + ngx_http_upstream_rr_peers_t *peers, *backup; + ngx_http_upstream_srv_conf_t *uscf; + + uscf = shm_zone->data; + + if (ouscf) { + ngx_log_error(NGX_LOG_EMERG, shm_zone->shm.log, 0, + "zone \"%V\" cannot be reused", &shm_zone->shm.name); + return NGX_ERROR; + } + + shpool = (ngx_slab_pool_t *) shm_zone->shm.addr; + + if (shm_zone->shm.exists) { + return NGX_ERROR; + } + + + /* copy peers to shared memory */ + + len = sizeof(" in upstream zone \"\"") + shm_zone->shm.name.len; + + shpool->log_ctx = ngx_slab_alloc(shpool, len); + if (shpool->log_ctx == NULL) { + return NGX_ERROR; + } + + ngx_sprintf(shpool->log_ctx, " in upstream zone \"%V\"%Z", + &shm_zone->shm.name); + + peers = ngx_slab_alloc(shpool, sizeof(ngx_http_upstream_rr_peers_t)); + if (peers == NULL) { + return NGX_ERROR; + } + + ngx_memcpy(peers, uscf->peer.data, sizeof(ngx_http_upstream_rr_peers_t)); + + peers->shpool = shpool; + + for (peerp = &peers->peer; *peerp; peerp = &peer->next) { + /* pool is unlocked */ + peer = ngx_slab_calloc_locked(shpool, + sizeof(ngx_http_upstream_rr_peer_t)); + if (peer == NULL) { + return NGX_ERROR; + } + + ngx_memcpy(peer, *peerp, sizeof(ngx_http_upstream_rr_peer_t)); + + *peerp = peer; + } + + if (peers->next == NULL) { + goto done; + } + + backup = ngx_slab_alloc(shpool, sizeof(ngx_http_upstream_rr_peers_t)); + if (backup == NULL) { + return NGX_ERROR; + } + + ngx_memcpy(backup, peers->next, sizeof(ngx_http_upstream_rr_peers_t)); + + backup->shpool = shpool; + + for (peerp = &backup->peer; *peerp; peerp = &peer->next) { + /* pool is unlocked */ + peer = ngx_slab_calloc_locked(shpool, + sizeof(ngx_http_upstream_rr_peer_t)); + if (peer == NULL) { + return NGX_ERROR; + } + + ngx_memcpy(peer, *peerp, sizeof(ngx_http_upstream_rr_peer_t)); + + *peerp = peer; + } + + peers->next = backup; + +done: + + uscf->peer.data = peers; + + return NGX_OK; +}
--- a/src/http/ngx_http_upstream.h +++ b/src/http/ngx_http_upstream.h @@ -123,6 +123,10 @@ struct ngx_http_upstream_srv_conf_s { in_port_t port; in_port_t default_port; ngx_uint_t no_port; /* unsigned no_port:1 */ + +#if (NGX_HTTP_UPSTREAM_ZONE) + ngx_shm_zone_t *shm_zone; +#endif };
--- a/src/http/ngx_http_upstream_round_robin.c +++ b/src/http/ngx_http_upstream_round_robin.c @@ -657,12 +657,55 @@ ngx_http_upstream_set_round_robin_peer_s { ngx_http_upstream_rr_peer_data_t *rrp = data; - ngx_int_t rc; - ngx_ssl_session_t *ssl_session; - ngx_http_upstream_rr_peer_t *peer; + ngx_int_t rc; + ngx_ssl_session_t *ssl_session; + ngx_http_upstream_rr_peer_t *peer; +#if (NGX_HTTP_UPSTREAM_ZONE) + int len; +#if OPENSSL_VERSION_NUMBER >= 0x0090707fL + const +#endif + u_char *p; + ngx_http_upstream_rr_peers_t *peers; + u_char buf[NGX_SSL_MAX_SESSION_SIZE]; +#endif peer = rrp->current; +#if (NGX_HTTP_UPSTREAM_ZONE) + peers = rrp->peers; + + if (peers->shpool) { + ngx_http_upstream_rr_peers_rlock(peers); + ngx_http_upstream_rr_peer_lock(peers, peer); + + if (peer->ssl_session == NULL) { + ngx_http_upstream_rr_peer_unlock(peers, peer); + ngx_http_upstream_rr_peers_unlock(peers); + return NGX_OK; + } + + len = peer->ssl_session_len; + + ngx_memcpy(buf, peer->ssl_session, len); + + ngx_http_upstream_rr_peer_unlock(peers, peer); + ngx_http_upstream_rr_peers_unlock(peers); + + p = buf; + ssl_session = d2i_SSL_SESSION(NULL, &p, len); + + rc = ngx_ssl_set_session(pc->connection, ssl_session); + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "set session: %p", ssl_session); + + ngx_ssl_free_session(ssl_session); + + return rc; + } +#endif + ssl_session = peer->ssl_session; rc = ngx_ssl_set_session(pc->connection, ssl_session); @@ -680,8 +723,75 @@ ngx_http_upstream_save_round_robin_peer_ { ngx_http_upstream_rr_peer_data_t *rrp = data; - ngx_ssl_session_t *old_ssl_session, *ssl_session; - ngx_http_upstream_rr_peer_t *peer; + ngx_ssl_session_t *old_ssl_session, *ssl_session; + ngx_http_upstream_rr_peer_t *peer; +#if (NGX_HTTP_UPSTREAM_ZONE) + int len; + u_char *p; + ngx_http_upstream_rr_peers_t *peers; + u_char buf[NGX_SSL_MAX_SESSION_SIZE]; +#endif + +#if (NGX_HTTP_UPSTREAM_ZONE) + peers = rrp->peers; + + if (peers->shpool) { + + ssl_session = SSL_get0_session(pc->connection->ssl->connection); + + if (ssl_session == NULL) { + return; + } + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "save session: %p", ssl_session); + + len = i2d_SSL_SESSION(ssl_session, NULL); + + /* do not cache too big session */ + + if (len > NGX_SSL_MAX_SESSION_SIZE) { + return; + } + + p = buf; + (void) i2d_SSL_SESSION(ssl_session, &p); + + peer = rrp->current; + + ngx_http_upstream_rr_peers_rlock(peers); + ngx_http_upstream_rr_peer_lock(peers, peer); + + if (len > peer->ssl_session_len) { + ngx_shmtx_lock(&peers->shpool->mutex); + + if (peer->ssl_session) { + ngx_slab_free_locked(peers->shpool, peer->ssl_session); + } + + peer->ssl_session = ngx_slab_alloc_locked(peers->shpool, len); + + ngx_shmtx_unlock(&peers->shpool->mutex); + + if (peer->ssl_session == NULL) { + peer->ssl_session_len = 0; + + ngx_http_upstream_rr_peer_unlock(peers, peer); + ngx_http_upstream_rr_peers_unlock(peers); + return; + } + + peer->ssl_session_len = len; + } + + ngx_memcpy(peer->ssl_session, buf, len); + + ngx_http_upstream_rr_peer_unlock(peers, peer); + ngx_http_upstream_rr_peers_unlock(peers); + + return; + } +#endif ssl_session = ngx_ssl_get_session(pc->connection);
--- a/src/http/ngx_http_upstream_round_robin.h +++ b/src/http/ngx_http_upstream_round_robin.h @@ -38,10 +38,15 @@ struct ngx_http_upstream_rr_peer_s { ngx_uint_t down; /* unsigned down:1; */ #if (NGX_HTTP_SSL) - ngx_ssl_session_t *ssl_session; /* local to a process */ + void *ssl_session; + int ssl_session_len; #endif ngx_http_upstream_rr_peer_t *next; + +#if (NGX_HTTP_UPSTREAM_ZONE) + ngx_atomic_t lock; +#endif }; @@ -50,6 +55,11 @@ typedef struct ngx_http_upstream_rr_peer struct ngx_http_upstream_rr_peers_s { ngx_uint_t number; +#if (NGX_HTTP_UPSTREAM_ZONE) + ngx_slab_pool_t *shpool; + ngx_atomic_t rwlock; +#endif + ngx_uint_t total_weight; unsigned single:1; @@ -63,12 +73,49 @@ struct ngx_http_upstream_rr_peers_s { }; +#if (NGX_HTTP_UPSTREAM_ZONE) + +#define ngx_http_upstream_rr_peers_rlock(peers) \ + \ + if (peers->shpool) { \ + ngx_rwlock_rlock(&peers->rwlock); \ + } + +#define ngx_http_upstream_rr_peers_wlock(peers) \ + \ + if (peers->shpool) { \ + ngx_rwlock_wlock(&peers->rwlock); \ + } + +#define ngx_http_upstream_rr_peers_unlock(peers) \ + \ + if (peers->shpool) { \ + ngx_rwlock_unlock(&peers->rwlock); \ + } + + +#define ngx_http_upstream_rr_peer_lock(peers, peer) \ + \ + if (peers->shpool) { \ + ngx_rwlock_wlock(&peer->lock); \ + } + +#define ngx_http_upstream_rr_peer_unlock(peers, peer) \ + \ + if (peers->shpool) { \ + ngx_rwlock_unlock(&peer->lock); \ + } + +#else + #define ngx_http_upstream_rr_peers_rlock(peers) #define ngx_http_upstream_rr_peers_wlock(peers) #define ngx_http_upstream_rr_peers_unlock(peers) #define ngx_http_upstream_rr_peer_lock(peers, peer) #define ngx_http_upstream_rr_peer_unlock(peers, peer) +#endif + typedef struct { ngx_http_upstream_rr_peers_t *peers;