# HG changeset patch # User Sergey Kandaurov # Date 1531821203 -10800 # Node ID 696df3ac27ac0b4affbf87692dbc2348ac998d5b # Parent dcab8611526120b270841a10a307f66f0be44e0a SSL: save sessions for upstream peers using a callback function. In TLSv1.3, NewSessionTicket messages arrive after the handshake and can come at any time. Therefore we use a callback to save the session when we know about it. This approach works for < TLSv1.3 as well. The callback function is set once per location on merge phase. Since SSL_get_session() in BoringSSL returns an unresumable session for TLSv1.3, peer save_session() methods have been updated as well to use a session supplied within the callback. To preserve API, the session is cached in c->ssl->session. It is preferably accessed in save_session() methods by ngx_ssl_get_session() and ngx_ssl_get0_session() wrappers. diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -24,6 +24,8 @@ static int ngx_ssl_verify_callback(int o static void ngx_ssl_info_callback(const ngx_ssl_conn_t *ssl_conn, int where, int ret); static void ngx_ssl_passwords_cleanup(void *data); +static int ngx_ssl_new_client_session(ngx_ssl_conn_t *ssl_conn, + ngx_ssl_session_t *sess); static void ngx_ssl_handshake_handler(ngx_event_t *ev); static ngx_int_t ngx_ssl_handle_recv(ngx_connection_t *c, int n); static void ngx_ssl_write_handler(ngx_event_t *wev); @@ -1162,6 +1164,42 @@ ngx_ssl_ecdh_curve(ngx_conf_t *cf, ngx_s ngx_int_t +ngx_ssl_client_session_cache(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_uint_t enable) +{ + if (!enable) { + return NGX_OK; + } + + SSL_CTX_set_session_cache_mode(ssl->ctx, + SSL_SESS_CACHE_CLIENT + |SSL_SESS_CACHE_NO_INTERNAL); + + SSL_CTX_sess_set_new_cb(ssl->ctx, ngx_ssl_new_client_session); + + return NGX_OK; +} + + +static int +ngx_ssl_new_client_session(ngx_ssl_conn_t *ssl_conn, ngx_ssl_session_t *sess) +{ + ngx_connection_t *c; + + c = ngx_ssl_get_connection(ssl_conn); + + if (c->ssl->save_session) { + c->ssl->session = sess; + + c->ssl->save_session(c); + + c->ssl->session = NULL; + } + + return 0; +} + + +ngx_int_t ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags) { ngx_ssl_connection_t *sc; @@ -1210,6 +1248,31 @@ ngx_ssl_create_connection(ngx_ssl_t *ssl } +ngx_ssl_session_t * +ngx_ssl_get_session(ngx_connection_t *c) +{ +#ifdef TLS1_3_VERSION + if (c->ssl->session) { + SSL_SESSION_up_ref(c->ssl->session); + return c->ssl->session; + } +#endif + + return SSL_get1_session(c->ssl->connection); +} + + +ngx_ssl_session_t * +ngx_ssl_get0_session(ngx_connection_t *c) +{ + if (c->ssl->session) { + return c->ssl->session; + } + + return SSL_get0_session(c->ssl->connection); +} + + ngx_int_t ngx_ssl_set_session(ngx_connection_t *c, ngx_ssl_session_t *session) { diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -77,6 +77,9 @@ struct ngx_ssl_connection_s { ngx_connection_handler_pt handler; + ngx_ssl_session_t *session; + ngx_connection_handler_pt save_session; + ngx_event_handler_pt saved_read_handler; ngx_event_handler_pt saved_write_handler; @@ -168,6 +171,8 @@ RSA *ngx_ssl_rsa512_key_callback(ngx_ssl ngx_array_t *ngx_ssl_read_password_file(ngx_conf_t *cf, ngx_str_t *file); ngx_int_t ngx_ssl_dhparam(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file); ngx_int_t ngx_ssl_ecdh_curve(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *name); +ngx_int_t ngx_ssl_client_session_cache(ngx_conf_t *cf, ngx_ssl_t *ssl, + ngx_uint_t enable); ngx_int_t ngx_ssl_session_cache(ngx_ssl_t *ssl, ngx_str_t *sess_ctx, ssize_t builtin_session_cache, ngx_shm_zone_t *shm_zone, time_t timeout); ngx_int_t ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl, @@ -178,7 +183,8 @@ ngx_int_t ngx_ssl_create_connection(ngx_ void ngx_ssl_remove_cached_session(SSL_CTX *ssl, ngx_ssl_session_t *sess); ngx_int_t ngx_ssl_set_session(ngx_connection_t *c, ngx_ssl_session_t *session); -#define ngx_ssl_get_session(c) SSL_get1_session(c->ssl->connection) +ngx_ssl_session_t *ngx_ssl_get_session(ngx_connection_t *c); +ngx_ssl_session_t *ngx_ssl_get0_session(ngx_connection_t *c); #define ngx_ssl_free_session SSL_SESSION_free #define ngx_ssl_get_connection(ssl_conn) \ SSL_get_ex_data(ssl_conn, ngx_ssl_connection_index) diff --git a/src/http/modules/ngx_http_grpc_module.c b/src/http/modules/ngx_http_grpc_module.c --- a/src/http/modules/ngx_http_grpc_module.c +++ b/src/http/modules/ngx_http_grpc_module.c @@ -4627,6 +4627,13 @@ ngx_http_grpc_set_ssl(ngx_conf_t *cf, ng } } + if (ngx_ssl_client_session_cache(cf, glcf->upstream.ssl, + glcf->upstream.ssl_session_reuse) + != NGX_OK) + { + return NGX_ERROR; + } + #ifdef TLSEXT_TYPE_application_layer_protocol_negotiation if (SSL_CTX_set_alpn_protos(glcf->upstream.ssl->ctx, diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c --- a/src/http/modules/ngx_http_proxy_module.c +++ b/src/http/modules/ngx_http_proxy_module.c @@ -4308,6 +4308,13 @@ ngx_http_proxy_set_ssl(ngx_conf_t *cf, n } } + if (ngx_ssl_client_session_cache(cf, plcf->upstream.ssl, + plcf->upstream.ssl_session_reuse) + != NGX_OK) + { + return NGX_ERROR; + } + return NGX_OK; } diff --git a/src/http/modules/ngx_http_uwsgi_module.c b/src/http/modules/ngx_http_uwsgi_module.c --- a/src/http/modules/ngx_http_uwsgi_module.c +++ b/src/http/modules/ngx_http_uwsgi_module.c @@ -2391,6 +2391,13 @@ ngx_http_uwsgi_set_ssl(ngx_conf_t *cf, n } } + if (ngx_ssl_client_session_cache(cf, uwcf->upstream.ssl, + uwcf->upstream.ssl_session_reuse) + != NGX_OK) + { + return NGX_ERROR; + } + return NGX_OK; } diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -187,6 +187,7 @@ static void ngx_http_upstream_ssl_init_c static void ngx_http_upstream_ssl_handshake_handler(ngx_connection_t *c); static void ngx_http_upstream_ssl_handshake(ngx_http_request_t *, ngx_http_upstream_t *u, ngx_connection_t *c); +static void ngx_http_upstream_ssl_save_session(ngx_connection_t *c); static ngx_int_t ngx_http_upstream_ssl_name(ngx_http_request_t *r, ngx_http_upstream_t *u, ngx_connection_t *c); #endif @@ -1675,6 +1676,8 @@ ngx_http_upstream_ssl_init_connection(ng } if (u->conf->ssl_session_reuse) { + c->ssl->save_session = ngx_http_upstream_ssl_save_session; + if (u->peer.set_session(&u->peer, u->peer.data) != NGX_OK) { ngx_http_upstream_finalize_request(r, u, NGX_HTTP_INTERNAL_SERVER_ERROR); @@ -1759,10 +1762,6 @@ ngx_http_upstream_ssl_handshake(ngx_http } } - if (u->conf->ssl_session_reuse) { - u->peer.save_session(&u->peer, u->peer.data); - } - c->write->handler = ngx_http_upstream_handler; c->read->handler = ngx_http_upstream_handler; @@ -1782,6 +1781,27 @@ failed: } +static void +ngx_http_upstream_ssl_save_session(ngx_connection_t *c) +{ + ngx_http_request_t *r; + ngx_http_upstream_t *u; + + if (c->idle) { + return; + } + + r = c->data; + + u = r->upstream; + c = r->connection; + + ngx_http_set_log_request(c->log, r); + + u->peer.save_session(&u->peer, u->peer.data); +} + + static ngx_int_t ngx_http_upstream_ssl_name(ngx_http_request_t *r, ngx_http_upstream_t *u, ngx_connection_t *c) diff --git a/src/http/ngx_http_upstream_round_robin.c b/src/http/ngx_http_upstream_round_robin.c --- a/src/http/ngx_http_upstream_round_robin.c +++ b/src/http/ngx_http_upstream_round_robin.c @@ -744,7 +744,7 @@ ngx_http_upstream_save_round_robin_peer_ if (peers->shpool) { - ssl_session = SSL_get0_session(pc->connection->ssl->connection); + ssl_session = ngx_ssl_get0_session(pc->connection); if (ssl_session == NULL) { return; diff --git a/src/stream/ngx_stream_proxy_module.c b/src/stream/ngx_stream_proxy_module.c --- a/src/stream/ngx_stream_proxy_module.c +++ b/src/stream/ngx_stream_proxy_module.c @@ -92,6 +92,7 @@ static char *ngx_stream_proxy_ssl_passwo ngx_command_t *cmd, void *conf); static void ngx_stream_proxy_ssl_init_connection(ngx_stream_session_t *s); static void ngx_stream_proxy_ssl_handshake(ngx_connection_t *pc); +static void ngx_stream_proxy_ssl_save_session(ngx_connection_t *c); static ngx_int_t ngx_stream_proxy_ssl_name(ngx_stream_session_t *s); static ngx_int_t ngx_stream_proxy_set_ssl(ngx_conf_t *cf, ngx_stream_proxy_srv_conf_t *pscf); @@ -1008,6 +1009,8 @@ ngx_stream_proxy_ssl_init_connection(ngx } if (pscf->ssl_session_reuse) { + pc->ssl->save_session = ngx_stream_proxy_ssl_save_session; + if (u->peer.set_session(&u->peer, u->peer.data) != NGX_OK) { ngx_stream_proxy_finalize(s, NGX_STREAM_INTERNAL_SERVER_ERROR); return; @@ -1066,11 +1069,6 @@ ngx_stream_proxy_ssl_handshake(ngx_conne } } - if (pscf->ssl_session_reuse) { - u = s->upstream; - u->peer.save_session(&u->peer, u->peer.data); - } - if (pc->write->timer_set) { ngx_del_timer(pc->write); } @@ -1086,6 +1084,19 @@ failed: } +static void +ngx_stream_proxy_ssl_save_session(ngx_connection_t *c) +{ + ngx_stream_session_t *s; + ngx_stream_upstream_t *u; + + s = c->data; + u = s->upstream; + + u->peer.save_session(&u->peer, u->peer.data); +} + + static ngx_int_t ngx_stream_proxy_ssl_name(ngx_stream_session_t *s) { @@ -2051,6 +2062,12 @@ ngx_stream_proxy_set_ssl(ngx_conf_t *cf, } } + if (ngx_ssl_client_session_cache(cf, pscf->ssl, pscf->ssl_session_reuse) + != NGX_OK) + { + return NGX_ERROR; + } + return NGX_OK; } diff --git a/src/stream/ngx_stream_upstream_round_robin.c b/src/stream/ngx_stream_upstream_round_robin.c --- a/src/stream/ngx_stream_upstream_round_robin.c +++ b/src/stream/ngx_stream_upstream_round_robin.c @@ -776,7 +776,7 @@ ngx_stream_upstream_save_round_robin_pee if (peers->shpool) { - ssl_session = SSL_get0_session(pc->connection->ssl->connection); + ssl_session = ngx_ssl_get0_session(pc->connection); if (ssl_session == NULL) { return;