# HG changeset patch # User Roman Arutyunyan # Date 1585062345 -10800 # Node ID 618a65de08b30748f2567ff9b824661dbc5894ee # Parent b364af7f9f3f6fac89dc922e4280a90836545197 When closing a QUIC connection, wait for all streams to finish. Additionally, streams are now removed from the tree in cleanup handler. diff --git a/src/event/ngx_event_quic.c b/src/event/ngx_event_quic.c --- a/src/event/ngx_event_quic.c +++ b/src/event/ngx_event_quic.c @@ -50,8 +50,9 @@ struct ngx_quic_connection_s { ngx_quic_streams_t streams; ngx_uint_t max_data; - ngx_uint_t send_timer_set; - /* unsigned send_timer_set:1 */ + + unsigned send_timer_set:1; + unsigned closing:1; #define SSL_ECRYPTION_LAST ((ssl_encryption_application) + 1) uint64_t crypto_offset[SSL_ECRYPTION_LAST]; @@ -308,6 +309,10 @@ ngx_quic_send_alert(ngx_ssl_conn_t *ssl_ c = ngx_ssl_get_connection((ngx_ssl_conn_t *) ssl_conn); + if (c->quic->closing) { + return 1; + } + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, "ngx_quic_send_alert(), lvl=%d, alert=%d", (int) level, (int) alert); @@ -536,9 +541,15 @@ ngx_quic_input_handler(ngx_event_t *rev) b.pos = b.last = b.start; c = rev->data; + qc = c->quic; ngx_log_debug0(NGX_LOG_DEBUG_EVENT, rev->log, 0, "quic input handler"); + if (qc->closing) { + ngx_quic_close_connection(c); + return; + } + if (rev->timedout) { ngx_log_error(NGX_LOG_INFO, c->log, NGX_ETIMEDOUT, "client timed out"); ngx_quic_close_connection(c); @@ -569,8 +580,6 @@ ngx_quic_input_handler(ngx_event_t *rev) return; } - qc = c->quic; - qc->send_timer_set = 0; ngx_add_timer(rev, qc->tp.max_idle_timeout); } @@ -579,12 +588,56 @@ ngx_quic_input_handler(ngx_event_t *rev) static void ngx_quic_close_connection(ngx_connection_t *c) { - ngx_pool_t *pool; +#if (NGX_DEBUG) + ngx_uint_t ns; +#endif + ngx_pool_t *pool; + ngx_event_t *rev; + ngx_rbtree_t *tree; + ngx_rbtree_node_t *node; + ngx_quic_stream_t *qs; + ngx_quic_connection_t *qc; + + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "close quic connection"); + + qc = c->quic; + + if (qc) { + tree = &qc->streams.tree; + + if (tree->root != tree->sentinel) { + if (c->read->timer_set) { + ngx_del_timer(c->read); + } - /* XXX wait for all streams to close */ +#if (NGX_DEBUG) + ns = 0; +#endif + + for (node = ngx_rbtree_min(tree->root, tree->sentinel); + node; + node = ngx_rbtree_next(tree, node)) + { + qs = (ngx_quic_stream_t *) node; + + rev = qs->c->read; + rev->ready = 1; + rev->pending_eof = 1; - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, - "close quic connection: %d", c->fd); + ngx_post_event(rev, &ngx_posted_events); + +#if (NGX_DEBUG) + ns++; +#endif + } + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic connection has %ui active streams", ns); + + qc->closing = 1; + return; + } + } if (c->ssl) { (void) ngx_ssl_shutdown(c); @@ -1587,12 +1640,16 @@ ngx_quic_stream_send(ngx_connection_t *c ngx_quic_stream_t *qs; ngx_quic_connection_t *qc; - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic send: %uz", size); - qs = c->qs; pc = qs->parent; qc = pc->quic; + if (qc->closing) { + return NGX_ERROR; + } + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic send: %uz", size); + frame = ngx_pcalloc(pc->pool, sizeof(ngx_quic_frame_t)); if (frame == NULL) { return 0; @@ -1642,6 +1699,15 @@ ngx_quic_stream_cleanup_handler(void *da pc = qs->parent; qc = pc->quic; + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic stream cleanup"); + + ngx_rbtree_delete(&qc->streams.tree, &qs->node); + + if (qc->closing) { + ngx_post_event(pc->read, &ngx_posted_events); + return; + } + if ((qs->id & 0x03) == NGX_QUIC_STREAM_UNIDIRECTIONAL) { /* do not send fin for client unidirectional streams */ return; diff --git a/src/http/v3/ngx_http_v3_streams.c b/src/http/v3/ngx_http_v3_streams.c --- a/src/http/v3/ngx_http_v3_streams.c +++ b/src/http/v3/ngx_http_v3_streams.c @@ -29,6 +29,7 @@ static void ngx_http_v3_close_uni_stream static void ngx_http_v3_uni_stream_cleanup(void *data); static void ngx_http_v3_read_uni_stream_type(ngx_event_t *rev); static void ngx_http_v3_uni_read_handler(ngx_event_t *rev); +static void ngx_http_v3_dummy_write_handler(ngx_event_t *wev); static ngx_connection_t *ngx_http_v3_create_uni_stream(ngx_connection_t *c, ngx_uint_t type); static ngx_connection_t *ngx_http_v3_get_control(ngx_connection_t *c); @@ -74,6 +75,8 @@ ngx_http_v3_handle_client_uni_stream(ngx cln->data = c; c->read->handler = ngx_http_v3_read_uni_stream_type; + c->write->handler = ngx_http_v3_dummy_write_handler; + ngx_http_v3_read_uni_stream_type(c->read); } @@ -310,6 +313,21 @@ failed: } +static void +ngx_http_v3_dummy_write_handler(ngx_event_t *wev) +{ + ngx_connection_t *c; + + c = wev->data; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http3 dummy write handler"); + + if (ngx_handle_write_event(wev, 0) != NGX_OK) { + ngx_http_v3_close_uni_stream(c); + } +} + + /* XXX async & buffered stream writes */ static ngx_connection_t * @@ -338,6 +356,9 @@ ngx_http_v3_create_uni_stream(ngx_connec us->type = type; sc->data = us; + sc->read->handler = ngx_http_v3_uni_read_handler; + sc->write->handler = ngx_http_v3_dummy_write_handler; + cln = ngx_pool_cleanup_add(sc->pool, 0); if (cln == NULL) { goto failed;