Mercurial > hg > nginx

comparison src/event/ngx_event_quic.c @ 8355:ad3a6f069498 quic

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added proper handling of connection close phases. There are following flags in quic connection: closing - true, when a connection close is initiated, for whatever reason draining - true, when a CC frame is received from peer The following state machine is used for closing: +------------------+ | I/HS/AD | +------------------+ | | | | | V | | immediate close initiated: | | reasons: close by top-level protocol, fatal error | | + sends CC (probably with app-level message) | | + starts close_timer: 3 * PTO (current probe timeout) | | | | | V | | +---------+ - Reply to input with CC (rate-limited) | | | CLOSING | - Close/Reset all streams | | +---------+ | | | | | V V | | receives CC | | | | idle | | timer | | | V | | +----------+ | - MUST NOT send anything (MAY send a single CC) | | DRAINING | | - if not already started, starts close_timer: 3 * PTO | +----------+ | - if not already done, close all streams | | | | | | | close_timer fires | | V V +------------------------+ | CLOSED | - clean up all the resources, drop connection +------------------------+ state completely The ngx_quic_close_connection() function gets an "rc" argument, that signals reason of connection closing: NGX_OK - initiated by application (i.e. http/3), follow state machine NGX_DONE - timedout (while idle or draining) NGX_ERROR - fatal error, destroy connection immediately The PTO calculations are not yet implemented, hardcoded value of 5s is used.
author Vladimir Homutov <vl@nginx.com>
date Thu, 23 Apr 2020 13:41:08 +0300
parents d11bc25fc4c3
children 42198f77ac85
comparison
equal deleted inserted replaced
8354:d11bc25fc4c3 8355:ad3a6f069498
92 92
93 ngx_ssl_t *ssl; 93 ngx_ssl_t *ssl;
94 94
95 ngx_event_t push; 95 ngx_event_t push;
96 ngx_event_t retry; 96 ngx_event_t retry;
97 ngx_event_t close;
97 ngx_queue_t free_frames; 98 ngx_queue_t free_frames;
99 ngx_msec_t last_cc;
98 100
99 #if (NGX_DEBUG) 101 #if (NGX_DEBUG)
100 ngx_uint_t nframes; 102 ngx_uint_t nframes;
101 #endif 103 #endif
102 104
106 uint64_t cur_streams; 108 uint64_t cur_streams;
107 uint64_t max_streams; 109 uint64_t max_streams;
108 110
109 unsigned send_timer_set:1; 111 unsigned send_timer_set:1;
110 unsigned closing:1; 112 unsigned closing:1;
113 unsigned draining:1;
111 unsigned key_phase:1; 114 unsigned key_phase:1;
112 }; 115 };
113 116
114 117
115 typedef ngx_int_t (*ngx_quic_frame_handler_pt)(ngx_connection_t *c, 118 typedef ngx_int_t (*ngx_quic_frame_handler_pt)(ngx_connection_t *c,
140 ngx_quic_tp_t *tp, ngx_quic_header_t *pkt, 143 ngx_quic_tp_t *tp, ngx_quic_header_t *pkt,
141 ngx_connection_handler_pt handler); 144 ngx_connection_handler_pt handler);
142 static ngx_int_t ngx_quic_init_connection(ngx_connection_t *c); 145 static ngx_int_t ngx_quic_init_connection(ngx_connection_t *c);
143 static void ngx_quic_input_handler(ngx_event_t *rev); 146 static void ngx_quic_input_handler(ngx_event_t *rev);
144 147
145 static void ngx_quic_close_connection(ngx_connection_t *c); 148 static void ngx_quic_close_connection(ngx_connection_t *c, ngx_int_t rc);
146 static ngx_int_t ngx_quic_close_quic(ngx_connection_t *c); 149 static ngx_int_t ngx_quic_close_quic(ngx_connection_t *c, ngx_int_t rc);
150 static void ngx_quic_close_timer_handler(ngx_event_t *ev);
147 static ngx_int_t ngx_quic_close_streams(ngx_connection_t *c, 151 static ngx_int_t ngx_quic_close_streams(ngx_connection_t *c,
148 ngx_quic_connection_t *qc); 152 ngx_quic_connection_t *qc);
149 153
150 static ngx_int_t ngx_quic_input(ngx_connection_t *c, ngx_buf_t *b); 154 static ngx_int_t ngx_quic_input(ngx_connection_t *c, ngx_buf_t *b);
151 static ngx_int_t ngx_quic_initial_input(ngx_connection_t *c, 155 static ngx_int_t ngx_quic_initial_input(ngx_connection_t *c,
156 ngx_quic_header_t *pkt); 160 ngx_quic_header_t *pkt);
157 static ngx_int_t ngx_quic_app_input(ngx_connection_t *c, 161 static ngx_int_t ngx_quic_app_input(ngx_connection_t *c,
158 ngx_quic_header_t *pkt); 162 ngx_quic_header_t *pkt);
159 static ngx_int_t ngx_quic_payload_handler(ngx_connection_t *c, 163 static ngx_int_t ngx_quic_payload_handler(ngx_connection_t *c,
160 ngx_quic_header_t *pkt); 164 ngx_quic_header_t *pkt);
165 static ngx_int_t ngx_quic_send_cc(ngx_connection_t *c,
166 enum ssl_encryption_level_t level, ngx_uint_t err);
161 167
162 static ngx_int_t ngx_quic_handle_ack_frame(ngx_connection_t *c, 168 static ngx_int_t ngx_quic_handle_ack_frame(ngx_connection_t *c,
163 ngx_quic_header_t *pkt, ngx_quic_ack_frame_t *f); 169 ngx_quic_header_t *pkt, ngx_quic_ack_frame_t *f);
164 static ngx_int_t ngx_quic_handle_ack_frame_range(ngx_connection_t *c, 170 static ngx_int_t ngx_quic_handle_ack_frame_range(ngx_connection_t *c,
165 ngx_quic_send_ctx_t *ctx, uint64_t min, uint64_t max); 171 ngx_quic_send_ctx_t *ctx, uint64_t min, uint64_t max);
433 static int 439 static int
434 ngx_quic_send_alert(ngx_ssl_conn_t *ssl_conn, enum ssl_encryption_level_t level, 440 ngx_quic_send_alert(ngx_ssl_conn_t *ssl_conn, enum ssl_encryption_level_t level,
435 uint8_t alert) 441 uint8_t alert)
436 { 442 {
437 ngx_connection_t *c; 443 ngx_connection_t *c;
438 ngx_quic_frame_t *frame;
439 444
440 c = ngx_ssl_get_connection((ngx_ssl_conn_t *) ssl_conn); 445 c = ngx_ssl_get_connection((ngx_ssl_conn_t *) ssl_conn);
441 446
442 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, 447 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
443 "ngx_quic_send_alert(), lvl=%d, alert=%d", 448 "ngx_quic_send_alert(), lvl=%d, alert=%d",
444 (int) level, (int) alert); 449 (int) level, (int) alert);
445 450
446 frame = ngx_quic_alloc_frame(c, 0); 451 if (c->quic == NULL) {
447 if (frame == NULL) { 452 return 1;
448 return 0; 453 }
449 } 454
450 455 if (ngx_quic_send_cc(c, level, 0x100 + alert) != NGX_OK) {
451 frame->level = level;
452 frame->type = NGX_QUIC_FT_CONNECTION_CLOSE;
453 frame->u.close.error_code = 0x100 + alert;
454 ngx_sprintf(frame->info, "cc from send_alert level=%d", frame->level);
455
456 ngx_quic_queue_frame(c->quic, frame);
457
458 if (ngx_quic_output(c) != NGX_OK) {
459 return 0; 456 return 0;
460 } 457 }
461 458
462 return 1; 459 return 1;
463 } 460 }
482 pkt.raw = b; 479 pkt.raw = b;
483 pkt.data = b->start; 480 pkt.data = b->start;
484 pkt.len = b->last - b->start; 481 pkt.len = b->last - b->start;
485 482
486 if (ngx_quic_new_connection(c, ssl, tp, &pkt, handler) != NGX_OK) { 483 if (ngx_quic_new_connection(c, ssl, tp, &pkt, handler) != NGX_OK) {
487 ngx_quic_close_connection(c); 484 ngx_quic_close_connection(c, NGX_ERROR);
488 return; 485 return;
489 } 486 }
490 487
491 ngx_add_timer(c->read, c->quic->tp.max_idle_timeout); 488 ngx_add_timer(c->read, c->quic->tp.max_idle_timeout);
492 489
719 c = rev->data; 716 c = rev->data;
720 qc = c->quic; 717 qc = c->quic;
721 718
722 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, rev->log, 0, "quic input handler"); 719 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, rev->log, 0, "quic input handler");
723 720
724 if (qc->closing) {
725 ngx_quic_close_connection(c);
726 return;
727 }
728
729 if (rev->timedout) { 721 if (rev->timedout) {
730 ngx_log_error(NGX_LOG_INFO, c->log, NGX_ETIMEDOUT, "client timed out"); 722 ngx_log_error(NGX_LOG_INFO, c->log, NGX_ETIMEDOUT, "client timed out");
731 ngx_quic_close_connection(c); 723 ngx_quic_close_connection(c, NGX_DONE);
732 return; 724 return;
733 } 725 }
734 726
735 if (c->close) { 727 if (c->close) {
736 ngx_quic_close_connection(c); 728 ngx_quic_close_connection(c, NGX_ERROR);
737 return; 729 return;
738 } 730 }
739 731
740 n = c->recv(c, b.start, b.end - b.start); 732 n = c->recv(c, b.start, b.end - b.start);
741 733
742 if (n == NGX_AGAIN) { 734 if (n == NGX_AGAIN) {
735 if (qc->closing) {
736 ngx_quic_close_connection(c, NGX_OK);
737 }
743 return; 738 return;
744 } 739 }
745 740
746 if (n == NGX_ERROR) { 741 if (n == NGX_ERROR) {
747 c->read->eof = 1; 742 c->read->eof = 1;
748 ngx_quic_close_connection(c); 743 ngx_quic_close_connection(c, NGX_ERROR);
749 return; 744 return;
750 } 745 }
751 746
752 b.last += n; 747 b.last += n;
753 748
754 if (ngx_quic_input(c, &b) != NGX_OK) { 749 if (ngx_quic_input(c, &b) != NGX_OK) {
755 ngx_quic_close_connection(c); 750 ngx_quic_close_connection(c, NGX_ERROR);
756 return; 751 return;
757 } 752 }
758 753
759 qc->send_timer_set = 0; 754 qc->send_timer_set = 0;
760 ngx_add_timer(rev, qc->tp.max_idle_timeout); 755 ngx_add_timer(rev, qc->tp.max_idle_timeout);
761 } 756 }
762 757
763 758
764 static void 759 static void
765 ngx_quic_close_connection(ngx_connection_t *c) 760 ngx_quic_close_connection(ngx_connection_t *c, ngx_int_t rc)
766 { 761 {
767 ngx_pool_t *pool; 762 ngx_pool_t *pool;
768 763
769 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "close quic connection"); 764 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
770 765 "close quic connection, rc: %i", rc);
771 if (c->quic && ngx_quic_close_quic(c) == NGX_AGAIN) { 766
767 if (!c->quic) {
768 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
769 "close quic connection: early error");
770
771 } else if (ngx_quic_close_quic(c, rc) == NGX_AGAIN) {
772 return; 772 return;
773 } 773 }
774 774
775 if (c->ssl) { 775 if (c->ssl) {
776 (void) ngx_ssl_shutdown(c); 776 (void) ngx_ssl_shutdown(c);
793 ngx_destroy_pool(pool); 793 ngx_destroy_pool(pool);
794 } 794 }
795 795
796 796
797 static ngx_int_t 797 static ngx_int_t
798 ngx_quic_close_quic(ngx_connection_t *c) 798 ngx_quic_close_quic(ngx_connection_t *c, ngx_int_t rc)
799 { 799 {
800 ngx_uint_t i; 800 ngx_uint_t i;
801 ngx_quic_connection_t *qc; 801 ngx_quic_connection_t *qc;
802 enum ssl_encryption_level_t level;
802 803
803 qc = c->quic; 804 qc = c->quic;
804 805
805 qc->closing = 1; 806 if (!qc->closing) {
807
808 if (rc == NGX_OK) {
809
810 /*
811 * 10.3. Immediate Close
812 *
813 * An endpoint sends a CONNECTION_CLOSE frame (Section 19.19) to
814 * terminate the connection immediately.
815 */
816 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
817 "quic immediate close, drain = %d", qc->draining);
818
819 switch (qc->state) {
820 case NGX_QUIC_ST_INITIAL:
821 level = ssl_encryption_initial;
822 break;
823
824 case NGX_QUIC_ST_HANDSHAKE:
825 level = ssl_encryption_handshake;
826 break;
827
828 default: /* NGX_QUIC_ST_APPLICATION/EARLY_DATA */
829 level = ssl_encryption_application;
830 break;
831 }
832
833 if (ngx_quic_send_cc(c, level, NGX_QUIC_ERR_NO_ERROR) == NGX_OK) {
834
835 qc->close.log = c->log;
836 qc->close.data = c;
837 qc->close.handler = ngx_quic_close_timer_handler;
838 qc->close.cancelable = 1;
839
840 ngx_add_timer(&qc->close, 3 * NGX_QUIC_HARDCODED_PTO);
841 }
842
843 } else if (rc == NGX_DONE) {
844
845 /*
846 * 10.2. Idle Timeout
847 *
848 * If the idle timeout is enabled by either peer, a connection is
849 * silently closed and its state is discarded when it remains idle
850 */
851
852 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
853 "quic closing %s connection",
854 qc->draining ? "drained" : "idle");
855
856 } else {
857 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
858 "quic immediate close due to fatal error");
859 }
860
861 qc->closing = 1;
862 }
863
864 if (rc == NGX_ERROR && qc->close.timer_set) {
865 /* do not wait for timer in case of fatal error */
866 ngx_del_timer(&qc->close);
867 }
806 868
807 if (ngx_quic_close_streams(c, qc) == NGX_AGAIN) { 869 if (ngx_quic_close_streams(c, qc) == NGX_AGAIN) {
870 return NGX_AGAIN;
871 }
872
873 if (qc->close.timer_set) {
808 return NGX_AGAIN; 874 return NGX_AGAIN;
809 } 875 }
810 876
811 for (i = 0; i < NGX_QUIC_ENCRYPTION_LAST; i++) { 877 for (i = 0; i < NGX_QUIC_ENCRYPTION_LAST; i++) {
812 ngx_quic_free_frames(c, &qc->crypto[i].frames); 878 ngx_quic_free_frames(c, &qc->crypto[i].frames);
823 889
824 if (qc->retry.timer_set) { 890 if (qc->retry.timer_set) {
825 ngx_del_timer(&qc->retry); 891 ngx_del_timer(&qc->retry);
826 } 892 }
827 893
894 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
895 "quic part of connection is terminated");
896
897 /* may be tested from SSL callback during SSL shutdown */
898 c->quic = NULL;
899
828 return NGX_OK; 900 return NGX_OK;
901 }
902
903
904 static void
905 ngx_quic_close_timer_handler(ngx_event_t *ev)
906 {
907 ngx_connection_t *c;
908
909 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, "close timer");
910
911 c = ev->data;
912 ngx_quic_close_connection(c, NGX_DONE);
829 } 913 }
830 914
831 915
832 static ngx_int_t 916 static ngx_int_t
833 ngx_quic_close_streams(ngx_connection_t *c, ngx_quic_connection_t *qc) 917 ngx_quic_close_streams(ngx_connection_t *c, ngx_quic_connection_t *qc)
1201 ssize_t len; 1285 ssize_t len;
1202 ngx_uint_t ack_this, do_close; 1286 ngx_uint_t ack_this, do_close;
1203 ngx_quic_frame_t frame, *ack_frame; 1287 ngx_quic_frame_t frame, *ack_frame;
1204 ngx_quic_connection_t *qc; 1288 ngx_quic_connection_t *qc;
1205 1289
1206
1207 qc = c->quic; 1290 qc = c->quic;
1291
1292 if (qc->closing) {
1293 /*
1294 * 10.1 Closing and Draining Connection States
1295 * ... delayed or reordered packets are properly discarded.
1296 *
1297 * An endpoint retains only enough information to generate
1298 * a packet containing a CONNECTION_CLOSE frame and to identify
1299 * packets as belonging to the connection.
1300 */
1301 return ngx_quic_send_cc(c, pkt->level, NGX_QUIC_ERR_NO_ERROR);
1302 }
1208 1303
1209 p = pkt->payload.data; 1304 p = pkt->payload.data;
1210 end = p + pkt->payload.len; 1305 end = p + pkt->payload.len;
1211 1306
1212 ack_this = 0; 1307 ack_this = 0;
1337 "trailing garbage in payload: %ui bytes", end - p); 1432 "trailing garbage in payload: %ui bytes", end - p);
1338 return NGX_ERROR; 1433 return NGX_ERROR;
1339 } 1434 }
1340 1435
1341 if (do_close) { 1436 if (do_close) {
1342 return NGX_DONE; 1437 qc->draining = 1;
1438 ngx_quic_close_connection(c, NGX_OK);
1439 return NGX_OK;
1343 } 1440 }
1344 1441
1345 if (ack_this == 0) { 1442 if (ack_this == 0) {
1346 /* do not ack packets with ACKs and PADDING */ 1443 /* do not ack packets with ACKs and PADDING */
1347 return NGX_OK; 1444 return NGX_OK;
1368 1465
1369 ngx_sprintf(ack_frame->info, "ACK for PN=%d from frame handler level=%d", pkt->pn, ack_frame->level); 1466 ngx_sprintf(ack_frame->info, "ACK for PN=%d from frame handler level=%d", pkt->pn, ack_frame->level);
1370 ngx_quic_queue_frame(qc, ack_frame); 1467 ngx_quic_queue_frame(qc, ack_frame);
1371 1468
1372 return NGX_OK; 1469 return NGX_OK;
1470 }
1471
1472
1473 static ngx_int_t
1474 ngx_quic_send_cc(ngx_connection_t *c, enum ssl_encryption_level_t level,
1475 ngx_uint_t err)
1476 {
1477 ngx_quic_frame_t *frame;
1478 ngx_quic_connection_t *qc;
1479
1480 qc = c->quic;
1481
1482 if (qc->draining) {
1483 return NGX_OK;
1484 }
1485
1486 if (qc->closing
1487 && ngx_current_msec - qc->last_cc < NGX_QUIC_CC_MIN_INTERVAL)
1488 {
1489 /* dot not send CC too often */
1490 return NGX_OK;
1491 }
1492
1493 frame = ngx_quic_alloc_frame(c, 0);
1494 if (frame == NULL) {
1495 return NGX_ERROR;
1496 }
1497
1498 frame->level = level;
1499 frame->type = NGX_QUIC_FT_CONNECTION_CLOSE;
1500 frame->u.close.error_code = err;
1501 ngx_sprintf(frame->info, "cc from send_cc err=%ui level=%d", err,
1502 frame->level);
1503
1504 ngx_quic_queue_frame(c->quic, frame);
1505
1506 qc->last_cc = ngx_current_msec;
1507
1508 return ngx_quic_output(c);
1373 } 1509 }
1374 1510
1375 1511
1376 static ngx_int_t 1512 static ngx_int_t
1377 ngx_quic_handle_ack_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, 1513 ngx_quic_handle_ack_frame(ngx_connection_t *c, ngx_quic_header_t *pkt,
2155 if (rc == NGX_OK) { 2291 if (rc == NGX_OK) {
2156 /* 2292 /*
2157 * frames are moved into the sent queue 2293 * frames are moved into the sent queue
2158 * to wait for ack/be retransmitted 2294 * to wait for ack/be retransmitted
2159 */ 2295 */
2160 ngx_queue_add(&ctx->sent, &range); 2296 if (qc->closing) {
2297 /* if we are closing, any ack will be discarded */
2298 ngx_quic_free_frames(c, &range);
2299
2300 } else {
2301 ngx_queue_add(&ctx->sent, &range);
2302 }
2161 2303
2162 } else if (rc == NGX_DONE) { 2304 } else if (rc == NGX_DONE) {
2163 2305
2164 /* no ack is expected for this frames, can free them */ 2306 /* no ack is expected for this frames, can free them */
2165 ngx_quic_free_frames(c, &range); 2307 ngx_quic_free_frames(c, &range);
2361 2503
2362 wait = 0; 2504 wait = 0;
2363 2505
2364 for (i = 0; i < NGX_QUIC_SEND_CTX_LAST; i++) { 2506 for (i = 0; i < NGX_QUIC_SEND_CTX_LAST; i++) {
2365 if (ngx_quic_retransmit(c, &qc->send_ctx[i], &nswait) != NGX_OK) { 2507 if (ngx_quic_retransmit(c, &qc->send_ctx[i], &nswait) != NGX_OK) {
2366 ngx_quic_close_connection(c); 2508 ngx_quic_close_connection(c, NGX_ERROR);
2367 return; 2509 return;
2368 } 2510 }
2369 2511
2370 if (i == 0) { 2512 if (i == 0) {
2371 wait = nswait; 2513 wait = nswait;
2389 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, "push timer"); 2531 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, "push timer");
2390 2532
2391 c = ev->data; 2533 c = ev->data;
2392 2534
2393 if (ngx_quic_output(c) != NGX_OK) { 2535 if (ngx_quic_output(c) != NGX_OK) {
2394 ngx_quic_close_connection(c); 2536 ngx_quic_close_connection(c, NGX_ERROR);
2395 return; 2537 return;
2396 } 2538 }
2397 } 2539 }
2398 2540
2399 2541
2807 2949
2808 ngx_rbtree_delete(&qc->streams.tree, &qs->node); 2950 ngx_rbtree_delete(&qc->streams.tree, &qs->node);
2809 ngx_quic_free_frames(pc, &qs->fs.frames); 2951 ngx_quic_free_frames(pc, &qs->fs.frames);
2810 2952
2811 if (qc->closing) { 2953 if (qc->closing) {
2954 /* schedule handler call to continue ngx_quic_close_connection() */
2812 ngx_post_event(pc->read, &ngx_posted_events); 2955 ngx_post_event(pc->read, &ngx_posted_events);
2813 return; 2956 return;
2814 } 2957 }
2815 2958
2816 if ((qs->id & 0x03) == NGX_QUIC_STREAM_UNIDIRECTIONAL) { 2959 if ((qs->id & 0x03) == NGX_QUIC_STREAM_UNIDIRECTIONAL) {