Mercurial > hg > nginx
changeset 8763:4117aa7fa38e quic
QUIC: connection migration.
The patch adds proper transitions between multiple networking addresses that
can be used by a single quic connection. New networking paths are validated
using PATH_CHALLENGE/PATH_RESPONSE frames.
| author | Vladimir Homutov <vl@nginx.com> |
|---|---|
| date | Thu, 29 Apr 2021 15:35:02 +0300 |
| parents | 12f18e0bca09 |
| children | 0bd1dd39a08b |
| files | README auto/modules src/event/quic/ngx_event_quic.c src/event/quic/ngx_event_quic_ack.c src/event/quic/ngx_event_quic_ack.h src/event/quic/ngx_event_quic_connection.h src/event/quic/ngx_event_quic_connid.c src/event/quic/ngx_event_quic_connid.h src/event/quic/ngx_event_quic_migration.c src/event/quic/ngx_event_quic_migration.h src/event/quic/ngx_event_quic_output.c src/event/quic/ngx_event_quic_output.h src/event/quic/ngx_event_quic_socket.c src/event/quic/ngx_event_quic_socket.h src/event/quic/ngx_event_quic_ssl.c src/event/quic/ngx_event_quic_tokens.c src/event/quic/ngx_event_quic_tokens.h |
| diffstat | 17 files changed, 1723 insertions(+), 416 deletions(-) [+] |
line wrap: on
line diff
--- a/README +++ b/README @@ -51,14 +51,13 @@ 1. Introduction subsequently reference them from header blocks + Version Negotiation packet is sent to client with unknown version + Lost packets are detected and retransmitted properly + + Clients may migrate to new address Not (yet) supported features: - Explicit Congestion Notification (ECN) as specified in quic-recovery [5] - A connection with the spin bit succeeds and the bit is spinning - Structured Logging - - NAT Rebinding - - Address Mobility - HTTP/3 trailers Since the code is experimental and still under development,
--- a/auto/modules +++ b/auto/modules @@ -1350,7 +1350,8 @@ if [ $USE_OPENSSL$USE_OPENSSL_QUIC = YES src/event/quic/ngx_event_quic_ssl.h \ src/event/quic/ngx_event_quic_tokens.h \ src/event/quic/ngx_event_quic_ack.h \ - src/event/quic/ngx_event_quic_output.h" + src/event/quic/ngx_event_quic_output.h \ + src/event/quic/ngx_event_quic_socket.h" ngx_module_srcs="src/event/quic/ngx_event_quic.c \ src/event/quic/ngx_event_quic_transport.c \ src/event/quic/ngx_event_quic_protection.c \ @@ -1361,7 +1362,8 @@ if [ $USE_OPENSSL$USE_OPENSSL_QUIC = YES src/event/quic/ngx_event_quic_ssl.c \ src/event/quic/ngx_event_quic_tokens.c \ src/event/quic/ngx_event_quic_ack.c \ - src/event/quic/ngx_event_quic_output.c" + src/event/quic/ngx_event_quic_output.c \ + src/event/quic/ngx_event_quic_socket.c" ngx_module_libs= ngx_module_link=YES
--- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -87,7 +87,6 @@ ngx_quic_connstate_dbg(ngx_connection_t p = ngx_slprintf(p, last, "%s", qc->closing ? " closing" : ""); p = ngx_slprintf(p, last, "%s", qc->draining ? " draining" : ""); p = ngx_slprintf(p, last, "%s", qc->key_phase ? " kp" : ""); - p = ngx_slprintf(p, last, "%s", qc->validated? " valid" : ""); } else { p = ngx_slprintf(p, last, " early"); @@ -127,12 +126,16 @@ ngx_quic_connstate_dbg(ngx_connection_t ngx_int_t ngx_quic_apply_transport_params(ngx_connection_t *c, ngx_quic_tp_t *ctp) { + ngx_str_t scid; ngx_quic_connection_t *qc; qc = ngx_quic_get_connection(c); - if (qc->scid.len != ctp->initial_scid.len - || ngx_memcmp(qc->scid.data, ctp->initial_scid.data, qc->scid.len) != 0) + scid.data = qc->socket->cid->id; + scid.len = qc->socket->cid->len; + + if (scid.len != ctp->initial_scid.len + || ngx_memcmp(scid.data, ctp->initial_scid.data, scid.len) != 0) { ngx_log_error(NGX_LOG_INFO, c->log, 0, "quic client initial_source_connection_id mismatch"); @@ -277,8 +280,6 @@ ngx_quic_new_connection(ngx_connection_t * qc->latest_rtt = 0 */ - qc->received = pkt->raw->last - pkt->raw->start; - qc->pto.log = c->log; qc->pto.data = c; qc->pto.handler = ngx_quic_pto_handler; @@ -289,19 +290,14 @@ ngx_quic_new_connection(ngx_connection_t qc->push.handler = ngx_quic_push_handler; qc->push.cancelable = 1; + qc->path_validation.log = c->log; + qc->path_validation.data = c; + qc->path_validation.handler = ngx_quic_path_validation_handler; + qc->path_validation.cancelable = 1; + qc->conf = conf; qc->tp = conf->tp; - if (qc->tp.disable_active_migration) { - qc->sockaddr = ngx_palloc(c->pool, c->socklen); - if (qc->sockaddr == NULL) { - return NULL; - } - - ngx_memcpy(qc->sockaddr, c->sockaddr, c->socklen); - qc->socklen = c->socklen; - } - ctp = &qc->ctp; /* defaults to be used before actual client parameters are received */ @@ -338,10 +334,13 @@ ngx_quic_new_connection(ngx_connection_t qc->validated = pkt->validated; - if (ngx_quic_setup_connection_ids(c, qc, pkt) != NGX_OK) { + if (ngx_quic_open_sockets(c, qc, pkt) != NGX_OK) { return NULL; } + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic connection created"); + return qc; } @@ -425,20 +424,8 @@ ngx_quic_input_handler(ngx_event_t *rev) return; } - if (qc->tp.disable_active_migration) { - if (c->socklen != qc->socklen - || ngx_memcmp(c->sockaddr, qc->sockaddr, c->socklen) != 0) - { - ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic dropping packet from new address"); - return; - } - } - b = c->udp->dgram->buffer; - qc->received += (b->last - b->pos); - rc = ngx_quic_input(c, b, NULL); if (rc == NGX_ERROR) { @@ -506,9 +493,7 @@ static ngx_int_t ngx_quic_close_quic(ngx_connection_t *c, ngx_int_t rc) { ngx_uint_t i; - ngx_queue_t *q; ngx_quic_send_ctx_t *ctx; - ngx_quic_server_id_t *sid; ngx_quic_connection_t *qc; qc = ngx_quic_get_connection(c); @@ -601,23 +586,20 @@ ngx_quic_close_quic(ngx_connection_t *c, ngx_del_timer(&qc->pto); } - if (qc->push.posted) { - ngx_delete_posted_event(&qc->push); + if (qc->path_validation.timer_set) { + ngx_del_timer(&qc->path_validation); } - while (!ngx_queue_empty(&qc->server_ids)) { - q = ngx_queue_head(&qc->server_ids); - sid = ngx_queue_data(q, ngx_quic_server_id_t, queue); - - ngx_queue_remove(q); - ngx_rbtree_delete(&c->listening->rbtree, &sid->udp.node); - qc->nserver_ids--; + if (qc->push.posted) { + ngx_delete_posted_event(&qc->push); } if (qc->close.timer_set) { return NGX_AGAIN; } + ngx_quic_close_sockets(c); + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic part of connection is terminated"); @@ -801,6 +783,11 @@ ngx_quic_process_packet(ngx_connection_t return NGX_DECLINED; } + rc = ngx_quic_check_migration(c, pkt); + if (rc != NGX_OK) { + return rc; + } + if (pkt->level != ssl_encryption_application) { if (pkt->version != qc->version) { @@ -946,6 +933,10 @@ ngx_quic_process_payload(ngx_connection_ pkt->decrypted = 1; + if (ngx_quic_update_paths(c, pkt) != NGX_OK) { + return NGX_ERROR; + } + if (c->ssl == NULL) { if (ngx_quic_init_connection(c) != NGX_OK) { return NGX_ERROR; @@ -959,8 +950,8 @@ ngx_quic_process_payload(ngx_connection_ */ ngx_quic_discard_ctx(c, ssl_encryption_initial); - if (qc->validated == 0) { - qc->validated = 1; + if (qc->socket->path->state != NGX_QUIC_PATH_VALIDATED) { + qc->socket->path->state = NGX_QUIC_PATH_VALIDATED; ngx_post_event(&qc->push, &ngx_posted_events); } } @@ -1015,6 +1006,7 @@ ngx_quic_discard_ctx(ngx_connection_t *c { ngx_queue_t *q; ngx_quic_frame_t *f; + ngx_quic_socket_t *qsock; ngx_quic_send_ctx_t *ctx; ngx_quic_connection_t *qc; @@ -1049,7 +1041,11 @@ ngx_quic_discard_ctx(ngx_connection_t *c } if (level == ssl_encryption_initial) { - ngx_quic_clear_temp_server_ids(c); + /* close temporary listener with odcid */ + qsock = ngx_quic_find_socket(c, NGX_QUIC_UNSET_PN); + if (qsock) { + ngx_quic_close_socket(c, qsock); + } } ctx->send_ack = 0; @@ -1088,9 +1084,10 @@ ngx_quic_handle_frames(ngx_connection_t u_char *end, *p; ssize_t len; ngx_buf_t buf; - ngx_uint_t do_close; + ngx_uint_t do_close, nonprobing; ngx_chain_t chain; ngx_quic_frame_t frame; + ngx_quic_socket_t *qsock; ngx_quic_connection_t *qc; qc = ngx_quic_get_connection(c); @@ -1099,11 +1096,13 @@ ngx_quic_handle_frames(ngx_connection_t end = p + pkt->payload.len; do_close = 0; + nonprobing = 0; while (p < end) { c->log->action = "parsing frames"; + ngx_memzero(&frame, sizeof(ngx_quic_frame_t)); ngx_memzero(&buf, sizeof(ngx_buf_t)); buf.temporary = 1; @@ -1125,6 +1124,19 @@ ngx_quic_handle_frames(ngx_connection_t p += len; switch (frame.type) { + /* probing frames */ + case NGX_QUIC_FT_PADDING: + case NGX_QUIC_FT_PATH_CHALLENGE: + case NGX_QUIC_FT_PATH_RESPONSE: + break; + + /* non-probing frames */ + default: + nonprobing = 1; + break; + } + + switch (frame.type) { case NGX_QUIC_FT_ACK: if (ngx_quic_handle_ack_frame(c, pkt, &frame) != NGX_OK) { @@ -1313,6 +1325,26 @@ ngx_quic_handle_frames(ngx_connection_t ngx_quic_close_connection(c, NGX_OK); } + qsock = ngx_quic_get_socket(c); + + if (qsock != qc->socket) { + + if (qsock->path != qc->socket->path && nonprobing) { + /* + * An endpoint can migrate a connection to a new local + * address by sending packets containing non-probing frames + * from that address. + */ + if (ngx_quic_handle_migration(c, pkt) != NGX_OK) { + return NGX_ERROR; + } + } + /* + * else: packet arrived via non-default socket; + * no reason to change active path + */ + } + if (ngx_quic_ack_packet(c, pkt) != NGX_OK) { return NGX_ERROR; }
--- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -722,7 +722,7 @@ ngx_quic_pto_handler(ngx_event_t *ev) for (i = 0; i < NGX_QUIC_SEND_CTX_LAST; i++) { - ctx = &qc->send_ctx[i]; + ctx = &qc->send_ctx[i]; if (ngx_queue_empty(&ctx->sent)) { continue;
--- a/src/event/quic/ngx_event_quic_ack.h +++ b/src/event/quic/ngx_event_quic_ack.h @@ -17,8 +17,7 @@ ngx_int_t ngx_quic_handle_ack_frame(ngx_ void ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *frame); -void ngx_quic_resend_frames(ngx_connection_t *c, - ngx_quic_send_ctx_t *ctx); +void ngx_quic_resend_frames(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx); void ngx_quic_set_lost_timer(ngx_connection_t *c); void ngx_quic_pto_handler(ngx_event_t *ev); ngx_msec_t ngx_quic_pto(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx);
--- a/src/event/quic/ngx_event_quic_connection.h +++ b/src/event/quic/ngx_event_quic_connection.h @@ -12,7 +12,11 @@ #include <ngx_event.h> typedef struct ngx_quic_connection_s ngx_quic_connection_t; +typedef struct ngx_quic_server_id_s ngx_quic_server_id_t; +typedef struct ngx_quic_client_id_s ngx_quic_client_id_t; typedef struct ngx_quic_send_ctx_s ngx_quic_send_ctx_t; +typedef struct ngx_quic_socket_s ngx_quic_socket_t; +typedef struct ngx_quic_path_s ngx_quic_path_t; typedef struct ngx_quic_keys_s ngx_quic_keys_t; #include <ngx_event_quic_transport.h> @@ -25,6 +29,7 @@ typedef struct ngx_quic_keys_s ng #include <ngx_event_quic_tokens.h> #include <ngx_event_quic_ack.h> #include <ngx_event_quic_output.h> +#include <ngx_event_quic_socket.h> /* quic-recovery, section 6.2.2, kInitialRtt */ @@ -47,26 +52,57 @@ typedef struct ngx_quic_keys_s ng : &((qc)->send_ctx[2])) #define ngx_quic_get_connection(c) \ - (((c)->udp) ? (((ngx_quic_server_id_t *)((c)->udp))->quic) : NULL) + (((c)->udp) ? (((ngx_quic_socket_t *)((c)->udp))->quic) : NULL) + +#define ngx_quic_get_socket(c) ((ngx_quic_socket_t *)((c)->udp)) -typedef struct { +struct ngx_quic_client_id_s { ngx_queue_t queue; uint64_t seqnum; size_t len; u_char id[NGX_QUIC_CID_LEN_MAX]; u_char sr_token[NGX_QUIC_SR_TOKEN_LEN]; -} ngx_quic_client_id_t; + ngx_uint_t refcnt; +}; + + +struct ngx_quic_server_id_s { + uint64_t seqnum; + size_t len; + u_char id[NGX_QUIC_CID_LEN_MAX]; +}; -typedef struct { +struct ngx_quic_path_s { + ngx_queue_t queue; + struct sockaddr *sockaddr; + socklen_t socklen; + ngx_uint_t state; + ngx_msec_t expires; + ngx_uint_t tries; + off_t sent; + off_t received; + u_char challenge1[8]; + u_char challenge2[8]; + ngx_uint_t refcnt; + uint64_t seqnum; + time_t validated_at; + ngx_str_t addr_text; + u_char text[NGX_SOCKADDR_STRLEN]; +}; + + +struct ngx_quic_socket_s { ngx_udp_connection_t udp; ngx_quic_connection_t *quic; ngx_queue_t queue; - uint64_t seqnum; - size_t len; - u_char id[NGX_QUIC_CID_LEN_MAX]; -} ngx_quic_server_id_t; + + ngx_quic_server_id_t sid; + + ngx_quic_path_t *path; + ngx_quic_client_id_t *cid; +}; typedef struct { @@ -138,22 +174,22 @@ struct ngx_quic_send_ctx_s { struct ngx_quic_connection_s { uint32_t version; - ngx_str_t scid; /* initial client ID */ - ngx_str_t dcid; /* server (our own) ID */ - ngx_str_t odcid; /* original server ID */ - - struct sockaddr *sockaddr; - socklen_t socklen; + ngx_quic_socket_t *socket; + ngx_quic_socket_t *backup; + ngx_queue_t sockets; + ngx_queue_t paths; ngx_queue_t client_ids; - ngx_queue_t server_ids; + ngx_queue_t free_sockets; + ngx_queue_t free_paths; ngx_queue_t free_client_ids; - ngx_queue_t free_server_ids; + + ngx_uint_t nsockets; ngx_uint_t nclient_ids; - ngx_uint_t nserver_ids; uint64_t max_retired_seqnum; uint64_t client_seqnum; uint64_t server_seqnum; + uint64_t path_seqnum; ngx_uint_t client_tp_done; ngx_quic_tp_t tp; @@ -170,6 +206,7 @@ struct ngx_quic_connection_s { ngx_event_t push; ngx_event_t pto; ngx_event_t close; + ngx_event_t path_validation; ngx_msec_t last_cc; ngx_msec_t latest_rtt; @@ -190,7 +227,6 @@ struct ngx_quic_connection_s { ngx_quic_streams_t streams; ngx_quic_congestion_t congestion; - off_t received; ngx_uint_t error; enum ssl_encryption_level_t error_level;
--- a/src/event/quic/ngx_event_quic_connid.c +++ b/src/event/quic/ngx_event_quic_connid.c @@ -9,103 +9,24 @@ #include <ngx_event.h> #include <ngx_event_quic_connection.h> - #define NGX_QUIC_MAX_SERVER_IDS 8 -static ngx_int_t ngx_quic_create_server_id(ngx_connection_t *c, u_char *id); #if (NGX_QUIC_BPF) static ngx_int_t ngx_quic_bpf_attach_id(ngx_connection_t *c, u_char *id); #endif -static ngx_int_t ngx_quic_retire_connection_id(ngx_connection_t *c, +static ngx_int_t ngx_quic_send_retire_connection_id(ngx_connection_t *c, enum ssl_encryption_level_t level, uint64_t seqnum); -static ngx_quic_server_id_t *ngx_quic_insert_server_id(ngx_connection_t *c, - ngx_quic_connection_t *qc, ngx_str_t *id); + static ngx_quic_client_id_t *ngx_quic_alloc_client_id(ngx_connection_t *c, ngx_quic_connection_t *qc); -static ngx_quic_server_id_t *ngx_quic_alloc_server_id(ngx_connection_t *c, - ngx_quic_connection_t *qc); +static ngx_int_t ngx_quic_replace_retired_client_id(ngx_connection_t *c, + ngx_quic_client_id_t *retired_cid); +static ngx_int_t ngx_quic_send_server_id(ngx_connection_t *c, + ngx_quic_server_id_t *sid); ngx_int_t -ngx_quic_setup_connection_ids(ngx_connection_t *c, ngx_quic_connection_t *qc, - ngx_quic_header_t *pkt) -{ - ngx_quic_server_id_t *sid, *osid; - ngx_quic_client_id_t *cid; - - /* - * qc->nclient_ids = 0 - * qc->nserver_ids = 0 - * qc->max_retired_seqnum = 0 - */ - - ngx_queue_init(&qc->client_ids); - ngx_queue_init(&qc->server_ids); - ngx_queue_init(&qc->free_client_ids); - ngx_queue_init(&qc->free_server_ids); - - qc->odcid.len = pkt->odcid.len; - qc->odcid.data = ngx_pstrdup(c->pool, &pkt->odcid); - if (qc->odcid.data == NULL) { - return NGX_ERROR; - } - - qc->tp.original_dcid = qc->odcid; - - qc->scid.len = pkt->scid.len; - qc->scid.data = ngx_pstrdup(c->pool, &pkt->scid); - if (qc->scid.data == NULL) { - return NGX_ERROR; - } - - qc->dcid.len = NGX_QUIC_SERVER_CID_LEN; - qc->dcid.data = ngx_pnalloc(c->pool, qc->dcid.len); - if (qc->dcid.data == NULL) { - return NGX_ERROR; - } - - if (ngx_quic_create_server_id(c, qc->dcid.data) != NGX_OK) { - return NGX_ERROR; - } - - qc->tp.initial_scid = qc->dcid; - - cid = ngx_quic_alloc_client_id(c, qc); - if (cid == NULL) { - return NGX_ERROR; - } - - cid->seqnum = 0; - cid->len = pkt->scid.len; - ngx_memcpy(cid->id, pkt->scid.data, pkt->scid.len); - - ngx_queue_insert_tail(&qc->client_ids, &cid->queue); - qc->nclient_ids++; - qc->client_seqnum = 0; - - qc->server_seqnum = NGX_QUIC_UNSET_PN; - - osid = ngx_quic_insert_server_id(c, qc, &qc->odcid); - if (osid == NULL) { - return NGX_ERROR; - } - - qc->server_seqnum = 0; - - sid = ngx_quic_insert_server_id(c, qc, &qc->dcid); - if (sid == NULL) { - ngx_rbtree_delete(&c->listening->rbtree, &osid->udp.node); - return NGX_ERROR; - } - - c->udp = &sid->udp; - - return NGX_OK; -} - - -static ngx_int_t ngx_quic_create_server_id(ngx_connection_t *c, u_char *id) { if (RAND_bytes(id, NGX_QUIC_SERVER_CID_LEN) != 1) { @@ -120,9 +41,6 @@ ngx_quic_create_server_id(ngx_connection } #endif - ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic create server id %*xs", - (size_t) NGX_QUIC_SERVER_CID_LEN, id); return NGX_OK; } @@ -155,12 +73,11 @@ ngx_quic_bpf_attach_id(ngx_connection_t #endif - - ngx_int_t ngx_quic_handle_new_connection_id_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_new_conn_id_frame_t *f) { + ngx_str_t id; ngx_queue_t *q; ngx_quic_client_id_t *cid, *item; ngx_quic_connection_t *qc; @@ -177,7 +94,9 @@ ngx_quic_handle_new_connection_id_frame( * done so for that sequence number. */ - if (ngx_quic_retire_connection_id(c, pkt->level, f->seqnum) != NGX_OK) { + if (ngx_quic_send_retire_connection_id(c, pkt->level, f->seqnum) + != NGX_OK) + { return NGX_ERROR; } @@ -220,25 +139,11 @@ ngx_quic_handle_new_connection_id_frame( } else { - cid = ngx_quic_alloc_client_id(c, qc); - if (cid == NULL) { - return NGX_ERROR; - } - - cid->seqnum = f->seqnum; - cid->len = f->len; - ngx_memcpy(cid->id, f->cid, f->len); + id.data = f->cid; + id.len = f->len; - ngx_memcpy(cid->sr_token, f->srt, NGX_QUIC_SR_TOKEN_LEN); - - ngx_queue_insert_tail(&qc->client_ids, &cid->queue); - qc->nclient_ids++; - - /* always use latest available connection id */ - if (f->seqnum > qc->client_seqnum) { - qc->scid.len = cid->len; - qc->scid.data = cid->id; - qc->client_seqnum = f->seqnum; + if (ngx_quic_create_client_id(c, &id, f->seqnum, f->srt) == NULL) { + return NGX_ERROR; } } @@ -269,15 +174,20 @@ retire: /* this connection id must be retired */ - if (ngx_quic_retire_connection_id(c, pkt->level, cid->seqnum) + if (ngx_quic_send_retire_connection_id(c, pkt->level, cid->seqnum) != NGX_OK) { return NGX_ERROR; } - ngx_queue_remove(&cid->queue); - ngx_queue_insert_head(&qc->free_client_ids, &cid->queue); - qc->nclient_ids--; + if (cid->refcnt) { + /* we are going to retire client id which is in use */ + if (ngx_quic_replace_retired_client_id(c, cid) != NGX_OK) { + return NGX_ERROR; + } + } + + ngx_quic_unref_client_id(c, cid); } done: @@ -300,7 +210,7 @@ done: static ngx_int_t -ngx_quic_retire_connection_id(ngx_connection_t *c, +ngx_quic_send_retire_connection_id(ngx_connection_t *c, enum ssl_encryption_level_t level, uint64_t seqnum) { ngx_quic_frame_t *frame; @@ -319,165 +229,12 @@ ngx_quic_retire_connection_id(ngx_connec ngx_quic_queue_frame(qc, frame); - return NGX_OK; -} - - -ngx_int_t -ngx_quic_handle_retire_connection_id_frame(ngx_connection_t *c, - ngx_quic_header_t *pkt, ngx_quic_retire_cid_frame_t *f) -{ - ngx_queue_t *q; - ngx_quic_server_id_t *sid; - ngx_quic_connection_t *qc; - - qc = ngx_quic_get_connection(c); - - for (q = ngx_queue_head(&qc->server_ids); - q != ngx_queue_sentinel(&qc->server_ids); - q = ngx_queue_next(q)) - { - sid = ngx_queue_data(q, ngx_quic_server_id_t, queue); - - if (sid->seqnum == f->sequence_number) { - ngx_queue_remove(q); - ngx_queue_insert_tail(&qc->free_server_ids, &sid->queue); - ngx_rbtree_delete(&c->listening->rbtree, &sid->udp.node); - qc->nserver_ids--; - break; - } - } - - return ngx_quic_issue_server_ids(c); -} - - -ngx_int_t -ngx_quic_issue_server_ids(ngx_connection_t *c) -{ - ngx_str_t dcid; - ngx_uint_t n; - ngx_quic_frame_t *frame; - ngx_quic_server_id_t *sid; - ngx_quic_connection_t *qc; - u_char id[NGX_QUIC_SERVER_CID_LEN]; - - qc = ngx_quic_get_connection(c); - - n = ngx_min(NGX_QUIC_MAX_SERVER_IDS, qc->ctp.active_connection_id_limit); - - ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic issue server ids has:%ui max:%ui", qc->nserver_ids, n); - - while (qc->nserver_ids < n) { - if (ngx_quic_create_server_id(c, id) != NGX_OK) { - return NGX_ERROR; - } - - dcid.len = NGX_QUIC_SERVER_CID_LEN; - dcid.data = id; - - sid = ngx_quic_insert_server_id(c, qc, &dcid); - if (sid == NULL) { - return NGX_ERROR; - } - - frame = ngx_quic_alloc_frame(c); - if (frame == NULL) { - return NGX_ERROR; - } - - frame->level = ssl_encryption_application; - frame->type = NGX_QUIC_FT_NEW_CONNECTION_ID; - frame->u.ncid.seqnum = sid->seqnum; - frame->u.ncid.retire = 0; - frame->u.ncid.len = NGX_QUIC_SERVER_CID_LEN; - ngx_memcpy(frame->u.ncid.cid, id, NGX_QUIC_SERVER_CID_LEN); - - if (ngx_quic_new_sr_token(c, &dcid, qc->conf->sr_token_key, - frame->u.ncid.srt) - != NGX_OK) - { - return NGX_ERROR; - } - - ngx_quic_queue_frame(qc, frame); - } + /* we are no longer going to use this client id */ return NGX_OK; } -void -ngx_quic_clear_temp_server_ids(ngx_connection_t *c) -{ - ngx_queue_t *q, *next; - ngx_quic_server_id_t *sid; - ngx_quic_connection_t *qc; - - qc = ngx_quic_get_connection(c); - - ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic clear temp server ids"); - - for (q = ngx_queue_head(&qc->server_ids); - q != ngx_queue_sentinel(&qc->server_ids); - q = next) - { - next = ngx_queue_next(q); - sid = ngx_queue_data(q, ngx_quic_server_id_t, queue); - - if (sid->seqnum != NGX_QUIC_UNSET_PN) { - continue; - } - - ngx_queue_remove(q); - ngx_queue_insert_tail(&qc->free_server_ids, &sid->queue); - ngx_rbtree_delete(&c->listening->rbtree, &sid->udp.node); - qc->nserver_ids--; - } -} - - -static ngx_quic_server_id_t * -ngx_quic_insert_server_id(ngx_connection_t *c, ngx_quic_connection_t *qc, - ngx_str_t *id) -{ - ngx_str_t dcid; - ngx_quic_server_id_t *sid; - - sid = ngx_quic_alloc_server_id(c, qc); - if (sid == NULL) { - return NULL; - } - - sid->quic = qc; - - sid->seqnum = qc->server_seqnum; - - if (qc->server_seqnum != NGX_QUIC_UNSET_PN) { - qc->server_seqnum++; - } - - sid->len = id->len; - ngx_memcpy(sid->id, id->data, id->len); - - ngx_queue_insert_tail(&qc->server_ids, &sid->queue); - qc->nserver_ids++; - - dcid.data = sid->id; - dcid.len = sid->len; - - ngx_insert_udp_connection(c, &sid->udp, &dcid); - - ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic insert server id seqnum:%uL id len:%uz %xV", - sid->seqnum, id->len, id); - - return sid; -} - - static ngx_quic_client_id_t * ngx_quic_alloc_client_id(ngx_connection_t *c, ngx_quic_connection_t *qc) { @@ -505,28 +262,260 @@ ngx_quic_alloc_client_id(ngx_connection_ } -static ngx_quic_server_id_t * -ngx_quic_alloc_server_id(ngx_connection_t *c, ngx_quic_connection_t *qc) +ngx_quic_client_id_t * +ngx_quic_create_client_id(ngx_connection_t *c, ngx_str_t *id, + uint64_t seqnum, u_char *token) { - ngx_queue_t *q; - ngx_quic_server_id_t *sid; + ngx_quic_client_id_t *cid; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + cid = ngx_quic_alloc_client_id(c, qc); + if (cid == NULL) { + return NULL; + } - if (!ngx_queue_empty(&qc->free_server_ids)) { + cid->seqnum = seqnum; + + cid->len = id->len; + ngx_memcpy(cid->id, id->data, id->len); + + if (token) { + ngx_memcpy(cid->sr_token, token, NGX_QUIC_SR_TOKEN_LEN); + } + + ngx_queue_insert_tail(&qc->client_ids, &cid->queue); + qc->nclient_ids++; - q = ngx_queue_head(&qc->free_server_ids); - sid = ngx_queue_data(q, ngx_quic_server_id_t, queue); + if (seqnum > qc->client_seqnum) { + qc->client_seqnum = seqnum; + } - ngx_queue_remove(&sid->queue); + ngx_log_debug5(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic cid #%uL received id:%uz:%xV:%*xs", + cid->seqnum, id->len, id, + (size_t) NGX_QUIC_SR_TOKEN_LEN, cid->sr_token); + + return cid; +} + - ngx_memzero(sid, sizeof(ngx_quic_server_id_t)); +ngx_quic_client_id_t * +ngx_quic_next_client_id(ngx_connection_t *c) +{ + ngx_queue_t *q; + ngx_quic_client_id_t *cid; + ngx_quic_connection_t *qc; - } else { + qc = ngx_quic_get_connection(c); - sid = ngx_pcalloc(c->pool, sizeof(ngx_quic_server_id_t)); - if (sid == NULL) { - return NULL; + for (q = ngx_queue_head(&qc->client_ids); + q != ngx_queue_sentinel(&qc->client_ids); + q = ngx_queue_next(q)) + { + cid = ngx_queue_data(q, ngx_quic_client_id_t, queue); + + if (cid->refcnt == 0) { + return cid; } } - return sid; + return NULL; +} + + +ngx_int_t +ngx_quic_handle_retire_connection_id_frame(ngx_connection_t *c, + ngx_quic_header_t *pkt, ngx_quic_retire_cid_frame_t *f) +{ + ngx_quic_path_t *path; + ngx_quic_socket_t *qsock, **tmp; + ngx_quic_client_id_t *cid; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + qsock = ngx_quic_find_socket(c, f->sequence_number); + if (qsock == NULL) { + return NGX_OK; + } + + if (qsock->sid.seqnum == qc->socket->sid.seqnum) { + tmp = &qc->socket; + + } else if (qc->backup && qsock->sid.seqnum == qc->backup->sid.seqnum) { + tmp = &qc->backup; + + } else { + tmp = NULL; + } + + if (ngx_quic_create_sockets(c) != NGX_OK) { + return NGX_ERROR; + } + + if (tmp) { + /* replace socket in use (active or backup) */ + + ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic %s socket #%uL:%uL:%uL retired", + (*tmp) == qc->socket ? "active" : "backup", + (*tmp)->sid.seqnum, (*tmp)->cid->seqnum, + (*tmp)->path->seqnum); + + qsock = ngx_quic_get_unconnected_socket(c); + if (qsock == NULL) { + return NGX_ERROR; + } + + path = (*tmp)->path; + cid = (*tmp)->cid; + + ngx_quic_connect(c, qsock, path, cid); + + + ngx_log_debug5(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic %s socket is now #%uL:%uL:%uL (%s)", + (*tmp) == qc->socket ? "active" : "backup", + qsock->sid.seqnum, qsock->cid->seqnum, + qsock->path->seqnum, + ngx_quic_path_state_str(qsock->path)); + + ngx_quic_close_socket(c, *tmp); /* no longer used */ + + *tmp = qsock; + } + + return NGX_OK; } + + +ngx_int_t +ngx_quic_create_sockets(ngx_connection_t *c) +{ + ngx_uint_t n; + ngx_quic_socket_t *qsock; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + n = ngx_min(NGX_QUIC_MAX_SERVER_IDS, qc->ctp.active_connection_id_limit); + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic create sockets has:%ui max:%ui", qc->nsockets, n); + + while (qc->nsockets < n) { + + qsock = ngx_quic_alloc_socket(c, qc); + if (qsock == NULL) { + return NGX_ERROR; + } + + if (ngx_quic_listen(c, qc, qsock) != NGX_OK) { + return NGX_ERROR; + } + + if (ngx_quic_send_server_id(c, &qsock->sid) != NGX_OK) { + return NGX_ERROR; + } + } + + return NGX_OK; +} + + +static ngx_int_t +ngx_quic_send_server_id(ngx_connection_t *c, ngx_quic_server_id_t *sid) +{ + ngx_str_t dcid; + ngx_quic_frame_t *frame; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + dcid.len = sid->len; + dcid.data = sid->id; + + frame = ngx_quic_alloc_frame(c); + if (frame == NULL) { + return NGX_ERROR; + } + + frame->level = ssl_encryption_application; + frame->type = NGX_QUIC_FT_NEW_CONNECTION_ID; + frame->u.ncid.seqnum = sid->seqnum; + frame->u.ncid.retire = 0; + frame->u.ncid.len = NGX_QUIC_SERVER_CID_LEN; + ngx_memcpy(frame->u.ncid.cid, sid->id, NGX_QUIC_SERVER_CID_LEN); + + if (ngx_quic_new_sr_token(c, &dcid, qc->conf->sr_token_key, + frame->u.ncid.srt) + != NGX_OK) + { + return NGX_ERROR; + } + + ngx_quic_queue_frame(qc, frame); + + return NGX_OK; +} + + +static ngx_int_t +ngx_quic_replace_retired_client_id(ngx_connection_t *c, + ngx_quic_client_id_t *retired_cid) +{ + ngx_queue_t *q; + ngx_quic_socket_t *qsock; + ngx_quic_client_id_t *cid; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + for (q = ngx_queue_head(&qc->sockets); + q != ngx_queue_sentinel(&qc->sockets); + q = ngx_queue_next(q)) + { + qsock = ngx_queue_data(q, ngx_quic_socket_t, queue); + + if (qsock->cid == retired_cid) { + + cid = ngx_quic_next_client_id(c); + if (cid == NULL) { + return NGX_ERROR; + } + + qsock->cid = cid; + cid->refcnt++; + + ngx_quic_unref_client_id(c, retired_cid); + + if (retired_cid->refcnt == 0) { + return NGX_OK; + } + } + } + + return NGX_OK; +} + + +void +ngx_quic_unref_client_id(ngx_connection_t *c, ngx_quic_client_id_t *cid) +{ + ngx_quic_connection_t *qc; + + cid->refcnt--; + + if (cid->refcnt) { + return; + } + + qc = ngx_quic_get_connection(c); + + ngx_queue_remove(&cid->queue); + ngx_queue_insert_head(&qc->free_client_ids, &cid->queue); + + qc->nclient_ids--; +}
--- a/src/event/quic/ngx_event_quic_connid.h +++ b/src/event/quic/ngx_event_quic_connid.h @@ -12,14 +12,17 @@ #include <ngx_core.h> -ngx_int_t ngx_quic_setup_connection_ids(ngx_connection_t *c, - ngx_quic_connection_t *qc, ngx_quic_header_t *pkt); -void ngx_quic_clear_temp_server_ids(ngx_connection_t *c); -ngx_int_t ngx_quic_issue_server_ids(ngx_connection_t *c); - ngx_int_t ngx_quic_handle_retire_connection_id_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_retire_cid_frame_t *f); ngx_int_t ngx_quic_handle_new_connection_id_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_new_conn_id_frame_t *f); +ngx_int_t ngx_quic_create_sockets(ngx_connection_t *c); +ngx_int_t ngx_quic_create_server_id(ngx_connection_t *c, u_char *id); + +ngx_quic_client_id_t *ngx_quic_create_client_id(ngx_connection_t *c, + ngx_str_t *id, uint64_t seqnum, u_char *token); +ngx_quic_client_id_t *ngx_quic_next_client_id(ngx_connection_t *c); +void ngx_quic_unref_client_id(ngx_connection_t *c, ngx_quic_client_id_t *cid); + #endif /* _NGX_EVENT_QUIC_CONNID_H_INCLUDED_ */
--- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -10,25 +10,71 @@ #include <ngx_event_quic_connection.h> +static void ngx_quic_set_connection_path(ngx_connection_t *c, + ngx_quic_path_t *path); +static ngx_int_t ngx_quic_validate_path(ngx_connection_t *c, + ngx_quic_socket_t *qsock); +static ngx_int_t ngx_quic_send_path_challenge(ngx_connection_t *c, + ngx_quic_path_t *path); +static ngx_int_t ngx_quic_path_restore(ngx_connection_t *c); +static ngx_quic_path_t *ngx_quic_alloc_path(ngx_connection_t *c); + + ngx_int_t ngx_quic_handle_path_challenge_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_path_challenge_frame_t *f) { - ngx_quic_frame_t *frame; + off_t max, pad; + ssize_t sent; + ngx_quic_path_t *path; + ngx_quic_frame_t frame, *fp; + ngx_quic_socket_t *qsock; ngx_quic_connection_t *qc; qc = ngx_quic_get_connection(c); - frame = ngx_quic_alloc_frame(c); - if (frame == NULL) { + frame.level = pkt->level; + frame.type = NGX_QUIC_FT_PATH_RESPONSE; + frame.u.path_response = *f; + + /* + * A PATH_RESPONSE frame MUST be sent on the network path where the + * PATH_CHALLENGE was received. + */ + qsock = ngx_quic_get_socket(c); + path = qsock->path; + + /* + * An endpoint MUST NOT expand the datagram containing the PATH_RESPONSE + * if the resulting data exceeds the anti-amplification limit. + */ + max = path->received * 3; + max = (path->sent >= max) ? 0 : max - path->sent; + pad = ngx_min(1200, max); + + sent = ngx_quic_frame_sendto(c, &frame, pad, path->sockaddr, path->socklen); + if (sent == -1) { return NGX_ERROR; } - frame->level = pkt->level; - frame->type = NGX_QUIC_FT_PATH_RESPONSE; - frame->u.path_response = *f; + path->sent += sent; + + if (qsock == qc->socket) { + /* + * An endpoint that receives a PATH_CHALLENGE on an active path SHOULD + * send a non-probing packet in response. + */ - ngx_quic_queue_frame(qc, frame); + fp = ngx_quic_alloc_frame(c); + if (fp == NULL) { + return NGX_ERROR; + } + + fp->level = pkt->level; + fp->type = NGX_QUIC_FT_PING; + + ngx_quic_queue_frame(qc, fp); + } return NGX_OK; } @@ -38,7 +84,648 @@ ngx_int_t ngx_quic_handle_path_response_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_path_challenge_frame_t *f) { - /* TODO */ + ngx_queue_t *q; + ngx_quic_path_t *path, *prev; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + /* + * A PATH_RESPONSE frame received on any network path validates the path + * on which the PATH_CHALLENGE was sent. + */ + + for (q = ngx_queue_head(&qc->paths); + q != ngx_queue_sentinel(&qc->paths); + q = ngx_queue_next(q)) + { + path = ngx_queue_data(q, ngx_quic_path_t, queue); + + if (path->state != NGX_QUIC_PATH_VALIDATING) { + continue; + } + + if (ngx_memcmp(path->challenge1, f->data, sizeof(f->data)) == 0 + || ngx_memcmp(path->challenge2, f->data, sizeof(f->data)) == 0) + { + goto valid; + } + } + + ngx_log_error(NGX_LOG_INFO, c->log, 0, + "quic stale PATH_RESPONSE ignored"); + + return NGX_OK; + +valid: + + /* + * On confirming a peer's ownership of its new address, + * an endpoint MUST immediately reset the congestion controller + * and round-trip time estimator for the new path + * to initial values + * ...unless the only change in the peer's address is its port number. + */ + + prev = qc->backup->path; + + if (ngx_cmp_sockaddr(prev->sockaddr, prev->socklen, + path->sockaddr, path->socklen, 0) + != NGX_OK) + { + /* address has changed */ + ngx_memzero(&qc->congestion, sizeof(ngx_quic_congestion_t)); + + qc->congestion.window = ngx_min(10 * qc->tp.max_udp_payload_size, + ngx_max(2 * qc->tp.max_udp_payload_size, + 14720)); + qc->congestion.ssthresh = (size_t) -1; + qc->congestion.recovery_start = ngx_current_msec; + } + + /* + * After verifying a new client address, the server SHOULD + * send new address validation tokens (Section 8) to the client. + */ + + if (ngx_quic_send_new_token(c, path) != NGX_OK) { + return NGX_ERROR; + } + + ngx_log_error(NGX_LOG_INFO, c->log, 0, + "quic path #%uL successfully validated", path->seqnum); + + path->state = NGX_QUIC_PATH_VALIDATED; + path->validated_at = ngx_time(); + + return NGX_OK; +} + + +static ngx_quic_path_t * +ngx_quic_alloc_path(ngx_connection_t *c) +{ + ngx_queue_t *q; + struct sockaddr *sa; + ngx_quic_path_t *path; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + if (!ngx_queue_empty(&qc->free_paths)) { + + q = ngx_queue_head(&qc->free_paths); + path = ngx_queue_data(q, ngx_quic_path_t, queue); + + ngx_queue_remove(&path->queue); + + sa = path->sockaddr; + ngx_memzero(path, sizeof(ngx_quic_path_t)); + path->sockaddr = sa; + + } else { + + path = ngx_pcalloc(c->pool, sizeof(ngx_quic_path_t)); + if (path == NULL) { + return NULL; + } + + path->sockaddr = ngx_palloc(c->pool, NGX_SOCKADDRLEN); + if (path->sockaddr == NULL) { + return NULL; + } + } + + return path; +} + + +ngx_quic_path_t * +ngx_quic_add_path(ngx_connection_t *c, struct sockaddr *sockaddr, + socklen_t socklen) +{ + ngx_quic_path_t *path; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + path = ngx_quic_alloc_path(c); + if (path == NULL) { + return NULL; + } + + path->seqnum = qc->path_seqnum++; + + path->socklen = socklen; + ngx_memcpy(path->sockaddr, sockaddr, socklen); + + path->addr_text.data = path->text; + path->addr_text.len = ngx_sock_ntop(sockaddr, socklen, path->text, + NGX_SOCKADDR_STRLEN, 1); + + ngx_queue_insert_tail(&qc->paths, &path->queue); + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic path #%uL created src:%V", + path->seqnum, &path->addr_text); + + return path; +} + + +ngx_quic_path_t * +ngx_quic_find_path(ngx_connection_t *c, struct sockaddr *sockaddr, + socklen_t socklen) +{ + ngx_queue_t *q; + ngx_quic_path_t *path; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + for (q = ngx_queue_head(&qc->paths); + q != ngx_queue_sentinel(&qc->paths); + q = ngx_queue_next(q)) + { + path = ngx_queue_data(q, ngx_quic_path_t, queue); + + if (ngx_cmp_sockaddr(sockaddr, socklen, + path->sockaddr, path->socklen, 1) + == NGX_OK) + { + return path; + } + } + + return NULL; +} + + +ngx_int_t +ngx_quic_check_migration(ngx_connection_t *c, ngx_quic_header_t *pkt) +{ + ngx_quic_path_t *path; + ngx_quic_socket_t *qsock; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + qsock = ngx_quic_get_socket(c); + + if (c->udp->dgram == NULL) { + /* 2nd QUIC packet in first UDP datagram */ + return NGX_OK; + } + + path = ngx_quic_find_path(c, c->udp->dgram->sockaddr, + c->udp->dgram->socklen); + if (path == NULL) { + /* packet comes from unknown path, possibly migration */ + + if (qc->tp.disable_active_migration) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic migration disabled, dropping packet " + "from unknown path"); + return NGX_DECLINED; + } + + if (pkt->level != ssl_encryption_application) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic too early migration attempt"); + return NGX_DECLINED; + } + + return NGX_OK; + } + + /* packet from known path */ + + if (qsock->path == NULL) { + /* client switched to previously unused server id */ + return NGX_OK; + } + + if (path == qsock->path) { + /* regular packet to expected path */ + return NGX_OK; + } + + /* client is trying to use server id already used on other path */ + + ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic attempt to use socket #%uL:%uL:%uL with path #%uL", + qsock->sid.seqnum, qsock->cid->seqnum, + qsock->path->seqnum, path->seqnum); + + return NGX_DECLINED; +} + + +ngx_int_t +ngx_quic_update_paths(ngx_connection_t *c, ngx_quic_header_t *pkt) +{ + off_t len; + ngx_quic_path_t *path; + ngx_quic_socket_t *qsock; + ngx_quic_client_id_t *cid; + ngx_quic_connection_t *qc; + + qsock = ngx_quic_get_socket(c); + path = qsock->path; + + if (path) { + goto update; + } + + path = ngx_quic_find_path(c, c->udp->dgram->sockaddr, + c->udp->dgram->socklen); + + if (path == NULL) { + path = ngx_quic_add_path(c, c->udp->dgram->sockaddr, + c->udp->dgram->socklen); + if (path == NULL) { + return NGX_ERROR; + } + } + + cid = ngx_quic_next_client_id(c); + if (cid == NULL) { + qc = ngx_quic_get_connection(c); + qc->error = NGX_QUIC_ERR_CONNECTION_ID_LIMIT_ERROR; + qc->error_reason = "no available client ids for new path"; + + ngx_log_error(NGX_LOG_ERR, c->log, 0, + "no available client ids for new path"); + + return NGX_ERROR; + } + + ngx_quic_connect(c, qsock, path, cid); + +update: + + if (pkt->raw->start == pkt->data) { + len = pkt->raw->last - pkt->raw->start; + + } else { + len = 0; + } + + /* TODO: this may be too late in some cases; + * for example, if error happens during decrypt(), we cannot + * send CC, if error happens in 1st packet, due to amplification + * limit, because path->received = 0 + * + * should we account garbage as received or only decrypting packets? + */ + path->received += len; + + ngx_log_debug6(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic packet via #%uL:%uL:%uL" + " size:%O path recvd:%O sent:%O", + qsock->sid.seqnum, qsock->cid->seqnum, path->seqnum, + len, path->received, path->sent); + return NGX_OK; } + +static void +ngx_quic_set_connection_path(ngx_connection_t *c, ngx_quic_path_t *path) +{ + size_t len; + + ngx_memcpy(c->sockaddr, path->sockaddr, path->socklen); + c->socklen = path->socklen; + + if (c->addr_text.data) { + len = ngx_min(c->addr_text.len, path->addr_text.len); + + ngx_memcpy(c->addr_text.data, path->addr_text.data, len); + c->addr_text.len = len; + } + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic send path set to #%uL addr:%V", + path->seqnum, &path->addr_text); +} + + +ngx_int_t +ngx_quic_handle_migration(ngx_connection_t *c, ngx_quic_header_t *pkt) +{ + ngx_quic_path_t *next; + ngx_quic_socket_t *qsock; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; + + /* got non-probing packet via non-active socket with different path */ + + qc = ngx_quic_get_connection(c); + + /* current socket, different from active */ + qsock = ngx_quic_get_socket(c); + + next = qsock->path; /* going to migrate to this path... */ + + ngx_log_error(NGX_LOG_INFO, c->log, 0, + "quic migration from #%uL:%uL:%uL (%s)" + " to #%uL:%uL:%uL (%s)", + qc->socket->sid.seqnum, qc->socket->cid->seqnum, + qc->socket->path->seqnum, + ngx_quic_path_state_str(qc->socket->path), + qsock->sid.seqnum, qsock->cid->seqnum, next->seqnum, + ngx_quic_path_state_str(next)); + + switch (next->state) { + case NGX_QUIC_PATH_NEW: + if (ngx_quic_validate_path(c, qsock) != NGX_OK) { + return NGX_ERROR; + } + break; + + /* migration to previously known path */ + + case NGX_QUIC_PATH_VALIDATING: + /* alredy validating, nothing to do */ + break; + + case NGX_QUIC_PATH_VALIDATED: + /* if path is old enough, revalidate */ + if (ngx_time() - next->validated_at > NGX_QUIC_PATH_VALID_TIME) { + + next->state = NGX_QUIC_PATH_NEW; + + if (ngx_quic_validate_path(c, qsock) != NGX_OK) { + return NGX_ERROR; + } + } + + break; + } + + ctx = ngx_quic_get_send_ctx(qc, pkt->level); + + /* + * An endpoint only changes the address to which it sends packets in + * response to the highest-numbered non-probing packet. + */ + if (pkt->pn != ctx->largest_pn) { + return NGX_OK; + } + + /* switching connection to new path */ + + ngx_quic_set_connection_path(c, next); + + /* + * An endpoint MUST NOT reuse a connection ID when sending to + * more than one destination address. + */ + + /* preserve valid path we are migrating from */ + if (qc->socket->path->state == NGX_QUIC_PATH_VALIDATED) { + + if (qc->backup) { + ngx_quic_close_socket(c, qc->backup); + } + + qc->backup = qc->socket; + + ngx_log_error(NGX_LOG_INFO, c->log, 0, + "quic backup socket is now #%uL:%uL:%uL (%s)", + qc->backup->sid.seqnum, qc->backup->cid->seqnum, + qc->backup->path->seqnum, + ngx_quic_path_state_str(qc->backup->path)); + } + + qc->socket = qsock; + + ngx_log_error(NGX_LOG_INFO, c->log, 0, + "quic active socket is now #%uL:%uL:%uL (%s)", + qsock->sid.seqnum, qsock->cid->seqnum, + qsock->path->seqnum, ngx_quic_path_state_str(qsock->path)); + + return NGX_OK; +} + + +static ngx_int_t +ngx_quic_validate_path(ngx_connection_t *c, ngx_quic_socket_t *qsock) +{ + ngx_msec_t pto; + ngx_quic_path_t *path; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + path = qsock->path; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic initiated validation of new path #%uL", + path->seqnum); + + path->state = NGX_QUIC_PATH_VALIDATING; + + if (RAND_bytes(path->challenge1, 8) != 1) { + return NGX_ERROR; + } + + if (RAND_bytes(path->challenge2, 8) != 1) { + return NGX_ERROR; + } + + if (ngx_quic_send_path_challenge(c, path) != NGX_OK) { + return NGX_ERROR; + } + + ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); + pto = ngx_quic_pto(c, ctx); + + path->expires = ngx_current_msec + pto; + path->tries = NGX_QUIC_PATH_RETRIES; + + if (!qc->path_validation.timer_set) { + ngx_add_timer(&qc->path_validation, pto); + } + + + return NGX_OK; +} + + +static ngx_int_t +ngx_quic_send_path_challenge(ngx_connection_t *c, ngx_quic_path_t *path) +{ + off_t max, pad; + ssize_t sent; + ngx_quic_frame_t frame; + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic path #%uL send path challenge tries:%ui", + path->seqnum, path->tries); + + frame.level = ssl_encryption_application; + frame.type = NGX_QUIC_FT_PATH_CHALLENGE; + + ngx_memcpy(frame.u.path_challenge.data, path->challenge1, 8); + + /* + * An endpoint MUST expand datagrams that contain a PATH_CHALLENGE frame + * to at least the smallest allowed maximum datagram size of 1200 bytes, + * unless the anti-amplification limit for the path does not permit + * sending a datagram of this size. + */ + + /* same applies to PATH_RESPONSE frames */ + + max = path->received * 3; + max = (path->sent >= max) ? 0 : max - path->sent; + pad = ngx_min(1200, max); + + sent = ngx_quic_frame_sendto(c, &frame, pad, path->sockaddr, path->socklen); + if (sent == -1) { + return NGX_ERROR; + } + + path->sent += sent; + + ngx_memcpy(frame.u.path_challenge.data, path->challenge2, 8); + + max = (path->sent >= max) ? 0 : max - path->sent; + pad = ngx_min(1200, max); + + sent = ngx_quic_frame_sendto(c, &frame, pad, path->sockaddr, path->socklen); + if (sent == -1) { + return NGX_ERROR; + } + + path->sent += sent; + + return NGX_OK; +} + + +void +ngx_quic_path_validation_handler(ngx_event_t *ev) +{ + ngx_msec_t now; + ngx_queue_t *q; + ngx_msec_int_t left, next, pto; + ngx_quic_path_t *path; + ngx_connection_t *c; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; + + c = ev->data; + qc = ngx_quic_get_connection(c); + + ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); + pto = ngx_quic_pto(c, ctx); + + next = -1; + now = ngx_current_msec; + + for (q = ngx_queue_head(&qc->paths); + q != ngx_queue_sentinel(&qc->paths); + q = ngx_queue_next(q)) + { + path = ngx_queue_data(q, ngx_quic_path_t, queue); + + if (path->state != NGX_QUIC_PATH_VALIDATING) { + continue; + } + + left = path->expires - now; + + if (left > 0) { + + if (next == -1 || left < next) { + next = path->expires; + } + + continue; + } + + if (--path->tries) { + path->expires = ngx_current_msec + pto; + + if (next == -1 || pto < next) { + next = pto; + } + + /* retransmit */ + (void) ngx_quic_send_path_challenge(c, path); + + continue; + } + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0, + "quic path #%uL validation failed", path->seqnum); + + /* found expired path */ + + path->state = NGX_QUIC_PATH_NEW; + + /* + * If the timer fires before the PATH_RESPONSE is received, the + * endpoint might send a new PATH_CHALLENGE, and restart the timer for + * a longer period of time. This timer SHOULD be set as described in + * Section 6.2.1 of [QUIC-RECOVERY] and MUST NOT be more aggressive. + */ + + if (qc->socket->path != path) { + /* the path was not actually used */ + continue; + } + + if (ngx_quic_path_restore(c) != NGX_OK) { + qc->error = NGX_QUIC_ERR_NO_VIABLE_PATH; + qc->error_reason = "no viable path"; + ngx_quic_close_connection(c, NGX_ERROR); + return; + } + } + + if (next != -1) { + ngx_add_timer(&qc->path_validation, next); + } +} + + +static ngx_int_t +ngx_quic_path_restore(ngx_connection_t *c) +{ + ngx_quic_socket_t *qsock; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + /* Failure to validate a path does not cause the connection to end */ + + /* + * To protect the connection from failing due to such a spurious + * migration, an endpoint MUST revert to using the last validated + * peer address when validation of a new peer address fails. + */ + + if (qc->backup == NULL) { + return NGX_ERROR; + } + + qc->socket = qc->backup; + qc->backup = NULL; + + qsock = qc->socket; + + ngx_log_error(NGX_LOG_INFO, c->log, 0, + "quic active socket is restored to #%uL:%uL:%uL" + " (%s), no backup", + qsock->sid.seqnum, qsock->cid->seqnum, qsock->path->seqnum, + ngx_quic_path_state_str(qsock->path)); + + ngx_quic_set_connection_path(c, qsock->path); + + return NGX_OK; +}
--- a/src/event/quic/ngx_event_quic_migration.h +++ b/src/event/quic/ngx_event_quic_migration.h @@ -11,10 +11,34 @@ #include <ngx_config.h> #include <ngx_core.h> +#define NGX_QUIC_PATH_RETRIES 3 + +#define NGX_QUIC_PATH_NEW 0 +#define NGX_QUIC_PATH_VALIDATING 1 +#define NGX_QUIC_PATH_VALIDATED 2 + +#define NGX_QUIC_PATH_VALID_TIME 600 /* seconds */ + + +#define ngx_quic_path_state_str(p) \ + ((p)->state == NGX_QUIC_PATH_NEW) ? "new" : \ + (((p)->state == NGX_QUIC_PATH_VALIDATED) ? "validated" : "validating") + ngx_int_t ngx_quic_handle_path_challenge_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_path_challenge_frame_t *f); ngx_int_t ngx_quic_handle_path_response_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_path_challenge_frame_t *f); +ngx_quic_path_t *ngx_quic_add_path(ngx_connection_t *c, + struct sockaddr *sockaddr, socklen_t socklen); + +ngx_int_t ngx_quic_check_migration(ngx_connection_t *c, + ngx_quic_header_t *pkt); +ngx_int_t ngx_quic_update_paths(ngx_connection_t *c, ngx_quic_header_t *pkt); +ngx_int_t ngx_quic_handle_migration(ngx_connection_t *c, + ngx_quic_header_t *pkt); + +void ngx_quic_path_validation_handler(ngx_event_t *ev); + #endif /* _NGX_EVENT_QUIC_MIGRATION_H_INCLUDED_ */
--- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -35,10 +35,14 @@ #define NGX_QUIC_CC_MIN_INTERVAL 1000 /* 1s */ +static ngx_int_t ngx_quic_socket_output(ngx_connection_t *c, + ngx_quic_socket_t *qsock); static ssize_t ngx_quic_output_packet(ngx_connection_t *c, - ngx_quic_send_ctx_t *ctx, u_char *data, size_t max, size_t min); + ngx_quic_send_ctx_t *ctx, u_char *data, size_t max, size_t min, + ngx_quic_socket_t *qsock); static ngx_uint_t ngx_quic_get_padding_level(ngx_connection_t *c); -static ssize_t ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len); +static ssize_t ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len, + struct sockaddr *sockaddr, socklen_t socklen); static void ngx_quic_set_packet_number(ngx_quic_header_t *pkt, ngx_quic_send_ctx_t *ctx); @@ -61,11 +65,29 @@ ngx_quic_max_udp_payload(ngx_connection_ ngx_int_t ngx_quic_output(ngx_connection_t *c) { + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + if (ngx_quic_socket_output(c, qc->socket) != NGX_OK) { + return NGX_ERROR; + } + + ngx_quic_set_lost_timer(c); + + return NGX_OK; +} + + +static ngx_int_t +ngx_quic_socket_output(ngx_connection_t *c, ngx_quic_socket_t *qsock) +{ off_t max; size_t len, min, in_flight; ssize_t n; u_char *p; ngx_uint_t i, pad; + ngx_quic_path_t *path; ngx_quic_send_ctx_t *ctx; ngx_quic_congestion_t *cg; ngx_quic_connection_t *qc; @@ -78,15 +100,18 @@ ngx_quic_output(ngx_connection_t *c) in_flight = cg->in_flight; + path = qsock->path; + for ( ;; ) { p = dst; len = ngx_min(qc->ctp.max_udp_payload_size, NGX_QUIC_MAX_UDP_PAYLOAD_SIZE); - if (!qc->validated) { - max = qc->received * 3; - max = (c->sent >= max) ? 0 : max - c->sent; + if (path->state != NGX_QUIC_PATH_VALIDATED) { + max = path->received * 3; + max = (path->sent >= max) ? 0 : max - path->sent; + len = ngx_min(len, (size_t) max); } @@ -103,7 +128,7 @@ ngx_quic_output(ngx_connection_t *c) min = (i == pad && p - dst < NGX_QUIC_MIN_INITIAL_SIZE) ? NGX_QUIC_MIN_INITIAL_SIZE - (p - dst) : 0; - n = ngx_quic_output_packet(c, ctx, p, len, min); + n = ngx_quic_output_packet(c, ctx, p, len, min, qsock); if (n == NGX_ERROR) { return NGX_ERROR; } @@ -117,10 +142,13 @@ ngx_quic_output(ngx_connection_t *c) break; } - n = ngx_quic_send(c, dst, len); + n = ngx_quic_send(c, dst, len, path->sockaddr, path->socklen); + if (n == NGX_ERROR) { return NGX_ERROR; } + + path->sent += len; } if (in_flight != cg->in_flight && !qc->send_timer_set && !qc->closing) { @@ -128,7 +156,6 @@ ngx_quic_output(ngx_connection_t *c) ngx_add_timer(c->read, qc->tp.max_idle_timeout); } - ngx_quic_set_lost_timer(c); return NGX_OK; } @@ -176,14 +203,14 @@ ngx_quic_get_padding_level(ngx_connectio static ssize_t ngx_quic_output_packet(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, - u_char *data, size_t max, size_t min) + u_char *data, size_t max, size_t min, ngx_quic_socket_t *qsock) { size_t len, hlen, pad_len; u_char *p; ssize_t flen; ngx_str_t out, res; ngx_int_t rc; - ngx_uint_t nframes; + ngx_uint_t nframes, has_pr; ngx_msec_t now; ngx_queue_t *q; ngx_quic_frame_t *f; @@ -196,9 +223,10 @@ ngx_quic_output_packet(ngx_connection_t return 0; } - ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic output %s packet max:%uz min:%uz", - ngx_quic_level_name(ctx->level), max, min); + ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic output sock #%uL %s packet max:%uz min:%uz", + qsock->sid.seqnum, ngx_quic_level_name(ctx->level), + max, min); qc = ngx_quic_get_connection(c); cg = &qc->congestion; @@ -208,7 +236,7 @@ ngx_quic_output_packet(ngx_connection_t : NGX_QUIC_MAX_LONG_HEADER; hlen += EVP_GCM_TLS_TAG_LEN; - hlen -= NGX_QUIC_MAX_CID_LEN - qc->scid.len; + hlen -= NGX_QUIC_MAX_CID_LEN - qsock->cid->len; ngx_memzero(&pkt, sizeof(ngx_quic_header_t)); @@ -216,6 +244,7 @@ ngx_quic_output_packet(ngx_connection_t nframes = 0; p = src; len = 0; + has_pr = 0; for (q = ngx_queue_head(&ctx->frames); q != ngx_queue_sentinel(&ctx->frames); @@ -227,6 +256,12 @@ ngx_quic_output_packet(ngx_connection_t max = cg->window; } + if (f->type == NGX_QUIC_FT_PATH_RESPONSE + || f->type == NGX_QUIC_FT_PATH_CHALLENGE) + { + has_pr = 1; + } + if (hlen + len >= max) { break; } @@ -296,15 +331,33 @@ ngx_quic_output_packet(ngx_connection_t pkt.version = qc->version; pkt.log = c->log; pkt.level = ctx->level; - pkt.dcid = qc->scid; - pkt.scid = qc->dcid; + + pkt.dcid.data = qsock->cid->id; + pkt.dcid.len = qsock->cid->len; + + pkt.scid.data = qsock->sid.id; + pkt.scid.len = qsock->sid.len; pad_len = 4; - if (min) { + if (min || has_pr) { hlen = EVP_GCM_TLS_TAG_LEN + ngx_quic_create_header(&pkt, NULL, out.len, NULL); + /* + * An endpoint MUST expand datagrams that contain a + * PATH_CHALLENGE frame to at least the smallest allowed + * maximum datagram size of 1200 bytes, unless the + * anti-amplification limit for the path does not permit + * sending a datagram of this size. + * + * (same applies to PATH_RESPONSE frames) + */ + + if (has_pr) { + min = ngx_max(1200, min); + } + if (min > hlen + pad_len) { pad_len = min - hlen; } @@ -364,11 +417,14 @@ ngx_quic_output_packet(ngx_connection_t } -ssize_t -ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len) +static ssize_t +ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len, + struct sockaddr *sockaddr, socklen_t socklen) { - ngx_buf_t b; - ngx_chain_t cl, *res; + ngx_buf_t b; + socklen_t orig_socklen; + ngx_chain_t cl, *res; + struct sockaddr *orig_sockaddr; ngx_memzero(&b, sizeof(ngx_buf_t)); @@ -380,7 +436,17 @@ ngx_quic_send(ngx_connection_t *c, u_cha cl.buf = &b; cl.next= NULL; + orig_socklen = c->socklen; + orig_sockaddr = c->sockaddr; + + c->sockaddr = sockaddr; + c->socklen = socklen; + res = c->send_chain(c, &cl, 0); + + c->sockaddr = orig_sockaddr; + c->socklen = orig_socklen; + if (res == NGX_CHAIN_ERROR) { return NGX_ERROR; } @@ -441,7 +507,7 @@ ngx_quic_negotiate_version(ngx_connectio "quic vnego packet to send len:%uz %*xs", len, len, buf); #endif - (void) ngx_quic_send(c, buf, len); + (void) ngx_quic_send(c, buf, len, c->sockaddr, c->socklen); return NGX_ERROR; } @@ -524,7 +590,7 @@ ngx_quic_send_stateless_reset(ngx_connec return NGX_ERROR; } - (void) ngx_quic_send(c, buf, len); + (void) ngx_quic_send(c, buf, len, c->sockaddr, c->socklen); return NGX_DECLINED; } @@ -642,7 +708,9 @@ ngx_quic_send_early_cc(ngx_connection_t return NGX_ERROR; } - if (ngx_quic_send(c, res.data, res.len) == NGX_ERROR) { + if (ngx_quic_send(c, res.data, res.len, c->sockaddr, c->socklen) + == NGX_ERROR) + { return NGX_ERROR; } @@ -664,8 +732,8 @@ ngx_quic_send_retry(ngx_connection_t *c, expires = ngx_time() + NGX_QUIC_RETRY_TOKEN_LIFETIME; - if (ngx_quic_new_token(c, conf->av_token_key, &token, &inpkt->dcid, - expires, 1) + if (ngx_quic_new_token(c, c->sockaddr, c->socklen, conf->av_token_key, + &token, &inpkt->dcid, expires, 1) != NGX_OK) { return NGX_ERROR; @@ -700,7 +768,7 @@ ngx_quic_send_retry(ngx_connection_t *c, "quic packet to send len:%uz %xV", res.len, &res); #endif - len = ngx_quic_send(c, res.data, res.len); + len = ngx_quic_send(c, res.data, res.len, c->sockaddr, c->socklen); if (len == NGX_ERROR) { return NGX_ERROR; } @@ -718,7 +786,7 @@ ngx_quic_send_retry(ngx_connection_t *c, ngx_int_t -ngx_quic_send_new_token(ngx_connection_t *c) +ngx_quic_send_new_token(ngx_connection_t *c, ngx_quic_path_t *path) { time_t expires; ngx_str_t token; @@ -727,13 +795,10 @@ ngx_quic_send_new_token(ngx_connection_t qc = ngx_quic_get_connection(c); - if (!qc->conf->retry) { - return NGX_OK; - } - expires = ngx_time() + NGX_QUIC_NEW_TOKEN_LIFETIME; - if (ngx_quic_new_token(c, qc->conf->av_token_key, &token, NULL, expires, 0) + if (ngx_quic_new_token(c, path->sockaddr, path->socklen, + qc->conf->av_token_key, &token, NULL, expires, 0) != NGX_OK) { return NGX_ERROR; @@ -849,3 +914,75 @@ ngx_quic_send_ack_range(ngx_connection_t return NGX_OK; } + + +ssize_t +ngx_quic_frame_sendto(ngx_connection_t *c, ngx_quic_frame_t *frame, + size_t min, struct sockaddr *sockaddr, socklen_t socklen) +{ + ssize_t len; + ngx_str_t res; + ngx_quic_header_t pkt; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; + + static u_char src[NGX_QUIC_MAX_UDP_PAYLOAD_SIZE]; + static u_char dst[NGX_QUIC_MAX_UDP_PAYLOAD_SIZE]; + + qc = ngx_quic_get_connection(c); + + ngx_memzero(&pkt, sizeof(ngx_quic_header_t)); + + len = ngx_quic_create_frame(NULL, frame); + if (len > NGX_QUIC_MAX_UDP_PAYLOAD_SIZE) { + return -1; + } + + ngx_quic_log_frame(c->log, frame, 1); + + len = ngx_quic_create_frame(src, frame); + if (len == -1) { + return -1; + } + + if (len < (ssize_t) min) { + ngx_memset(src + len, NGX_QUIC_FT_PADDING, min - len); + len = min; + } + + pkt.keys = qc->keys; + pkt.flags = NGX_QUIC_PKT_FIXED_BIT; + + if (qc->key_phase) { + pkt.flags |= NGX_QUIC_PKT_KPHASE; + } + + ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); + + ngx_quic_set_packet_number(&pkt, ctx); + + pkt.version = qc->version; + pkt.log = c->log; + pkt.level = ctx->level; + + pkt.dcid.data = qc->socket->cid->id; + pkt.dcid.len = qc->socket->cid->len; + + pkt.scid.data = qc->socket->sid.id; + pkt.scid.len = qc->socket->sid.len; + + pkt.payload.data = src; + pkt.payload.len = len; + + res.data = dst; + + if (ngx_quic_encrypt(&pkt, &res) != NGX_OK) { + return -1; + } + + ctx->pnum++; + + len = ngx_quic_send(c, res.data, res.len, sockaddr, socklen); + + return len; +}
--- a/src/event/quic/ngx_event_quic_output.h +++ b/src/event/quic/ngx_event_quic_output.h @@ -30,11 +30,14 @@ ngx_int_t ngx_quic_send_early_cc(ngx_con ngx_int_t ngx_quic_send_retry(ngx_connection_t *c, ngx_quic_conf_t *conf, ngx_quic_header_t *pkt); -ngx_int_t ngx_quic_send_new_token(ngx_connection_t *c); +ngx_int_t ngx_quic_send_new_token(ngx_connection_t *c, ngx_quic_path_t *path); ngx_int_t ngx_quic_send_ack(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx); ngx_int_t ngx_quic_send_ack_range(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, uint64_t smallest, uint64_t largest); +ssize_t ngx_quic_frame_sendto(ngx_connection_t *c, ngx_quic_frame_t *frame, + size_t min, struct sockaddr *sockaddr, socklen_t socklen); + #endif /* _NGX_EVENT_QUIC_OUTPUT_H_INCLUDED_ */
new file mode 100644 --- /dev/null +++ b/src/event/quic/ngx_event_quic_socket.c @@ -0,0 +1,355 @@ + +/* + * Copyright (C) Nginx, Inc. + */ + + +#include <ngx_config.h> +#include <ngx_core.h> +#include <ngx_event.h> +#include <ngx_event_quic_connection.h> + + +static ngx_int_t ngx_quic_create_temp_socket(ngx_connection_t *c, + ngx_quic_connection_t *qc, ngx_str_t *dcid, ngx_quic_path_t *path, + ngx_quic_client_id_t *cid); + +static void ngx_quic_unref_path(ngx_connection_t *c, ngx_quic_path_t *path); + + +ngx_int_t +ngx_quic_open_sockets(ngx_connection_t *c, ngx_quic_connection_t *qc, + ngx_quic_header_t *pkt) +{ + ngx_quic_path_t *path; + ngx_quic_socket_t *qsock; + ngx_quic_client_id_t *cid; + + /* + * qc->nclient_ids = 0 + * qc->nsockets = 0 + * qc->max_retired_seqnum = 0 + * qc->client_seqnum = 0 + */ + + ngx_queue_init(&qc->sockets); + ngx_queue_init(&qc->free_sockets); + + ngx_queue_init(&qc->paths); + ngx_queue_init(&qc->free_paths); + + ngx_queue_init(&qc->client_ids); + ngx_queue_init(&qc->free_client_ids); + + qc->tp.original_dcid.len = pkt->odcid.len; + qc->tp.original_dcid.data = ngx_pstrdup(c->pool, &pkt->odcid); + if (qc->tp.original_dcid.data == NULL) { + return NGX_ERROR; + } + + /* socket to use for further processing */ + qsock = ngx_quic_alloc_socket(c, qc); + if (qsock == NULL) { + return NGX_ERROR; + } + + /* socket is listening at new server id (autogenerated) */ + if (ngx_quic_listen(c, qc, qsock) != NGX_OK) { + return NGX_ERROR; + } + + qc->tp.initial_scid.len = qsock->sid.len; + qc->tp.initial_scid.data = ngx_pnalloc(c->pool, qsock->sid.len); + if (qc->tp.initial_scid.data == NULL) { + goto failed; + } + ngx_memcpy(qc->tp.initial_scid.data, qsock->sid.id, qsock->sid.len); + + /* for all packets except first, this is set at udp layer */ + c->udp = &qsock->udp; + + /* ngx_quic_get_connection(c) macro is now usable */ + + /* we have a client identified by scid */ + cid = ngx_quic_create_client_id(c, &pkt->scid, 0, NULL); + if (cid == NULL) { + goto failed; + } + + /* the client arrived from this path */ + path = ngx_quic_add_path(c, c->sockaddr, c->socklen); + if (path == NULL) { + goto failed; + } + + if (pkt->validated) { + path->state = NGX_QUIC_PATH_VALIDATED; + path->validated_at = ngx_time(); + } + + /* now bind socket to client and path */ + ngx_quic_connect(c, qsock, path, cid); + + if (ngx_quic_create_temp_socket(c, qc, &pkt->odcid, path, cid) != NGX_OK) { + goto failed; + } + + /* use this socket as default destination */ + qc->socket = qsock; + + ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic active socket is #%uL:%uL:%uL (%s)", + qsock->sid.seqnum, qsock->cid->seqnum, qsock->path->seqnum, + ngx_quic_path_state_str(qsock->path)); + + return NGX_OK; + +failed: + + ngx_rbtree_delete(&c->listening->rbtree, &qsock->udp.node); + c->udp = NULL; + + return NGX_ERROR; +} + + +static ngx_int_t +ngx_quic_create_temp_socket(ngx_connection_t *c, ngx_quic_connection_t *qc, + ngx_str_t *dcid, ngx_quic_path_t *path, ngx_quic_client_id_t *cid) +{ + ngx_str_t id; + ngx_quic_socket_t *qsock; + ngx_quic_server_id_t *sid; + + qsock = ngx_quic_alloc_socket(c, qc); + if (qsock == NULL) { + return NGX_ERROR; + } + + sid = &qsock->sid; + + sid->seqnum = NGX_QUIC_UNSET_PN; /* mark socket as temporary */ + + sid->len = dcid->len; + ngx_memcpy(sid->id, dcid->data, dcid->len); + + id.len = sid->len; + id.data = sid->id; + + ngx_insert_udp_connection(c, &qsock->udp, &id); + + ngx_queue_insert_tail(&qc->sockets, &qsock->queue); + + qc->nsockets++; + qsock->quic = qc; + + ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic socket #%L listening at sid:%xV nsock:%ui", + (int64_t) sid->seqnum, &id, qc->nsockets); + + ngx_quic_connect(c, qsock, path, cid); + + return NGX_OK; +} + + +ngx_quic_socket_t * +ngx_quic_alloc_socket(ngx_connection_t *c, ngx_quic_connection_t *qc) +{ + ngx_queue_t *q; + ngx_quic_socket_t *sock; + + if (!ngx_queue_empty(&qc->free_sockets)) { + + q = ngx_queue_head(&qc->free_sockets); + sock = ngx_queue_data(q, ngx_quic_socket_t, queue); + + ngx_queue_remove(&sock->queue); + + ngx_memzero(sock, sizeof(ngx_quic_socket_t)); + + } else { + + sock = ngx_pcalloc(c->pool, sizeof(ngx_quic_socket_t)); + if (sock == NULL) { + return NULL; + } + } + + return sock; +} + + +void +ngx_quic_close_socket(ngx_connection_t *c, ngx_quic_socket_t *qsock) +{ + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + ngx_queue_remove(&qsock->queue); + ngx_queue_insert_head(&qc->free_sockets, &qsock->queue); + + ngx_rbtree_delete(&c->listening->rbtree, &qsock->udp.node); + qc->nsockets--; + + if (qsock->path) { + ngx_quic_unref_path(c, qsock->path); + } + + if (qsock->cid) { + ngx_quic_unref_client_id(c, qsock->cid); + } + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic socket #%L closed nsock:%ui", + (int64_t) qsock->sid.seqnum, qc->nsockets); +} + + +static void +ngx_quic_unref_path(ngx_connection_t *c, ngx_quic_path_t *path) +{ + ngx_quic_connection_t *qc; + + path->refcnt--; + + if (path->refcnt) { + return; + } + + qc = ngx_quic_get_connection(c); + + ngx_queue_remove(&path->queue); + ngx_queue_insert_head(&qc->free_paths, &path->queue); + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic path #%uL addr:%V removed", + path->seqnum, &path->addr_text); +} + + +ngx_int_t +ngx_quic_listen(ngx_connection_t *c, ngx_quic_connection_t *qc, + ngx_quic_socket_t *qsock) +{ + ngx_str_t id; + ngx_quic_server_id_t *sid; + + sid = &qsock->sid; + + sid->len = NGX_QUIC_SERVER_CID_LEN; + + if (ngx_quic_create_server_id(c, sid->id) != NGX_OK) { + return NGX_ERROR; + } + + sid->seqnum = qc->server_seqnum++; + + id.data = sid->id; + id.len = sid->len; + + ngx_insert_udp_connection(c, &qsock->udp, &id); + + ngx_queue_insert_tail(&qc->sockets, &qsock->queue); + + qc->nsockets++; + qsock->quic = qc; + + ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic socket #%uL listening at sid:%xV nsock:%ui", + sid->seqnum, &id, qc->nsockets); + + return NGX_OK; +} + + +void +ngx_quic_connect(ngx_connection_t *c, ngx_quic_socket_t *sock, + ngx_quic_path_t *path, ngx_quic_client_id_t *cid) +{ + sock->path = path; + path->refcnt++; + + sock->cid = cid; + cid->refcnt++; + + ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic socket #%L connected to cid #%uL path:%uL", + (int64_t) sock->sid.seqnum, + sock->cid->seqnum, path->seqnum); +} + + +void +ngx_quic_close_sockets(ngx_connection_t *c) +{ + ngx_queue_t *q; + ngx_quic_socket_t *qsock; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + ngx_quic_close_socket(c, qc->socket); + + if (qc->backup) { + ngx_quic_close_socket(c, qc->backup); + } + + while (!ngx_queue_empty(&qc->sockets)) { + q = ngx_queue_head(&qc->sockets); + qsock = ngx_queue_data(q, ngx_quic_socket_t, queue); + + ngx_quic_close_socket(c, qsock); + } +} + + +ngx_quic_socket_t * +ngx_quic_find_socket(ngx_connection_t *c, uint64_t seqnum) +{ + ngx_queue_t *q; + ngx_quic_socket_t *qsock; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + for (q = ngx_queue_head(&qc->sockets); + q != ngx_queue_sentinel(&qc->sockets); + q = ngx_queue_next(q)) + { + qsock = ngx_queue_data(q, ngx_quic_socket_t, queue); + + if (qsock->sid.seqnum == seqnum) { + return qsock; + } + } + + return NULL; +} + + +ngx_quic_socket_t * +ngx_quic_get_unconnected_socket(ngx_connection_t *c) +{ + ngx_queue_t *q; + ngx_quic_socket_t *sock; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + for (q = ngx_queue_head(&qc->sockets); + q != ngx_queue_sentinel(&qc->sockets); + q = ngx_queue_next(q)) + { + sock = ngx_queue_data(q, ngx_quic_socket_t, queue); + + if (sock->cid == NULL) { + return sock; + } + } + + return NULL; + } + +
new file mode 100644 --- /dev/null +++ b/src/event/quic/ngx_event_quic_socket.h @@ -0,0 +1,32 @@ + +/* + * Copyright (C) Nginx, Inc. + */ + + +#ifndef _NGX_EVENT_QUIC_SOCKET_H_INCLUDED_ +#define _NGX_EVENT_QUIC_SOCKET_H_INCLUDED_ + + +#include <ngx_config.h> +#include <ngx_core.h> + + +ngx_int_t ngx_quic_open_sockets(ngx_connection_t *c, + ngx_quic_connection_t *qc, ngx_quic_header_t *pkt); +void ngx_quic_close_sockets(ngx_connection_t *c); + +ngx_quic_socket_t *ngx_quic_alloc_socket(ngx_connection_t *c, + ngx_quic_connection_t *qc); +ngx_int_t ngx_quic_listen(ngx_connection_t *c, ngx_quic_connection_t *qc, + ngx_quic_socket_t *qsock); +void ngx_quic_close_socket(ngx_connection_t *c, ngx_quic_socket_t *qsock); + +void ngx_quic_connect(ngx_connection_t *c, ngx_quic_socket_t *qsock, + ngx_quic_path_t *path, ngx_quic_client_id_t *cid); + +ngx_quic_socket_t *ngx_quic_find_socket(ngx_connection_t *c, uint64_t seqnum); +ngx_quic_socket_t *ngx_quic_get_unconnected_socket(ngx_connection_t *c); + + +#endif /* _NGX_EVENT_QUIC_SOCKET_H_INCLUDED_ */
--- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -391,8 +391,10 @@ ngx_quic_crypto_input(ngx_connection_t * frame->type = NGX_QUIC_FT_HANDSHAKE_DONE; ngx_quic_queue_frame(qc, frame); - if (ngx_quic_send_new_token(c) != NGX_OK) { - return NGX_ERROR; + if (qc->conf->retry) { + if (ngx_quic_send_new_token(c, qc->socket->path) != NGX_OK) { + return NGX_ERROR; + } } /* @@ -410,7 +412,8 @@ ngx_quic_crypto_input(ngx_connection_t * */ ngx_quic_discard_ctx(c, ssl_encryption_handshake); - if (ngx_quic_issue_server_ids(c) != NGX_OK) { + /* start accepting clients on negotiated number of server ids */ + if (ngx_quic_create_sockets(c) != NGX_OK) { return NGX_ERROR; } @@ -424,6 +427,7 @@ ngx_quic_init_connection(ngx_connection_ u_char *p; size_t clen; ssize_t len; + ngx_str_t dcid; ngx_ssl_conn_t *ssl_conn; ngx_quic_connection_t *qc; @@ -453,8 +457,10 @@ ngx_quic_init_connection(ngx_connection_ SSL_set_quic_use_legacy_codepoint(ssl_conn, qc->version != 1); #endif - if (ngx_quic_new_sr_token(c, &qc->dcid, qc->conf->sr_token_key, - qc->tp.sr_token) + dcid.data = qc->socket->sid.id; + dcid.len = qc->socket->sid.len; + + if (ngx_quic_new_sr_token(c, &dcid, qc->conf->sr_token_key, qc->tp.sr_token) != NGX_OK) { return NGX_ERROR;
--- a/src/event/quic/ngx_event_quic_tokens.c +++ b/src/event/quic/ngx_event_quic_tokens.c @@ -15,8 +15,8 @@ /* SHA-1(addr)=20 + sizeof(time_t) + retry(1) + odcid.len(1) + odcid */ -static void ngx_quic_address_hash(ngx_connection_t *c, ngx_uint_t no_port, - u_char buf[20]); +static void ngx_quic_address_hash(struct sockaddr *sockaddr, socklen_t socklen, + ngx_uint_t no_port, u_char buf[20]); ngx_int_t @@ -46,8 +46,9 @@ ngx_quic_new_sr_token(ngx_connection_t * ngx_int_t -ngx_quic_new_token(ngx_connection_t *c, u_char *key, ngx_str_t *token, - ngx_str_t *odcid, time_t exp, ngx_uint_t is_retry) +ngx_quic_new_token(ngx_connection_t *c, struct sockaddr *sockaddr, + socklen_t socklen, u_char *key, ngx_str_t *token, ngx_str_t *odcid, + time_t exp, ngx_uint_t is_retry) { int len, iv_len; u_char *p, *iv; @@ -56,7 +57,7 @@ ngx_quic_new_token(ngx_connection_t *c, u_char in[NGX_QUIC_MAX_TOKEN_SIZE]; - ngx_quic_address_hash(c, !is_retry, in); + ngx_quic_address_hash(sockaddr, socklen, !is_retry, in); p = in + 20; @@ -125,7 +126,8 @@ ngx_quic_new_token(ngx_connection_t *c, static void -ngx_quic_address_hash(ngx_connection_t *c, ngx_uint_t no_port, u_char buf[20]) +ngx_quic_address_hash(struct sockaddr *sockaddr, socklen_t socklen, + ngx_uint_t no_port, u_char buf[20]) { size_t len; u_char *data; @@ -135,15 +137,15 @@ ngx_quic_address_hash(ngx_connection_t * struct sockaddr_in6 *sin6; #endif - len = (size_t) c->socklen; - data = (u_char *) c->sockaddr; + len = (size_t) socklen; + data = (u_char *) sockaddr; if (no_port) { - switch (c->sockaddr->sa_family) { + switch (sockaddr->sa_family) { #if (NGX_HAVE_INET6) case AF_INET6: - sin6 = (struct sockaddr_in6 *) c->sockaddr; + sin6 = (struct sockaddr_in6 *) sockaddr; len = sizeof(struct in6_addr); data = sin6->sin6_addr.s6_addr; @@ -152,7 +154,7 @@ ngx_quic_address_hash(ngx_connection_t * #endif case AF_INET: - sin = (struct sockaddr_in *) c->sockaddr; + sin = (struct sockaddr_in *) sockaddr; len = sizeof(in_addr_t); data = (u_char *) &sin->sin_addr; @@ -236,7 +238,7 @@ ngx_quic_validate_token(ngx_connection_t pkt->retried = (*p++ == 1); - ngx_quic_address_hash(c, !pkt->retried, addr_hash); + ngx_quic_address_hash(c->sockaddr, c->socklen, !pkt->retried, addr_hash); if (ngx_memcmp(tdec, addr_hash, 20) != 0) { goto bad_token;
--- a/src/event/quic/ngx_event_quic_tokens.h +++ b/src/event/quic/ngx_event_quic_tokens.h @@ -14,8 +14,9 @@ ngx_int_t ngx_quic_new_sr_token(ngx_connection_t *c, ngx_str_t *cid, u_char *secret, u_char *token); -ngx_int_t ngx_quic_new_token(ngx_connection_t *c, u_char *key, - ngx_str_t *token, ngx_str_t *odcid, time_t expires, ngx_uint_t is_retry); +ngx_int_t ngx_quic_new_token(ngx_connection_t *c, struct sockaddr *sockaddr, + socklen_t socklen, u_char *key, ngx_str_t *token, ngx_str_t *odcid, + time_t expires, ngx_uint_t is_retry); ngx_int_t ngx_quic_validate_token(ngx_connection_t *c, u_char *key, ngx_quic_header_t *pkt);