Mercurial > hg > nginx
changeset 8657:2dfc5ef29973 quic
QUIC: introduced QUIC buffers.
Buffers are used to hold frame data. They have a fixed size and are reused
after being freed.
| author | Roman Arutyunyan <arut@nginx.com> |
|---|---|
| date | Tue, 01 Dec 2020 19:11:01 +0000 |
| parents | 43f3574b3e6f |
| children | 0af4ec6d1f92 |
| files | README src/event/ngx_event_quic.c src/event/ngx_event_quic.h src/event/ngx_event_quic_transport.c src/event/ngx_event_quic_transport.h |
| diffstat | 5 files changed, 388 insertions(+), 153 deletions(-) [+] |
line wrap: on
line diff
--- a/README +++ b/README @@ -237,7 +237,7 @@ 5. Troubleshooting #define NGX_QUIC_DEBUG_PACKETS #define NGX_QUIC_DEBUG_FRAMES - #define NGX_QUIC_DEBUG_FRAMES_ALLOC + #define NGX_QUIC_DEBUG_ALLOC #define NGX_QUIC_DEBUG_CRYPTO 6. Contributing
--- a/src/event/ngx_event_quic.c +++ b/src/event/ngx_event_quic.c @@ -143,7 +143,6 @@ typedef struct { ngx_event_t push; ngx_event_t pto; ngx_event_t close; - ngx_queue_t free_frames; ngx_msec_t last_cc; ngx_msec_t latest_rtt; @@ -153,8 +152,12 @@ typedef struct { ngx_uint_t pto_count; -#if (NGX_DEBUG) + ngx_queue_t free_frames; + ngx_chain_t *free_bufs; + +#ifdef NGX_QUIC_DEBUG_ALLOC ngx_uint_t nframes; + ngx_uint_t nbufs; #endif ngx_quic_streams_t streams; @@ -265,7 +268,7 @@ static ngx_int_t ngx_quic_send_cc(ngx_co static ngx_int_t ngx_quic_send_new_token(ngx_connection_t *c); static ngx_int_t ngx_quic_handle_ack_frame(ngx_connection_t *c, - ngx_quic_header_t *pkt, ngx_quic_ack_frame_t *f); + ngx_quic_header_t *pkt, ngx_quic_frame_t *f); static ngx_int_t ngx_quic_handle_ack_frame_range(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, uint64_t min, uint64_t max, ngx_msec_t *send_time); @@ -361,7 +364,7 @@ static ngx_chain_t *ngx_quic_stream_send static size_t ngx_quic_max_stream_frame(ngx_quic_connection_t *qc); static size_t ngx_quic_max_stream_flow(ngx_connection_t *c); static void ngx_quic_stream_cleanup_handler(void *data); -static ngx_quic_frame_t *ngx_quic_alloc_frame(ngx_connection_t *c, size_t size); +static ngx_quic_frame_t *ngx_quic_alloc_frame(ngx_connection_t *c); static void ngx_quic_free_frame(ngx_connection_t *c, ngx_quic_frame_t *frame); static void ngx_quic_congestion_ack(ngx_connection_t *c, @@ -369,6 +372,13 @@ static void ngx_quic_congestion_ack(ngx_ static void ngx_quic_congestion_lost(ngx_connection_t *c, ngx_quic_frame_t *frame); +static ngx_chain_t *ngx_quic_alloc_buf(ngx_connection_t *c); +static void ngx_quic_free_bufs(ngx_connection_t *c, ngx_chain_t *in); +static ngx_chain_t *ngx_quic_copy_buf(ngx_connection_t *c, u_char *data, + size_t len); +static ngx_chain_t *ngx_quic_copy_chain(ngx_connection_t *c, ngx_chain_t *in, + size_t limit); + static SSL_QUIC_METHOD quic_method = { #if BORINGSSL_API_VERSION >= 10 @@ -414,8 +424,14 @@ ngx_quic_log_frame(ngx_log_t *log, ngx_q p = ngx_slprintf(p, last, "ACK n:%ui delay:%uL ", f->u.ack.range_count, f->u.ack.delay); - pos = f->u.ack.ranges_start; - end = f->u.ack.ranges_end; + if (f->data) { + pos = f->data->buf->pos; + end = f->data->buf->end; + + } else { + pos = NULL; + end = NULL; + } largest = f->u.ack.largest; smallest = f->u.ack.largest - f->u.ack.first_range; @@ -507,8 +523,16 @@ ngx_quic_log_frame(ngx_log_t *log, ngx_q } #ifdef NGX_QUIC_DEBUG_FRAMES - p = ngx_slprintf(p, last, " data len:%uL %*xs", f->u.stream.length, - (size_t) f->u.stream.length, f->u.stream.data); + { + ngx_chain_t *cl; + + p = ngx_slprintf(p, last, " data:"); + + for (cl = f->data; cl; cl = cl->next) { + p = ngx_slprintf(p, last, "%*xs", + cl->buf->last - cl->buf->pos, cl->buf->pos); + } + } #endif break; @@ -885,18 +909,20 @@ ngx_quic_add_handshake_data(ngx_ssl_conn fsize = ngx_min(limit, (size_t) (end - p)); - frame = ngx_quic_alloc_frame(c, fsize); + frame = ngx_quic_alloc_frame(c); if (frame == NULL) { return 0; } - ngx_memcpy(frame->data, p, fsize); + frame->data = ngx_quic_copy_buf(c, p, fsize); + if (frame->data == NGX_CHAIN_ERROR) { + return 0; + } frame->level = level; frame->type = NGX_QUIC_FT_CRYPTO; frame->u.crypto.offset = fs->sent; frame->u.crypto.length = fsize; - frame->u.crypto.data = frame->data; fs->sent += fsize; p += fsize; @@ -1870,15 +1896,6 @@ ngx_quic_close_quic(ngx_connection_t *c, ngx_delete_posted_event(&qc->push); } - for (i = 0; i < NGX_QUIC_ENCRYPTION_LAST; i++) { - ngx_quic_free_frames(c, &qc->crypto[i].frames); - } - - for (i = 0; i < NGX_QUIC_SEND_CTX_LAST; i++) { - ngx_quic_free_frames(c, &qc->send_ctx[i].frames); - ngx_quic_free_frames(c, &qc->send_ctx[i].sent); - } - while (!ngx_queue_empty(&qc->server_ids)) { q = ngx_queue_head(&qc->server_ids); sid = ngx_queue_data(q, ngx_quic_server_id_t, queue); @@ -2438,7 +2455,9 @@ ngx_quic_payload_handler(ngx_connection_ { u_char *end, *p; ssize_t len; + ngx_buf_t buf; ngx_uint_t do_close; + ngx_chain_t chain; ngx_quic_frame_t frame; ngx_quic_connection_t *qc; @@ -2472,6 +2491,12 @@ ngx_quic_payload_handler(ngx_connection_ c->log->action = "parsing frames"; + ngx_memzero(&buf, sizeof(ngx_buf_t)); + + chain.buf = &buf; + chain.next = NULL; + frame.data = &chain; + len = ngx_quic_parse_frame(pkt, p, end, &frame); if (len < 0) { @@ -2488,7 +2513,7 @@ ngx_quic_payload_handler(ngx_connection_ switch (frame.type) { case NGX_QUIC_FT_ACK: - if (ngx_quic_handle_ack_frame(c, pkt, &frame.u.ack) != NGX_OK) { + if (ngx_quic_handle_ack_frame(c, pkt, &frame) != NGX_OK) { return NGX_ERROR; } @@ -2922,7 +2947,7 @@ ngx_quic_send_ack_range(ngx_connection_t { ngx_quic_frame_t *frame; - frame = ngx_quic_alloc_frame(c, 0); + frame = ngx_quic_alloc_frame(c); if (frame == NULL) { return NGX_ERROR; } @@ -2999,10 +3024,11 @@ ngx_quic_drop_ack_ranges(ngx_connection_ static ngx_int_t ngx_quic_send_ack(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx) { - u_char *p; - size_t ranges_len; + size_t len, left; uint64_t ack_delay; + ngx_buf_t *b; ngx_uint_t i; + ngx_chain_t *cl, **ll; ngx_quic_frame_t *frame; ngx_quic_connection_t *qc; @@ -3017,24 +3043,44 @@ ngx_quic_send_ack(ngx_connection_t *c, n ack_delay = 0; } - ranges_len = 0; + frame = ngx_quic_alloc_frame(c); + if (frame == NULL) { + return NGX_ERROR; + } + + ll = &frame->data; + b = NULL; for (i = 0; i < ctx->nranges; i++) { - ranges_len += ngx_quic_create_ack_range(NULL, ctx->ranges[i].gap, - ctx->ranges[i].range); - } - - frame = ngx_quic_alloc_frame(c, ranges_len); - if (frame == NULL) { - return NGX_ERROR; - } - - p = frame->data; - - for (i = 0; i < ctx->nranges; i++) { - p += ngx_quic_create_ack_range(p, ctx->ranges[i].gap, - ctx->ranges[i].range); - } + len = ngx_quic_create_ack_range(NULL, ctx->ranges[i].gap, + ctx->ranges[i].range); + + left = b ? b->end - b->last : 0; + + if (left < len) { + cl = ngx_quic_alloc_buf(c); + if (cl == NULL) { + return NGX_ERROR; + } + + *ll = cl; + ll = &cl->next; + + b = cl->buf; + left = b->end - b->last; + + if (left < len) { + return NGX_ERROR; + } + } + + b->last += ngx_quic_create_ack_range(b->last, ctx->ranges[i].gap, + ctx->ranges[i].range); + + frame->u.ack.ranges_length += len; + } + + *ll = NULL; frame->level = ctx->level; frame->type = NGX_QUIC_FT_ACK; @@ -3042,8 +3088,6 @@ ngx_quic_send_ack(ngx_connection_t *c, n frame->u.ack.delay = ack_delay; frame->u.ack.range_count = ctx->nranges; frame->u.ack.first_range = ctx->first_range; - frame->u.ack.ranges_start = frame->data; - frame->u.ack.ranges_end = frame->data + ranges_len; ngx_quic_queue_frame(qc, frame); @@ -3077,7 +3121,7 @@ ngx_quic_send_cc(ngx_connection_t *c) return NGX_OK; } - frame = ngx_quic_alloc_frame(c, 0); + frame = ngx_quic_alloc_frame(c); if (frame == NULL) { return NGX_ERROR; } @@ -3118,7 +3162,7 @@ ngx_quic_send_new_token(ngx_connection_t return NGX_ERROR; } - frame = ngx_quic_alloc_frame(c, 0); + frame = ngx_quic_alloc_frame(c); if (frame == NULL) { return NGX_ERROR; } @@ -3136,7 +3180,7 @@ ngx_quic_send_new_token(ngx_connection_t static ngx_int_t ngx_quic_handle_ack_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, - ngx_quic_ack_frame_t *ack) + ngx_quic_frame_t *f) { ssize_t n; u_char *pos, *end; @@ -3144,6 +3188,7 @@ ngx_quic_handle_ack_frame(ngx_connection ngx_msec_t send_time; ngx_uint_t i; ngx_quic_send_ctx_t *ctx; + ngx_quic_ack_frame_t *ack; ngx_quic_connection_t *qc; qc = ngx_quic_get_connection(c); @@ -3153,6 +3198,8 @@ ngx_quic_handle_ack_frame(ngx_connection ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic ngx_quic_handle_ack_frame level:%d", pkt->level); + ack = &f->u.ack; + /* * If any computed packet number is negative, an endpoint MUST * generate a connection error of type FRAME_ENCODING_ERROR. @@ -3194,8 +3241,14 @@ ngx_quic_handle_ack_frame(ngx_connection } } - pos = ack->ranges_start; - end = ack->ranges_end; + if (f->data) { + pos = f->data->buf->pos; + end = f->data->buf->last; + + } else { + pos = NULL; + end = NULL; + } for (i = 0; i < ack->range_count; i++) { @@ -3537,7 +3590,9 @@ static ngx_int_t ngx_quic_adjust_frame_offset(ngx_connection_t *c, ngx_quic_frame_t *frame, uint64_t offset_in) { - size_t tail; + size_t tail, n; + ngx_buf_t *b; + ngx_chain_t *cl; ngx_quic_ordered_frame_t *f; f = &frame->u.ord; @@ -3558,9 +3613,21 @@ ngx_quic_adjust_frame_offset(ngx_connect /* intersecting range: adjust data size */ f->offset += tail; - f->data += tail; f->length -= tail; + for (cl = frame->data; cl; cl = cl->next) { + b = cl->buf; + n = ngx_buf_size(b); + + if (n >= tail) { + b->pos += tail; + break; + } + + cl->buf->pos = cl->buf->last; + tail -= n; + } + return NGX_OK; } @@ -3569,7 +3636,6 @@ static ngx_int_t ngx_quic_buffer_frame(ngx_connection_t *c, ngx_quic_frames_stream_t *fs, ngx_quic_frame_t *frame) { - u_char *data; ngx_queue_t *q; ngx_quic_frame_t *dst, *item; ngx_quic_ordered_frame_t *f, *df; @@ -3581,19 +3647,19 @@ ngx_quic_buffer_frame(ngx_connection_t * /* frame start offset is in the future, buffer it */ - dst = ngx_quic_alloc_frame(c, f->length); + dst = ngx_quic_alloc_frame(c); if (dst == NULL) { return NGX_ERROR; } - data = dst->data; ngx_memcpy(dst, frame, sizeof(ngx_quic_frame_t)); - dst->data = data; - - ngx_memcpy(dst->data, f->data, f->length); + + dst->data = ngx_quic_copy_chain(c, frame->data, 0); + if (dst->data == NGX_CHAIN_ERROR) { + return NGX_ERROR; + } df = &dst->u.ord; - df->data = dst->data; fs->total += f->length; @@ -3671,15 +3737,14 @@ ngx_quic_handle_crypto_frame(ngx_connect static ngx_int_t ngx_quic_crypto_input(ngx_connection_t *c, ngx_quic_frame_t *frame, void *data) { - int n, sslerr; - ngx_ssl_conn_t *ssl_conn; - ngx_quic_connection_t *qc; - ngx_quic_crypto_frame_t *f; + int n, sslerr; + ngx_buf_t *b; + ngx_chain_t *cl; + ngx_ssl_conn_t *ssl_conn; + ngx_quic_connection_t *qc; qc = ngx_quic_get_connection(c); - f = &frame->u.crypto; - ssl_conn = c->ssl->connection; ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, @@ -3687,12 +3752,16 @@ ngx_quic_crypto_input(ngx_connection_t * (int) SSL_quic_read_level(ssl_conn), (int) SSL_quic_write_level(ssl_conn)); - if (!SSL_provide_quic_data(ssl_conn, SSL_quic_read_level(ssl_conn), - f->data, f->length)) - { - ngx_ssl_error(NGX_LOG_INFO, c->log, 0, - "SSL_provide_quic_data() failed"); - return NGX_ERROR; + for (cl = frame->data; cl; cl = cl->next) { + b = cl->buf; + + if (!SSL_provide_quic_data(ssl_conn, SSL_quic_read_level(ssl_conn), + b->pos, b->last - b->pos)) + { + ngx_ssl_error(NGX_LOG_INFO, c->log, 0, + "SSL_provide_quic_data() failed"); + return NGX_ERROR; + } } n = SSL_do_handshake(ssl_conn); @@ -3730,7 +3799,7 @@ ngx_quic_crypto_input(ngx_connection_t * c->ssl->handshaked = 1; - frame = ngx_quic_alloc_frame(c, 0); + frame = ngx_quic_alloc_frame(c); if (frame == NULL) { return NGX_ERROR; } @@ -3858,6 +3927,7 @@ ngx_quic_stream_input(ngx_connection_t * uint64_t id; ngx_buf_t *b; ngx_event_t *rev; + ngx_chain_t *cl; ngx_quic_stream_t *sn; ngx_quic_connection_t *qc; ngx_quic_stream_frame_t *f; @@ -3881,7 +3951,10 @@ ngx_quic_stream_input(ngx_connection_t * b->pos = b->start; } - b->last = ngx_cpymem(b->last, f->data, f->length); + for (cl = frame->data; cl; cl = cl->next) { + b->last = ngx_cpymem(b->last, cl->buf->pos, + cl->buf->last - cl->buf->pos); + } rev = sn->c->read; rev->ready = 1; @@ -3992,7 +4065,7 @@ ngx_quic_handle_stream_data_blocked_fram n = sn->fs.received + (b->pos - b->start) + (b->end - b->last); } - frame = ngx_quic_alloc_frame(c, 0); + frame = ngx_quic_alloc_frame(c); if (frame == NULL) { return NGX_ERROR; } @@ -4215,7 +4288,7 @@ ngx_quic_handle_path_challenge_frame(ngx qc = ngx_quic_get_connection(c); - frame = ngx_quic_alloc_frame(c, 0); + frame = ngx_quic_alloc_frame(c); if (frame == NULL) { return NGX_ERROR; } @@ -4381,7 +4454,7 @@ ngx_quic_retire_connection_id(ngx_connec qc = ngx_quic_get_connection(c); - frame = ngx_quic_alloc_frame(c, 0); + frame = ngx_quic_alloc_frame(c); if (frame == NULL) { return NGX_ERROR; } @@ -4455,7 +4528,7 @@ ngx_quic_issue_server_ids(ngx_connection return NGX_ERROR; } - frame = ngx_quic_alloc_frame(c, 0); + frame = ngx_quic_alloc_frame(c); if (frame == NULL) { return NGX_ERROR; } @@ -5634,7 +5707,7 @@ ngx_quic_stream_recv(ngx_connection_t *c qs->id, len, size); if (!rev->pending_eof) { - frame = ngx_quic_alloc_frame(pc, 0); + frame = ngx_quic_alloc_frame(pc); if (frame == NULL) { return NGX_ERROR; } @@ -5650,7 +5723,7 @@ ngx_quic_stream_recv(ngx_connection_t *c if ((qc->streams.recv_max_data / 2) < qc->streams.received) { - frame = ngx_quic_alloc_frame(pc, 0); + frame = ngx_quic_alloc_frame(pc); if (frame == NULL) { return NGX_ERROR; @@ -5703,13 +5776,9 @@ ngx_quic_stream_send(ngx_connection_t *c static ngx_chain_t * ngx_quic_stream_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) { - u_char *p; - size_t n, max, max_frame, max_flow, max_limit, len; + size_t n, max, max_frame, max_flow, max_limit; #if (NGX_DEBUG) size_t sent; -#endif - ngx_buf_t *b; -#if (NGX_DEBUG) ngx_uint_t nframes; #endif ngx_event_t *wev; @@ -5763,7 +5832,7 @@ ngx_quic_stream_send_chain(ngx_connectio break; } - frame = ngx_quic_alloc_frame(pc, n); + frame = ngx_quic_alloc_frame(pc); if (frame == NULL) { return NGX_CHAIN_ERROR; } @@ -5778,7 +5847,6 @@ ngx_quic_stream_send_chain(ngx_connectio frame->u.stream.stream_id = qs->id; frame->u.stream.offset = c->sent; frame->u.stream.length = n; - frame->u.stream.data = frame->data; c->sent += n; qc->streams.sent += n; @@ -5793,18 +5861,9 @@ ngx_quic_stream_send_chain(ngx_connectio nframes++; #endif - for (p = frame->data; n > 0; cl = cl->next) { - b = cl->buf; - - if (!ngx_buf_in_memory(b)) { - continue; - } - - len = ngx_min(n, (size_t) (b->last - b->pos)); - p = ngx_cpymem(p, b->pos, len); - - b->pos += len; - n -= len; + frame->data = ngx_quic_copy_chain(pc, cl, n); + if (frame->data == NGX_CHAIN_ERROR) { + return NGX_CHAIN_ERROR; } ngx_quic_queue_frame(qc, frame); @@ -5916,7 +5975,7 @@ ngx_quic_stream_cleanup_handler(void *da || (qs->id & NGX_QUIC_STREAM_UNIDIRECTIONAL) == 0) { if (!c->read->pending_eof && !c->read->error) { - frame = ngx_quic_alloc_frame(pc, 0); + frame = ngx_quic_alloc_frame(pc); if (frame == NULL) { return; } @@ -5931,7 +5990,7 @@ ngx_quic_stream_cleanup_handler(void *da } if ((qs->id & NGX_QUIC_STREAM_SERVER_INITIATED) == 0) { - frame = ngx_quic_alloc_frame(pc, 0); + frame = ngx_quic_alloc_frame(pc); if (frame == NULL) { return; } @@ -5959,7 +6018,7 @@ ngx_quic_stream_cleanup_handler(void *da ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic stream id:0x%xL send fin", qs->id); - frame = ngx_quic_alloc_frame(pc, 0); + frame = ngx_quic_alloc_frame(pc); if (frame == NULL) { return; } @@ -5974,7 +6033,6 @@ ngx_quic_stream_cleanup_handler(void *da frame->u.stream.stream_id = qs->id; frame->u.stream.offset = c->sent; frame->u.stream.length = 0; - frame->u.stream.data = NULL; ngx_quic_queue_frame(qc, frame); @@ -5983,23 +6041,12 @@ ngx_quic_stream_cleanup_handler(void *da static ngx_quic_frame_t * -ngx_quic_alloc_frame(ngx_connection_t *c, size_t size) +ngx_quic_alloc_frame(ngx_connection_t *c) { - u_char *p; ngx_queue_t *q; ngx_quic_frame_t *frame; ngx_quic_connection_t *qc; - if (size) { - p = ngx_alloc(size, c->log); - if (p == NULL) { - return NULL; - } - - } else { - p = NULL; - } - qc = ngx_quic_get_connection(c); if (!ngx_queue_empty(&qc->free_frames)) { @@ -6009,7 +6056,7 @@ ngx_quic_alloc_frame(ngx_connection_t *c ngx_queue_remove(&frame->queue); -#ifdef NGX_QUIC_DEBUG_FRAMES_ALLOC +#ifdef NGX_QUIC_DEBUG_ALLOC ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic reuse frame n:%ui", qc->nframes); #endif @@ -6017,15 +6064,12 @@ ngx_quic_alloc_frame(ngx_connection_t *c } else { frame = ngx_pcalloc(c->pool, sizeof(ngx_quic_frame_t)); if (frame == NULL) { - ngx_free(p); return NULL; } -#if (NGX_DEBUG) +#ifdef NGX_QUIC_DEBUG_ALLOC ++qc->nframes; -#endif - -#ifdef NGX_QUIC_DEBUG_FRAMES_ALLOC + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic alloc frame n:%ui", qc->nframes); #endif @@ -6033,8 +6077,6 @@ ngx_quic_alloc_frame(ngx_connection_t *c ngx_memzero(frame, sizeof(ngx_quic_frame_t)); - frame->data = p; - return frame; } @@ -6140,13 +6182,12 @@ ngx_quic_free_frame(ngx_connection_t *c, qc = ngx_quic_get_connection(c); if (frame->data) { - ngx_free(frame->data); - frame->data = NULL; + ngx_quic_free_bufs(c, frame->data); } ngx_queue_insert_head(&qc->free_frames, &frame->queue); -#ifdef NGX_QUIC_DEBUG_FRAMES_ALLOC +#ifdef NGX_QUIC_DEBUG_ALLOC ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic free frame n:%ui", qc->nframes); #endif @@ -6165,3 +6206,170 @@ ngx_quic_version(ngx_connection_t *c) return (version & 0xff000000) == 0xff000000 ? version & 0xff : version; } + + +static ngx_chain_t * +ngx_quic_alloc_buf(ngx_connection_t *c) +{ + ngx_buf_t *b; + ngx_chain_t *cl; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + if (qc->free_bufs) { + cl = qc->free_bufs; + qc->free_bufs = cl->next; + + b = cl->buf; + b->pos = b->start; + b->last = b->start; + +#ifdef NGX_QUIC_DEBUG_ALLOC + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic reuse buffer n:%ui", qc->nbufs); +#endif + + return cl; + } + + cl = ngx_alloc_chain_link(c->pool); + if (cl == NULL) { + return NULL; + } + + b = ngx_create_temp_buf(c->pool, NGX_QUIC_BUFFER_SIZE); + if (b == NULL) { + return NULL; + } + + b->tag = (ngx_buf_tag_t) &ngx_quic_alloc_buf; + + cl->buf = b; + +#ifdef NGX_QUIC_DEBUG_ALLOC + ++qc->nbufs; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic alloc buffer n:%ui", qc->nbufs); +#endif + + return cl; +} + + +static void +ngx_quic_free_bufs(ngx_connection_t *c, ngx_chain_t *in) +{ + ngx_chain_t *cl; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + while (in) { +#ifdef NGX_QUIC_DEBUG_ALLOC + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic free buffer n:%ui", qc->nbufs); +#endif + + cl = in; + in = in->next; + + cl->next = qc->free_bufs; + qc->free_bufs = cl; + } +} + + +static ngx_chain_t * +ngx_quic_copy_buf(ngx_connection_t *c, u_char *data, size_t len) +{ + size_t n; + ngx_buf_t *b; + ngx_chain_t *cl, *out, **ll; + + out = NULL; + ll = &out; + + while (len) { + cl = ngx_quic_alloc_buf(c); + if (cl == NULL) { + return NGX_CHAIN_ERROR; + } + + b = cl->buf; + n = ngx_min((size_t) (b->end - b->last), len); + + b->last = ngx_cpymem(b->last, data, n); + + data += n; + len -= n; + + *ll = cl; + ll = &cl->next; + } + + *ll = NULL; + + return out; +} + + +static ngx_chain_t * +ngx_quic_copy_chain(ngx_connection_t *c, ngx_chain_t *in, size_t limit) +{ + size_t n; + ngx_buf_t *b; + ngx_chain_t *cl, *out, **ll; + + out = NULL; + ll = &out; + + while (in) { + if (!ngx_buf_in_memory(in->buf) || ngx_buf_size(in->buf) == 0) { + in = in->next; + continue; + } + + cl = ngx_quic_alloc_buf(c); + if (cl == NULL) { + return NGX_CHAIN_ERROR; + } + + *ll = cl; + ll = &cl->next; + + b = cl->buf; + + while (in && b->last != b->end) { + + n = ngx_min(in->buf->last - in->buf->pos, b->end - b->last); + + if (limit > 0 && n > limit) { + n = limit; + } + + b->last = ngx_cpymem(b->last, in->buf->pos, n); + + in->buf->pos += n; + if (in->buf->pos == in->buf->last) { + in = in->next; + } + + if (limit > 0) { + if (limit == n) { + goto done; + } + + limit -= n; + } + } + + } + +done: + + *ll = NULL; + + return out; +}
--- a/src/event/ngx_event_quic.h +++ b/src/event/ngx_event_quic.h @@ -61,6 +61,8 @@ #define NGX_QUIC_MAX_SERVER_IDS 8 +#define NGX_QUIC_BUFFER_SIZE 4096 + #define ngx_quic_get_connection(c) ((ngx_quic_connection_t *)(c)->udp) @@ -135,7 +137,7 @@ ngx_int_t ngx_quic_get_packet_dcid(ngx_l /* #define NGX_QUIC_DEBUG_PACKETS */ /* dump packet contents */ /* #define NGX_QUIC_DEBUG_FRAMES */ /* dump frames contents */ -/* #define NGX_QUIC_DEBUG_FRAMES_ALLOC */ /* log frames alloc/reuse/free */ +/* #define NGX_QUIC_DEBUG_ALLOC */ /* log frames and bufs alloc */ /* #define NGX_QUIC_DEBUG_CRYPTO */ #endif /* _NGX_EVENT_QUIC_H_INCLUDED_ */
--- a/src/event/ngx_event_quic_transport.c +++ b/src/event/ngx_event_quic_transport.c @@ -87,15 +87,17 @@ static size_t ngx_quic_create_short_head static ngx_int_t ngx_quic_frame_allowed(ngx_quic_header_t *pkt, ngx_uint_t frame_type); -static size_t ngx_quic_create_ack(u_char *p, ngx_quic_ack_frame_t *ack); +static size_t ngx_quic_create_ack(u_char *p, ngx_quic_ack_frame_t *ack, + ngx_chain_t *ranges); static size_t ngx_quic_create_stop_sending(u_char *p, ngx_quic_stop_sending_frame_t *ss); static size_t ngx_quic_create_crypto(u_char *p, - ngx_quic_crypto_frame_t *crypto); + ngx_quic_crypto_frame_t *crypto, ngx_chain_t *data); static size_t ngx_quic_create_hs_done(u_char *p); static size_t ngx_quic_create_new_token(u_char *p, ngx_quic_new_token_frame_t *token); -static size_t ngx_quic_create_stream(u_char *p, ngx_quic_stream_frame_t *sf); +static size_t ngx_quic_create_stream(u_char *p, ngx_quic_stream_frame_t *sf, + ngx_chain_t *data); static size_t ngx_quic_create_max_streams(u_char *p, ngx_quic_max_streams_frame_t *ms); static size_t ngx_quic_create_max_stream_data(u_char *p, @@ -703,8 +705,11 @@ ngx_quic_parse_frame(ngx_quic_header_t * { u_char *p; uint64_t varint; + ngx_buf_t *b; ngx_uint_t i; + b = f->data->buf; + p = start; p = ngx_quic_parse_int(p, end, &varint); @@ -736,11 +741,13 @@ ngx_quic_parse_frame(ngx_quic_header_t * goto error; } - p = ngx_quic_read_bytes(p, end, f->u.crypto.length, &f->u.crypto.data); + p = ngx_quic_read_bytes(p, end, f->u.crypto.length, &b->pos); if (p == NULL) { goto error; } + b->last = p; + break; case NGX_QUIC_FT_PADDING: @@ -762,7 +769,7 @@ ngx_quic_parse_frame(ngx_quic_header_t * goto error; } - f->u.ack.ranges_start = p; + b->pos = p; /* process all ranges to get bounds, values are ignored */ for (i = 0; i < f->u.ack.range_count; i++) { @@ -777,7 +784,9 @@ ngx_quic_parse_frame(ngx_quic_header_t * } } - f->u.ack.ranges_end = p; + b->last = p; + + f->u.ack.ranges_length = b->last - b->pos; if (f->type == NGX_QUIC_FT_ACK_ECN) { @@ -914,12 +923,12 @@ ngx_quic_parse_frame(ngx_quic_header_t * f->u.stream.length = end - p; /* up to packet end */ } - p = ngx_quic_read_bytes(p, end, f->u.stream.length, - &f->u.stream.data); + p = ngx_quic_read_bytes(p, end, f->u.stream.length, &b->pos); if (p == NULL) { goto error; } + b->last = p; break; case NGX_QUIC_FT_MAX_DATA: @@ -1192,13 +1201,13 @@ ngx_quic_create_frame(u_char *p, ngx_qui switch (f->type) { case NGX_QUIC_FT_ACK: f->need_ack = 0; - return ngx_quic_create_ack(p, &f->u.ack); + return ngx_quic_create_ack(p, &f->u.ack, f->data); case NGX_QUIC_FT_STOP_SENDING: return ngx_quic_create_stop_sending(p, &f->u.stop_sending); case NGX_QUIC_FT_CRYPTO: - return ngx_quic_create_crypto(p, &f->u.crypto); + return ngx_quic_create_crypto(p, &f->u.crypto, f->data); case NGX_QUIC_FT_HANDSHAKE_DONE: return ngx_quic_create_hs_done(p); @@ -1214,7 +1223,7 @@ ngx_quic_create_frame(u_char *p, ngx_qui case NGX_QUIC_FT_STREAM5: case NGX_QUIC_FT_STREAM6: case NGX_QUIC_FT_STREAM7: - return ngx_quic_create_stream(p, &f->u.stream); + return ngx_quic_create_stream(p, &f->u.stream, f->data); case NGX_QUIC_FT_CONNECTION_CLOSE: case NGX_QUIC_FT_CONNECTION_CLOSE_APP: @@ -1247,10 +1256,11 @@ ngx_quic_create_frame(u_char *p, ngx_qui static size_t -ngx_quic_create_ack(u_char *p, ngx_quic_ack_frame_t *ack) +ngx_quic_create_ack(u_char *p, ngx_quic_ack_frame_t *ack, ngx_chain_t *ranges) { - size_t len; - u_char *start; + size_t len; + u_char *start; + ngx_buf_t *b; if (p == NULL) { len = ngx_quic_varint_len(NGX_QUIC_FT_ACK); @@ -1258,7 +1268,7 @@ ngx_quic_create_ack(u_char *p, ngx_quic_ len += ngx_quic_varint_len(ack->delay); len += ngx_quic_varint_len(ack->range_count); len += ngx_quic_varint_len(ack->first_range); - len += ack->ranges_end - ack->ranges_start; + len += ack->ranges_length; return len; } @@ -1270,7 +1280,12 @@ ngx_quic_create_ack(u_char *p, ngx_quic_ ngx_quic_build_int(&p, ack->delay); ngx_quic_build_int(&p, ack->range_count); ngx_quic_build_int(&p, ack->first_range); - p = ngx_cpymem(p, ack->ranges_start, ack->ranges_end - ack->ranges_start); + + while (ranges) { + b = ranges->buf; + p = ngx_cpymem(p, b->pos, b->last - b->pos); + ranges = ranges->next; + } return p - start; } @@ -1300,10 +1315,12 @@ ngx_quic_create_stop_sending(u_char *p, static size_t -ngx_quic_create_crypto(u_char *p, ngx_quic_crypto_frame_t *crypto) +ngx_quic_create_crypto(u_char *p, ngx_quic_crypto_frame_t *crypto, + ngx_chain_t *data) { - size_t len; - u_char *start; + size_t len; + u_char *start; + ngx_buf_t *b; if (p == NULL) { len = ngx_quic_varint_len(NGX_QUIC_FT_CRYPTO); @@ -1319,7 +1336,12 @@ ngx_quic_create_crypto(u_char *p, ngx_qu ngx_quic_build_int(&p, NGX_QUIC_FT_CRYPTO); ngx_quic_build_int(&p, crypto->offset); ngx_quic_build_int(&p, crypto->length); - p = ngx_cpymem(p, crypto->data, crypto->length); + + while (data) { + b = data->buf; + p = ngx_cpymem(p, b->pos, b->last - b->pos); + data = data->next; + } return p - start; } @@ -1367,10 +1389,12 @@ ngx_quic_create_new_token(u_char *p, ngx static size_t -ngx_quic_create_stream(u_char *p, ngx_quic_stream_frame_t *sf) +ngx_quic_create_stream(u_char *p, ngx_quic_stream_frame_t *sf, + ngx_chain_t *data) { - size_t len; - u_char *start; + size_t len; + u_char *start; + ngx_buf_t *b; if (p == NULL) { len = ngx_quic_varint_len(sf->type); @@ -1401,7 +1425,11 @@ ngx_quic_create_stream(u_char *p, ngx_qu /* length is always present in generated frames */ ngx_quic_build_int(&p, sf->length); - p = ngx_cpymem(p, sf->data, sf->length); + while (data) { + b = data->buf; + p = ngx_cpymem(p, b->pos, b->last - b->pos); + data = data->next; + } return p - start; }
--- a/src/event/ngx_event_quic_transport.h +++ b/src/event/ngx_event_quic_transport.h @@ -144,8 +144,7 @@ typedef struct { uint64_t ect0; uint64_t ect1; uint64_t ce; - u_char *ranges_start; - u_char *ranges_end; + uint64_t ranges_length; } ngx_quic_ack_frame_t; @@ -171,7 +170,6 @@ typedef struct { typedef struct { uint64_t offset; uint64_t length; - u_char *data; } ngx_quic_ordered_frame_t; typedef ngx_quic_ordered_frame_t ngx_quic_crypto_frame_t; @@ -181,7 +179,6 @@ typedef struct { /* initial fields same as in ngx_quic_ordered_frame_t */ uint64_t offset; uint64_t length; - u_char *data; uint8_t type; uint64_t stream_id; @@ -270,7 +267,7 @@ struct ngx_quic_frame_s { ngx_uint_t need_ack; /* unsigned need_ack:1; */ - u_char *data; + ngx_chain_t *data; union { ngx_quic_ack_frame_t ack; ngx_quic_crypto_frame_t crypto;