Mercurial > hg > nginx
diff src/event/quic/ngx_event_quic.c @ 8749:660c4a2f95f3 quic
QUIC: separate files for frames related processing.
author | Vladimir Homutov <vl@nginx.com> |
---|---|
date | Tue, 13 Apr 2021 14:38:46 +0300 |
parents | e0cb1e58ca13 |
children | 41807e581de9 |
line wrap: on
line diff
--- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -36,10 +36,6 @@ #define NGX_QUIC_MAX_ACK_GAP 2 -typedef ngx_int_t (*ngx_quic_frame_handler_pt)(ngx_connection_t *c, - ngx_quic_frame_t *frame, void *data); - - #if BORINGSSL_API_VERSION >= 10 static int ngx_quic_set_read_secret(ngx_ssl_conn_t *ssl_conn, enum ssl_encryption_level_t level, const SSL_CIPHER *cipher, @@ -122,17 +118,9 @@ static void ngx_quic_rtt_sample(ngx_conn static void ngx_quic_handle_stream_ack(ngx_connection_t *c, ngx_quic_frame_t *f); -static ngx_int_t ngx_quic_handle_ordered_frame(ngx_connection_t *c, - ngx_quic_frames_stream_t *fs, ngx_quic_frame_t *frame, - ngx_quic_frame_handler_pt handler, void *data); -static ngx_int_t ngx_quic_adjust_frame_offset(ngx_connection_t *c, - ngx_quic_frame_t *f, uint64_t offset_in); -static ngx_int_t ngx_quic_buffer_frame(ngx_connection_t *c, - ngx_quic_frames_stream_t *stream, ngx_quic_frame_t *f); - static ngx_int_t ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_frame_t *frame); -static ngx_int_t ngx_quic_crypto_input(ngx_connection_t *c, +ngx_int_t ngx_quic_crypto_input(ngx_connection_t *c, ngx_quic_frame_t *frame, void *data); static ngx_int_t ngx_quic_handle_stream_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_frame_t *frame); @@ -160,9 +148,6 @@ static ngx_int_t ngx_quic_generate_ack(n ngx_quic_send_ctx_t *ctx); static ssize_t ngx_quic_output_packet(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, u_char *data, size_t max, size_t min); -static ngx_int_t ngx_quic_split_frame(ngx_connection_t *c, ngx_quic_frame_t *f, - size_t len); -static void ngx_quic_free_frames(ngx_connection_t *c, ngx_queue_t *frames); static ssize_t ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len); static void ngx_quic_set_packet_number(ngx_quic_header_t *pkt, @@ -192,22 +177,12 @@ static ngx_chain_t *ngx_quic_stream_send static size_t ngx_quic_max_stream_flow(ngx_connection_t *c); static void ngx_quic_stream_cleanup_handler(void *data); static void ngx_quic_shutdown_quic(ngx_connection_t *c); -static void ngx_quic_free_frame(ngx_connection_t *c, ngx_quic_frame_t *frame); static void ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *frame); static void ngx_quic_congestion_lost(ngx_connection_t *c, ngx_quic_frame_t *frame); -static ngx_chain_t *ngx_quic_alloc_buf(ngx_connection_t *c); -static void ngx_quic_free_bufs(ngx_connection_t *c, ngx_chain_t *in); -static ngx_chain_t *ngx_quic_copy_buf(ngx_connection_t *c, u_char *data, - size_t len); -static ngx_chain_t *ngx_quic_copy_chain(ngx_connection_t *c, ngx_chain_t *in, - size_t limit); -static ngx_chain_t *ngx_quic_split_bufs(ngx_connection_t *c, ngx_chain_t *in, - size_t len); - static ngx_core_module_t ngx_quic_module_ctx = { ngx_string("quic"), @@ -248,225 +223,6 @@ static SSL_QUIC_METHOD quic_method = { #if (NGX_DEBUG) static void -ngx_quic_log_frame(ngx_log_t *log, ngx_quic_frame_t *f, ngx_uint_t tx) -{ - u_char *p, *last, *pos, *end; - ssize_t n; - uint64_t gap, range, largest, smallest; - ngx_uint_t i; - u_char buf[NGX_MAX_ERROR_STR]; - - p = buf; - last = buf + sizeof(buf); - - switch (f->type) { - - case NGX_QUIC_FT_CRYPTO: - p = ngx_slprintf(p, last, "CRYPTO len:%uL off:%uL", - f->u.crypto.length, f->u.crypto.offset); - break; - - case NGX_QUIC_FT_PADDING: - p = ngx_slprintf(p, last, "PADDING"); - break; - - case NGX_QUIC_FT_ACK: - case NGX_QUIC_FT_ACK_ECN: - - p = ngx_slprintf(p, last, "ACK n:%ui delay:%uL ", - f->u.ack.range_count, f->u.ack.delay); - - if (f->data) { - pos = f->data->buf->pos; - end = f->data->buf->last; - - } else { - pos = NULL; - end = NULL; - } - - largest = f->u.ack.largest; - smallest = f->u.ack.largest - f->u.ack.first_range; - - if (largest == smallest) { - p = ngx_slprintf(p, last, "%uL", largest); - - } else { - p = ngx_slprintf(p, last, "%uL-%uL", largest, smallest); - } - - for (i = 0; i < f->u.ack.range_count; i++) { - n = ngx_quic_parse_ack_range(log, pos, end, &gap, &range); - if (n == NGX_ERROR) { - break; - } - - pos += n; - - largest = smallest - gap - 2; - smallest = largest - range; - - if (largest == smallest) { - p = ngx_slprintf(p, last, " %uL", largest); - - } else { - p = ngx_slprintf(p, last, " %uL-%uL", largest, smallest); - } - } - - if (f->type == NGX_QUIC_FT_ACK_ECN) { - p = ngx_slprintf(p, last, " ECN counters ect0:%uL ect1:%uL ce:%uL", - f->u.ack.ect0, f->u.ack.ect1, f->u.ack.ce); - } - break; - - case NGX_QUIC_FT_PING: - p = ngx_slprintf(p, last, "PING"); - break; - - case NGX_QUIC_FT_NEW_CONNECTION_ID: - p = ngx_slprintf(p, last, - "NEW_CONNECTION_ID seq:%uL retire:%uL len:%ud", - f->u.ncid.seqnum, f->u.ncid.retire, f->u.ncid.len); - break; - - case NGX_QUIC_FT_RETIRE_CONNECTION_ID: - p = ngx_slprintf(p, last, "RETIRE_CONNECTION_ID seqnum:%uL", - f->u.retire_cid.sequence_number); - break; - - case NGX_QUIC_FT_CONNECTION_CLOSE: - case NGX_QUIC_FT_CONNECTION_CLOSE_APP: - p = ngx_slprintf(p, last, "CONNECTION_CLOSE%s err:%ui", - f->type == NGX_QUIC_FT_CONNECTION_CLOSE ? "" : "_APP", - f->u.close.error_code); - - if (f->u.close.reason.len) { - p = ngx_slprintf(p, last, " %V", &f->u.close.reason); - } - - if (f->type == NGX_QUIC_FT_CONNECTION_CLOSE) { - p = ngx_slprintf(p, last, " ft:%ui", f->u.close.frame_type); - } - - break; - - case NGX_QUIC_FT_STREAM0: - case NGX_QUIC_FT_STREAM1: - case NGX_QUIC_FT_STREAM2: - case NGX_QUIC_FT_STREAM3: - case NGX_QUIC_FT_STREAM4: - case NGX_QUIC_FT_STREAM5: - case NGX_QUIC_FT_STREAM6: - case NGX_QUIC_FT_STREAM7: - - p = ngx_slprintf(p, last, "STREAM id:0x%xL", f->u.stream.stream_id); - - if (f->u.stream.off) { - p = ngx_slprintf(p, last, " off:%uL", f->u.stream.offset); - } - - if (f->u.stream.len) { - p = ngx_slprintf(p, last, " len:%uL", f->u.stream.length); - } - - if (f->u.stream.fin) { - p = ngx_slprintf(p, last, " fin:1"); - } - -#ifdef NGX_QUIC_DEBUG_FRAMES - { - ngx_chain_t *cl; - - p = ngx_slprintf(p, last, " data:"); - - for (cl = f->data; cl; cl = cl->next) { - p = ngx_slprintf(p, last, "%*xs", - cl->buf->last - cl->buf->pos, cl->buf->pos); - } - } -#endif - - break; - - case NGX_QUIC_FT_MAX_DATA: - p = ngx_slprintf(p, last, "MAX_DATA max_data:%uL on recv", - f->u.max_data.max_data); - break; - - case NGX_QUIC_FT_RESET_STREAM: - p = ngx_slprintf(p, last, "RESET_STREAM" - " id:0x%xL error_code:0x%xL final_size:0x%xL", - f->u.reset_stream.id, f->u.reset_stream.error_code, - f->u.reset_stream.final_size); - break; - - case NGX_QUIC_FT_STOP_SENDING: - p = ngx_slprintf(p, last, "STOP_SENDING id:0x%xL err:0x%xL", - f->u.stop_sending.id, f->u.stop_sending.error_code); - break; - - case NGX_QUIC_FT_STREAMS_BLOCKED: - case NGX_QUIC_FT_STREAMS_BLOCKED2: - p = ngx_slprintf(p, last, "STREAMS_BLOCKED limit:%uL bidi:%ui", - f->u.streams_blocked.limit, f->u.streams_blocked.bidi); - break; - - case NGX_QUIC_FT_MAX_STREAMS: - case NGX_QUIC_FT_MAX_STREAMS2: - p = ngx_slprintf(p, last, "MAX_STREAMS limit:%uL bidi:%ui", - f->u.max_streams.limit, f->u.max_streams.bidi); - break; - - case NGX_QUIC_FT_MAX_STREAM_DATA: - p = ngx_slprintf(p, last, "MAX_STREAM_DATA id:0x%xL limit:%uL", - f->u.max_stream_data.id, f->u.max_stream_data.limit); - break; - - - case NGX_QUIC_FT_DATA_BLOCKED: - p = ngx_slprintf(p, last, "DATA_BLOCKED limit:%uL", - f->u.data_blocked.limit); - break; - - case NGX_QUIC_FT_STREAM_DATA_BLOCKED: - p = ngx_slprintf(p, last, "STREAM_DATA_BLOCKED id:0x%xL limit:%uL", - f->u.stream_data_blocked.id, - f->u.stream_data_blocked.limit); - break; - - case NGX_QUIC_FT_PATH_CHALLENGE: - p = ngx_slprintf(p, last, "PATH_CHALLENGE data:0x%*xs", - sizeof(f->u.path_challenge.data), - f->u.path_challenge.data); - break; - - case NGX_QUIC_FT_PATH_RESPONSE: - p = ngx_slprintf(p, last, "PATH_RESPONSE data:0x%*xs", - sizeof(f->u.path_challenge.data), - f->u.path_challenge.data); - break; - - case NGX_QUIC_FT_NEW_TOKEN: - p = ngx_slprintf(p, last, "NEW_TOKEN"); - break; - - case NGX_QUIC_FT_HANDSHAKE_DONE: - p = ngx_slprintf(p, last, "HANDSHAKE DONE"); - break; - - default: - p = ngx_slprintf(p, last, "unknown type 0x%xi", f->type); - break; - } - - ngx_log_debug4(NGX_LOG_DEBUG_EVENT, log, 0, "quic frame %s %s %*s", - tx ? "tx" : "rx", ngx_quic_level_name(f->level), - p - buf, buf); -} - - -static void ngx_quic_connstate_dbg(ngx_connection_t *c) { u_char *p, *last; @@ -531,7 +287,6 @@ ngx_quic_connstate_dbg(ngx_connection_t #else -#define ngx_quic_log_frame(log, f, tx) #define ngx_quic_connstate_dbg(c) #endif @@ -3430,222 +3185,6 @@ ngx_quic_handle_stream_ack(ngx_connectio static ngx_int_t -ngx_quic_handle_ordered_frame(ngx_connection_t *c, ngx_quic_frames_stream_t *fs, - ngx_quic_frame_t *frame, ngx_quic_frame_handler_pt handler, void *data) -{ - size_t full_len; - ngx_int_t rc; - ngx_queue_t *q; - ngx_quic_ordered_frame_t *f; - - f = &frame->u.ord; - - if (f->offset > fs->received) { - ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic out-of-order frame: expecting:%uL got:%uL", - fs->received, f->offset); - - return ngx_quic_buffer_frame(c, fs, frame); - } - - if (f->offset < fs->received) { - - if (ngx_quic_adjust_frame_offset(c, frame, fs->received) - == NGX_DONE) - { - /* old/duplicate data range */ - return handler == ngx_quic_crypto_input ? NGX_DECLINED : NGX_OK; - } - - /* intersecting data range, frame modified */ - } - - /* f->offset == fs->received */ - - rc = handler(c, frame, data); - if (rc == NGX_ERROR) { - return NGX_ERROR; - - } else if (rc == NGX_DONE) { - /* handler destroyed stream, queue no longer exists */ - return NGX_OK; - } - - /* rc == NGX_OK */ - - fs->received += f->length; - - /* now check the queue if we can continue with buffered frames */ - - do { - q = ngx_queue_head(&fs->frames); - if (q == ngx_queue_sentinel(&fs->frames)) { - break; - } - - frame = ngx_queue_data(q, ngx_quic_frame_t, queue); - f = &frame->u.ord; - - if (f->offset > fs->received) { - /* gap found, nothing more to do */ - break; - } - - full_len = f->length; - - if (f->offset < fs->received) { - - if (ngx_quic_adjust_frame_offset(c, frame, fs->received) - == NGX_DONE) - { - /* old/duplicate data range */ - ngx_queue_remove(q); - fs->total -= f->length; - - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic skipped buffered frame, total:%ui", - fs->total); - ngx_quic_free_frame(c, frame); - continue; - } - - /* frame was adjusted, proceed to input */ - } - - /* f->offset == fs->received */ - - rc = handler(c, frame, data); - - if (rc == NGX_ERROR) { - return NGX_ERROR; - - } else if (rc == NGX_DONE) { - /* handler destroyed stream, queue no longer exists */ - return NGX_OK; - } - - fs->received += f->length; - fs->total -= full_len; - - ngx_queue_remove(q); - - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic consumed buffered frame, total:%ui", fs->total); - - ngx_quic_free_frame(c, frame); - - } while (1); - - return NGX_OK; -} - - -static ngx_int_t -ngx_quic_adjust_frame_offset(ngx_connection_t *c, ngx_quic_frame_t *frame, - uint64_t offset_in) -{ - size_t tail, n; - ngx_buf_t *b; - ngx_chain_t *cl; - ngx_quic_ordered_frame_t *f; - - f = &frame->u.ord; - - tail = offset_in - f->offset; - - if (tail >= f->length) { - /* range preceeding already received data or duplicate, ignore */ - - ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic old or duplicate data in ordered frame, ignored"); - return NGX_DONE; - } - - ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic adjusted ordered frame data start to expected offset"); - - /* intersecting range: adjust data size */ - - f->offset += tail; - f->length -= tail; - - for (cl = frame->data; cl; cl = cl->next) { - b = cl->buf; - n = ngx_buf_size(b); - - if (n >= tail) { - b->pos += tail; - break; - } - - cl->buf->pos = cl->buf->last; - tail -= n; - } - - return NGX_OK; -} - - -static ngx_int_t -ngx_quic_buffer_frame(ngx_connection_t *c, ngx_quic_frames_stream_t *fs, - ngx_quic_frame_t *frame) -{ - ngx_queue_t *q; - ngx_quic_frame_t *dst, *item; - ngx_quic_ordered_frame_t *f, *df; - - ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic ngx_quic_buffer_frame"); - - f = &frame->u.ord; - - /* frame start offset is in the future, buffer it */ - - dst = ngx_quic_alloc_frame(c); - if (dst == NULL) { - return NGX_ERROR; - } - - ngx_memcpy(dst, frame, sizeof(ngx_quic_frame_t)); - - dst->data = ngx_quic_copy_chain(c, frame->data, 0); - if (dst->data == NGX_CHAIN_ERROR) { - return NGX_ERROR; - } - - df = &dst->u.ord; - - fs->total += f->length; - - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic ordered frame with unexpected offset:" - " buffered total:%ui", fs->total); - - if (ngx_queue_empty(&fs->frames)) { - ngx_queue_insert_after(&fs->frames, &dst->queue); - return NGX_OK; - } - - for (q = ngx_queue_last(&fs->frames); - q != ngx_queue_sentinel(&fs->frames); - q = ngx_queue_prev(q)) - { - item = ngx_queue_data(q, ngx_quic_frame_t, queue); - f = &item->u.ord; - - if (f->offset < df->offset) { - ngx_queue_insert_after(q, &dst->queue); - return NGX_OK; - } - } - - ngx_queue_insert_after(&fs->frames, &dst->queue); - - return NGX_OK; -} - - -static ngx_int_t ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_frame_t *frame) { @@ -3693,7 +3232,7 @@ ngx_quic_handle_crypto_frame(ngx_connect } -static ngx_int_t +ngx_int_t ngx_quic_crypto_input(ngx_connection_t *c, ngx_quic_frame_t *frame, void *data) { int n, sslerr; @@ -4238,26 +3777,6 @@ ngx_quic_handle_max_streams_frame(ngx_co } -void -ngx_quic_queue_frame(ngx_quic_connection_t *qc, ngx_quic_frame_t *frame) -{ - ngx_quic_send_ctx_t *ctx; - - ctx = ngx_quic_get_send_ctx(qc, frame->level); - - ngx_queue_insert_tail(&ctx->frames, &frame->queue); - - frame->len = ngx_quic_create_frame(NULL, frame); - /* always succeeds */ - - if (qc->closing) { - return; - } - - ngx_post_event(&qc->push, &ngx_posted_events); -} - - static ngx_int_t ngx_quic_output(ngx_connection_t *c) { @@ -4601,97 +4120,6 @@ ngx_quic_output_packet(ngx_connection_t } -static ngx_int_t -ngx_quic_split_frame(ngx_connection_t *c, ngx_quic_frame_t *f, size_t len) -{ - size_t shrink; - ngx_quic_frame_t *nf; - ngx_quic_ordered_frame_t *of, *onf; - - switch (f->type) { - case NGX_QUIC_FT_CRYPTO: - case NGX_QUIC_FT_STREAM0: - case NGX_QUIC_FT_STREAM1: - case NGX_QUIC_FT_STREAM2: - case NGX_QUIC_FT_STREAM3: - case NGX_QUIC_FT_STREAM4: - case NGX_QUIC_FT_STREAM5: - case NGX_QUIC_FT_STREAM6: - case NGX_QUIC_FT_STREAM7: - break; - - default: - return NGX_DECLINED; - } - - if ((size_t) f->len <= len) { - return NGX_OK; - } - - shrink = f->len - len; - - ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic split frame now:%uz need:%uz shrink:%uz", - f->len, len, shrink); - - of = &f->u.ord; - - if (of->length <= shrink) { - return NGX_DECLINED; - } - - of->length -= shrink; - f->len = ngx_quic_create_frame(NULL, f); - - if ((size_t) f->len > len) { - ngx_log_error(NGX_LOG_ERR, c->log, 0, "could not split QUIC frame"); - return NGX_ERROR; - } - - nf = ngx_quic_alloc_frame(c); - if (nf == NULL) { - return NGX_ERROR; - } - - *nf = *f; - onf = &nf->u.ord; - onf->offset += of->length; - onf->length = shrink; - nf->len = ngx_quic_create_frame(NULL, nf); - - nf->data = ngx_quic_split_bufs(c, f->data, of->length); - if (nf->data == NGX_CHAIN_ERROR) { - return NGX_ERROR; - } - - ngx_queue_insert_after(&f->queue, &nf->queue); - - return NGX_OK; -} - - -static void -ngx_quic_free_frames(ngx_connection_t *c, ngx_queue_t *frames) -{ - ngx_queue_t *q; - ngx_quic_frame_t *f; - - do { - q = ngx_queue_head(frames); - - if (q == ngx_queue_sentinel(frames)) { - break; - } - - ngx_queue_remove(q); - - f = ngx_queue_data(q, ngx_quic_frame_t, queue); - - ngx_quic_free_frame(c, f); - } while (1); -} - - static ssize_t ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len) { @@ -5836,46 +5264,6 @@ ngx_quic_shutdown_quic(ngx_connection_t } -ngx_quic_frame_t * -ngx_quic_alloc_frame(ngx_connection_t *c) -{ - ngx_queue_t *q; - ngx_quic_frame_t *frame; - ngx_quic_connection_t *qc; - - qc = ngx_quic_get_connection(c); - - if (!ngx_queue_empty(&qc->free_frames)) { - - q = ngx_queue_head(&qc->free_frames); - frame = ngx_queue_data(q, ngx_quic_frame_t, queue); - - ngx_queue_remove(&frame->queue); - -#ifdef NGX_QUIC_DEBUG_ALLOC - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic reuse frame n:%ui", qc->nframes); -#endif - - } else { - frame = ngx_pcalloc(c->pool, sizeof(ngx_quic_frame_t)); - if (frame == NULL) { - return NULL; - } - -#ifdef NGX_QUIC_DEBUG_ALLOC - ++qc->nframes; - - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic alloc frame n:%ui", qc->nframes); -#endif - } - - ngx_memzero(frame, sizeof(ngx_quic_frame_t)); - - return frame; -} - static void ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f) @@ -5970,26 +5358,6 @@ ngx_quic_congestion_lost(ngx_connection_ } -static void -ngx_quic_free_frame(ngx_connection_t *c, ngx_quic_frame_t *frame) -{ - ngx_quic_connection_t *qc; - - qc = ngx_quic_get_connection(c); - - if (frame->data) { - ngx_quic_free_bufs(c, frame->data); - } - - ngx_queue_insert_head(&qc->free_frames, &frame->queue); - -#ifdef NGX_QUIC_DEBUG_ALLOC - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic free frame n:%ui", qc->nframes); -#endif -} - - uint32_t ngx_quic_version(ngx_connection_t *c) { @@ -6002,278 +5370,3 @@ ngx_quic_version(ngx_connection_t *c) return (version & 0xff000000) == 0xff000000 ? version & 0xff : version; } - - -static ngx_chain_t * -ngx_quic_alloc_buf(ngx_connection_t *c) -{ - ngx_buf_t *b; - ngx_chain_t *cl; - ngx_quic_connection_t *qc; - - qc = ngx_quic_get_connection(c); - - if (qc->free_bufs) { - cl = qc->free_bufs; - qc->free_bufs = cl->next; - - b = cl->buf; - b->pos = b->start; - b->last = b->start; - -#ifdef NGX_QUIC_DEBUG_ALLOC - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic reuse buffer n:%ui", qc->nbufs); -#endif - - return cl; - } - - cl = ngx_alloc_chain_link(c->pool); - if (cl == NULL) { - return NULL; - } - - b = ngx_create_temp_buf(c->pool, NGX_QUIC_BUFFER_SIZE); - if (b == NULL) { - return NULL; - } - - b->tag = (ngx_buf_tag_t) &ngx_quic_alloc_buf; - - cl->buf = b; - -#ifdef NGX_QUIC_DEBUG_ALLOC - ++qc->nbufs; - - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic alloc buffer n:%ui", qc->nbufs); -#endif - - return cl; -} - - -static void -ngx_quic_free_bufs(ngx_connection_t *c, ngx_chain_t *in) -{ - ngx_buf_t *b, *shadow; - ngx_chain_t *cl; - ngx_quic_connection_t *qc; - - qc = ngx_quic_get_connection(c); - - while (in) { -#ifdef NGX_QUIC_DEBUG_ALLOC - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic free buffer n:%ui", qc->nbufs); -#endif - - cl = in; - in = in->next; - b = cl->buf; - - if (b->shadow) { - if (!b->last_shadow) { - b->recycled = 1; - ngx_free_chain(c->pool, cl); - continue; - } - - do { - shadow = b->shadow; - b->shadow = qc->free_shadow_bufs; - qc->free_shadow_bufs = b; - b = shadow; - } while (b->recycled); - - if (b->shadow) { - b->last_shadow = 1; - ngx_free_chain(c->pool, cl); - continue; - } - - cl->buf = b; - } - - cl->next = qc->free_bufs; - qc->free_bufs = cl; - } -} - - -static ngx_chain_t * -ngx_quic_copy_buf(ngx_connection_t *c, u_char *data, size_t len) -{ - size_t n; - ngx_buf_t *b; - ngx_chain_t *cl, *out, **ll; - - out = NULL; - ll = &out; - - while (len) { - cl = ngx_quic_alloc_buf(c); - if (cl == NULL) { - return NGX_CHAIN_ERROR; - } - - b = cl->buf; - n = ngx_min((size_t) (b->end - b->last), len); - - b->last = ngx_cpymem(b->last, data, n); - - data += n; - len -= n; - - *ll = cl; - ll = &cl->next; - } - - *ll = NULL; - - return out; -} - - -static ngx_chain_t * -ngx_quic_copy_chain(ngx_connection_t *c, ngx_chain_t *in, size_t limit) -{ - size_t n; - ngx_buf_t *b; - ngx_chain_t *cl, *out, **ll; - - out = NULL; - ll = &out; - - while (in) { - if (!ngx_buf_in_memory(in->buf) || ngx_buf_size(in->buf) == 0) { - in = in->next; - continue; - } - - cl = ngx_quic_alloc_buf(c); - if (cl == NULL) { - return NGX_CHAIN_ERROR; - } - - *ll = cl; - ll = &cl->next; - - b = cl->buf; - - while (in && b->last != b->end) { - - n = ngx_min(in->buf->last - in->buf->pos, b->end - b->last); - - if (limit > 0 && n > limit) { - n = limit; - } - - b->last = ngx_cpymem(b->last, in->buf->pos, n); - - in->buf->pos += n; - if (in->buf->pos == in->buf->last) { - in = in->next; - } - - if (limit > 0) { - if (limit == n) { - goto done; - } - - limit -= n; - } - } - - } - -done: - - *ll = NULL; - - return out; -} - - -static ngx_chain_t * -ngx_quic_split_bufs(ngx_connection_t *c, ngx_chain_t *in, size_t len) -{ - size_t n; - ngx_buf_t *b; - ngx_chain_t *out; - ngx_quic_connection_t *qc; - - qc = ngx_quic_get_connection(c); - - while (in) { - n = ngx_buf_size(in->buf); - - if (n == len) { - out = in->next; - in->next = NULL; - return out; - } - - if (n > len) { - break; - } - - len -= n; - in = in->next; - } - - if (in == NULL) { - return NULL; - } - - /* split in->buf by creating shadow bufs which reference it */ - - if (in->buf->shadow == NULL) { - if (qc->free_shadow_bufs) { - b = qc->free_shadow_bufs; - qc->free_shadow_bufs = b->shadow; - - } else { - b = ngx_alloc_buf(c->pool); - if (b == NULL) { - return NGX_CHAIN_ERROR; - } - } - - *b = *in->buf; - b->shadow = in->buf; - b->last_shadow = 1; - in->buf = b; - } - - out = ngx_alloc_chain_link(c->pool); - if (out == NULL) { - return NGX_CHAIN_ERROR; - } - - if (qc->free_shadow_bufs) { - b = qc->free_shadow_bufs; - qc->free_shadow_bufs = b->shadow; - - } else { - b = ngx_alloc_buf(c->pool); - if (b == NULL) { - ngx_free_chain(c->pool, out); - return NGX_CHAIN_ERROR; - } - } - - out->buf = b; - out->next = in->next; - in->next = NULL; - - *b = *in->buf; - b->last_shadow = 0; - b->pos = b->pos + len; - - in->buf->shadow = b; - in->buf->last = in->buf->pos + len; - - return out; -}