Mercurial > hg > nginx

comparison src/event/quic/ngx_event_quic.c @ 8673:046c951e393a quic

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
QUIC: moved all quic sources into src/event/quic.
author Vladimir Homutov <vl@nginx.com>
date Fri, 25 Dec 2020 14:01:28 +0300
parents src/event/ngx_event_quic.c@b14338acbf9d
children 2c7f927f7999
comparison
equal deleted inserted replaced
8672:13c537def699 8673:046c951e393a
1
2 /*
3 * Copyright (C) Nginx, Inc.
4 */
5
6
7 #include <ngx_config.h>
8 #include <ngx_core.h>
9 #include <ngx_event.h>
10 #include <ngx_event_quic_transport.h>
11 #include <ngx_event_quic_protection.h>
12
13
14 /* 0-RTT and 1-RTT data exist in the same packet number space,
15 * so we have 3 packet number spaces:
16 *
17 * 0 - Initial
18 * 1 - Handshake
19 * 2 - 0-RTT and 1-RTT
20 */
21 #define ngx_quic_get_send_ctx(qc, level) \
22 ((level) == ssl_encryption_initial) ? &((qc)->send_ctx[0]) \
23 : (((level) == ssl_encryption_handshake) ? &((qc)->send_ctx[1]) \
24 : &((qc)->send_ctx[2]))
25
26 #define NGX_QUIC_SEND_CTX_LAST (NGX_QUIC_ENCRYPTION_LAST - 1)
27
28 /*
29 * 7.4. Cryptographic Message Buffering
30 * Implementations MUST support buffering at least 4096 bytes of data
31 */
32 #define NGX_QUIC_MAX_BUFFERED 65535
33
34 #define NGX_QUIC_STREAM_GONE (void *) -1
35
36 #define NGX_QUIC_UNSET_PN (uint64_t) -1
37
38 /*
39 * Endpoints MUST discard packets that are too small to be valid QUIC
40 * packets. With the set of AEAD functions defined in [QUIC-TLS],
41 * packets that are smaller than 21 bytes are never valid.
42 */
43 #define NGX_QUIC_MIN_PKT_LEN 21
44
45 #define NGX_QUIC_MIN_SR_PACKET 43 /* 5 random + 16 srt + 22 padding */
46 #define NGX_QUIC_MAX_SR_PACKET 1200
47
48 #define NGX_QUIC_MAX_ACK_GAP 2
49
50
51 typedef struct {
52 ngx_rbtree_t tree;
53 ngx_rbtree_node_t sentinel;
54
55 uint64_t received;
56 uint64_t sent;
57 uint64_t recv_max_data;
58 uint64_t send_max_data;
59
60 uint64_t server_max_streams_uni;
61 uint64_t server_max_streams_bidi;
62 uint64_t server_streams_uni;
63 uint64_t server_streams_bidi;
64
65 uint64_t client_max_streams_uni;
66 uint64_t client_max_streams_bidi;
67 uint64_t client_streams_uni;
68 uint64_t client_streams_bidi;
69 } ngx_quic_streams_t;
70
71
72 typedef struct {
73 size_t in_flight;
74 size_t window;
75 size_t ssthresh;
76 ngx_msec_t recovery_start;
77 } ngx_quic_congestion_t;
78
79
80 /*
81 * 12.3. Packet Numbers
82 *
83 * Conceptually, a packet number space is the context in which a packet
84 * can be processed and acknowledged. Initial packets can only be sent
85 * with Initial packet protection keys and acknowledged in packets which
86 * are also Initial packets.
87 */
88 typedef struct {
89 enum ssl_encryption_level_t level;
90
91 uint64_t pnum; /* to be sent */
92 uint64_t largest_ack; /* received from peer */
93 uint64_t largest_pn; /* received from peer */
94
95 ngx_queue_t frames;
96 ngx_queue_t sent;
97
98 uint64_t pending_ack; /* non sent ack-eliciting */
99 uint64_t largest_range;
100 uint64_t first_range;
101 ngx_msec_t largest_received;
102 ngx_msec_t ack_delay_start;
103 ngx_uint_t nranges;
104 ngx_quic_ack_range_t ranges[NGX_QUIC_MAX_RANGES];
105 ngx_uint_t send_ack;
106 } ngx_quic_send_ctx_t;
107
108
109 typedef struct {
110 ngx_udp_connection_t udp;
111
112 uint32_t version;
113 ngx_str_t scid; /* initial client ID */
114 ngx_str_t dcid; /* server (our own) ID */
115 ngx_str_t odcid; /* original server ID */
116 ngx_str_t token;
117
118 struct sockaddr *sockaddr;
119 socklen_t socklen;
120
121 ngx_queue_t client_ids;
122 ngx_queue_t server_ids;
123 ngx_queue_t free_client_ids;
124 ngx_queue_t free_server_ids;
125 ngx_uint_t nclient_ids;
126 ngx_uint_t nserver_ids;
127 uint64_t max_retired_seqnum;
128 uint64_t client_seqnum;
129 uint64_t server_seqnum;
130
131 ngx_uint_t client_tp_done;
132 ngx_quic_tp_t tp;
133 ngx_quic_tp_t ctp;
134
135 ngx_quic_send_ctx_t send_ctx[NGX_QUIC_SEND_CTX_LAST];
136
137 ngx_quic_frames_stream_t crypto[NGX_QUIC_ENCRYPTION_LAST];
138
139 ngx_quic_keys_t *keys;
140
141 ngx_quic_conf_t *conf;
142
143 ngx_event_t push;
144 ngx_event_t pto;
145 ngx_event_t close;
146 ngx_msec_t last_cc;
147
148 ngx_msec_t latest_rtt;
149 ngx_msec_t avg_rtt;
150 ngx_msec_t min_rtt;
151 ngx_msec_t rttvar;
152
153 ngx_uint_t pto_count;
154
155 ngx_queue_t free_frames;
156 ngx_chain_t *free_bufs;
157 ngx_buf_t *free_shadow_bufs;
158
159 #ifdef NGX_QUIC_DEBUG_ALLOC
160 ngx_uint_t nframes;
161 ngx_uint_t nbufs;
162 #endif
163
164 ngx_quic_streams_t streams;
165 ngx_quic_congestion_t congestion;
166 off_t received;
167
168 ngx_uint_t error;
169 enum ssl_encryption_level_t error_level;
170 ngx_uint_t error_ftype;
171 const char *error_reason;
172
173 unsigned error_app:1;
174 unsigned send_timer_set:1;
175 unsigned closing:1;
176 unsigned draining:1;
177 unsigned key_phase:1;
178 unsigned in_retry:1;
179 unsigned initialized:1;
180 unsigned validated:1;
181 } ngx_quic_connection_t;
182
183
184 typedef struct {
185 ngx_queue_t queue;
186 uint64_t seqnum;
187 size_t len;
188 u_char id[NGX_QUIC_CID_LEN_MAX];
189 u_char sr_token[NGX_QUIC_SR_TOKEN_LEN];
190 } ngx_quic_client_id_t;
191
192
193 typedef struct {
194 ngx_udp_connection_t udp;
195 ngx_queue_t queue;
196 uint64_t seqnum;
197 size_t len;
198 u_char id[NGX_QUIC_CID_LEN_MAX];
199 } ngx_quic_server_id_t;
200
201
202 typedef ngx_int_t (*ngx_quic_frame_handler_pt)(ngx_connection_t *c,
203 ngx_quic_frame_t *frame, void *data);
204
205
206 #if BORINGSSL_API_VERSION >= 10
207 static int ngx_quic_set_read_secret(ngx_ssl_conn_t *ssl_conn,
208 enum ssl_encryption_level_t level, const SSL_CIPHER *cipher,
209 const uint8_t *secret, size_t secret_len);
210 static int ngx_quic_set_write_secret(ngx_ssl_conn_t *ssl_conn,
211 enum ssl_encryption_level_t level, const SSL_CIPHER *cipher,
212 const uint8_t *secret, size_t secret_len);
213 #else
214 static int ngx_quic_set_encryption_secrets(ngx_ssl_conn_t *ssl_conn,
215 enum ssl_encryption_level_t level, const uint8_t *read_secret,
216 const uint8_t *write_secret, size_t secret_len);
217 #endif
218
219 static int ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn,
220 enum ssl_encryption_level_t level, const uint8_t *data, size_t len);
221 static int ngx_quic_flush_flight(ngx_ssl_conn_t *ssl_conn);
222 static int ngx_quic_send_alert(ngx_ssl_conn_t *ssl_conn,
223 enum ssl_encryption_level_t level, uint8_t alert);
224
225
226 static ngx_quic_connection_t *ngx_quic_new_connection(ngx_connection_t *c,
227 ngx_quic_conf_t *conf, ngx_quic_header_t *pkt);
228 static ngx_int_t ngx_quic_send_stateless_reset(ngx_connection_t *c,
229 ngx_quic_conf_t *conf, ngx_quic_header_t *pkt);
230 static ngx_int_t ngx_quic_process_stateless_reset(ngx_connection_t *c,
231 ngx_quic_header_t *pkt);
232 static ngx_int_t ngx_quic_negotiate_version(ngx_connection_t *c,
233 ngx_quic_header_t *inpkt);
234 static ngx_int_t ngx_quic_create_server_id(ngx_connection_t *c, u_char *id);
235 static ngx_int_t ngx_quic_send_retry(ngx_connection_t *c);
236 static ngx_int_t ngx_quic_new_token(ngx_connection_t *c, ngx_str_t *token);
237 static ngx_int_t ngx_quic_validate_token(ngx_connection_t *c,
238 ngx_quic_header_t *pkt);
239 static ngx_int_t ngx_quic_init_connection(ngx_connection_t *c);
240 static ngx_inline size_t ngx_quic_max_udp_payload(ngx_connection_t *c);
241 static void ngx_quic_input_handler(ngx_event_t *rev);
242
243 static void ngx_quic_close_connection(ngx_connection_t *c, ngx_int_t rc);
244 static ngx_int_t ngx_quic_close_quic(ngx_connection_t *c, ngx_int_t rc);
245 static void ngx_quic_close_timer_handler(ngx_event_t *ev);
246 static ngx_int_t ngx_quic_close_streams(ngx_connection_t *c,
247 ngx_quic_connection_t *qc);
248
249 static ngx_int_t ngx_quic_input(ngx_connection_t *c, ngx_buf_t *b,
250 ngx_quic_conf_t *conf);
251 static ngx_int_t ngx_quic_process_packet(ngx_connection_t *c,
252 ngx_quic_conf_t *conf, ngx_quic_header_t *pkt);
253 static ngx_int_t ngx_quic_init_secrets(ngx_connection_t *c);
254 static void ngx_quic_discard_ctx(ngx_connection_t *c,
255 enum ssl_encryption_level_t level);
256 static ngx_int_t ngx_quic_check_peer(ngx_quic_connection_t *qc,
257 ngx_quic_header_t *pkt);
258 static ngx_int_t ngx_quic_payload_handler(ngx_connection_t *c,
259 ngx_quic_header_t *pkt);
260 static ngx_int_t ngx_quic_ack_packet(ngx_connection_t *c,
261 ngx_quic_header_t *pkt);
262 static ngx_int_t ngx_quic_send_ack_range(ngx_connection_t *c,
263 ngx_quic_send_ctx_t *ctx, uint64_t smallest, uint64_t largest);
264 static void ngx_quic_drop_ack_ranges(ngx_connection_t *c,
265 ngx_quic_send_ctx_t *ctx, uint64_t pn);
266 static ngx_int_t ngx_quic_send_ack(ngx_connection_t *c,
267 ngx_quic_send_ctx_t *ctx);
268 static ngx_int_t ngx_quic_send_cc(ngx_connection_t *c);
269 static ngx_int_t ngx_quic_send_new_token(ngx_connection_t *c);
270
271 static ngx_int_t ngx_quic_handle_ack_frame(ngx_connection_t *c,
272 ngx_quic_header_t *pkt, ngx_quic_frame_t *f);
273 static ngx_int_t ngx_quic_handle_ack_frame_range(ngx_connection_t *c,
274 ngx_quic_send_ctx_t *ctx, uint64_t min, uint64_t max,
275 ngx_msec_t *send_time);
276 static void ngx_quic_rtt_sample(ngx_connection_t *c, ngx_quic_ack_frame_t *ack,
277 enum ssl_encryption_level_t level, ngx_msec_t send_time);
278 static ngx_inline ngx_msec_t ngx_quic_pto(ngx_connection_t *c,
279 ngx_quic_send_ctx_t *ctx);
280 static void ngx_quic_handle_stream_ack(ngx_connection_t *c,
281 ngx_quic_frame_t *f);
282
283 static ngx_int_t ngx_quic_handle_ordered_frame(ngx_connection_t *c,
284 ngx_quic_frames_stream_t *fs, ngx_quic_frame_t *frame,
285 ngx_quic_frame_handler_pt handler, void *data);
286 static ngx_int_t ngx_quic_adjust_frame_offset(ngx_connection_t *c,
287 ngx_quic_frame_t *f, uint64_t offset_in);
288 static ngx_int_t ngx_quic_buffer_frame(ngx_connection_t *c,
289 ngx_quic_frames_stream_t *stream, ngx_quic_frame_t *f);
290
291 static ngx_int_t ngx_quic_handle_crypto_frame(ngx_connection_t *c,
292 ngx_quic_header_t *pkt, ngx_quic_frame_t *frame);
293 static ngx_int_t ngx_quic_crypto_input(ngx_connection_t *c,
294 ngx_quic_frame_t *frame, void *data);
295 static ngx_int_t ngx_quic_handle_stream_frame(ngx_connection_t *c,
296 ngx_quic_header_t *pkt, ngx_quic_frame_t *frame);
297 static ngx_int_t ngx_quic_stream_input(ngx_connection_t *c,
298 ngx_quic_frame_t *frame, void *data);
299
300 static ngx_int_t ngx_quic_handle_max_data_frame(ngx_connection_t *c,
301 ngx_quic_max_data_frame_t *f);
302 static ngx_int_t ngx_quic_handle_streams_blocked_frame(ngx_connection_t *c,
303 ngx_quic_header_t *pkt, ngx_quic_streams_blocked_frame_t *f);
304 static ngx_int_t ngx_quic_handle_stream_data_blocked_frame(ngx_connection_t *c,
305 ngx_quic_header_t *pkt, ngx_quic_stream_data_blocked_frame_t *f);
306 static ngx_int_t ngx_quic_handle_max_stream_data_frame(ngx_connection_t *c,
307 ngx_quic_header_t *pkt, ngx_quic_max_stream_data_frame_t *f);
308 static ngx_int_t ngx_quic_handle_reset_stream_frame(ngx_connection_t *c,
309 ngx_quic_header_t *pkt, ngx_quic_reset_stream_frame_t *f);
310 static ngx_int_t ngx_quic_handle_stop_sending_frame(ngx_connection_t *c,
311 ngx_quic_header_t *pkt, ngx_quic_stop_sending_frame_t *f);
312 static ngx_int_t ngx_quic_handle_max_streams_frame(ngx_connection_t *c,
313 ngx_quic_header_t *pkt, ngx_quic_max_streams_frame_t *f);
314 static ngx_int_t ngx_quic_handle_path_challenge_frame(ngx_connection_t *c,
315 ngx_quic_header_t *pkt, ngx_quic_path_challenge_frame_t *f);
316 static ngx_int_t ngx_quic_handle_new_connection_id_frame(ngx_connection_t *c,
317 ngx_quic_header_t *pkt, ngx_quic_new_conn_id_frame_t *f);
318 static ngx_int_t ngx_quic_retire_connection_id(ngx_connection_t *c,
319 enum ssl_encryption_level_t level, uint64_t seqnum);
320 static ngx_int_t ngx_quic_handle_retire_connection_id_frame(ngx_connection_t *c,
321 ngx_quic_header_t *pkt, ngx_quic_retire_cid_frame_t *f);
322 static ngx_int_t ngx_quic_issue_server_ids(ngx_connection_t *c);
323 static void ngx_quic_clear_temp_server_ids(ngx_connection_t *c);
324 static ngx_quic_server_id_t *ngx_quic_insert_server_id(ngx_connection_t *c,
325 ngx_str_t *id);
326 static ngx_quic_client_id_t *ngx_quic_alloc_client_id(ngx_connection_t *c,
327 ngx_quic_connection_t *qc);
328 static ngx_quic_server_id_t *ngx_quic_alloc_server_id(ngx_connection_t *c,
329 ngx_quic_connection_t *qc);
330
331 static void ngx_quic_queue_frame(ngx_quic_connection_t *qc,
332 ngx_quic_frame_t *frame);
333
334 static ngx_int_t ngx_quic_output(ngx_connection_t *c);
335 static ngx_uint_t ngx_quic_get_padding_level(ngx_connection_t *c);
336 static ngx_int_t ngx_quic_generate_ack(ngx_connection_t *c,
337 ngx_quic_send_ctx_t *ctx);
338 static ssize_t ngx_quic_output_packet(ngx_connection_t *c,
339 ngx_quic_send_ctx_t *ctx, u_char *data, size_t max, size_t min);
340 static ngx_int_t ngx_quic_split_frame(ngx_connection_t *c, ngx_quic_frame_t *f,
341 size_t len);
342 static void ngx_quic_free_frames(ngx_connection_t *c, ngx_queue_t *frames);
343 static ssize_t ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len);
344
345 static void ngx_quic_set_packet_number(ngx_quic_header_t *pkt,
346 ngx_quic_send_ctx_t *ctx);
347 static void ngx_quic_pto_handler(ngx_event_t *ev);
348 static void ngx_quic_lost_handler(ngx_event_t *ev);
349 static ngx_int_t ngx_quic_detect_lost(ngx_connection_t *c);
350 static void ngx_quic_resend_frames(ngx_connection_t *c,
351 ngx_quic_send_ctx_t *ctx);
352 static void ngx_quic_push_handler(ngx_event_t *ev);
353
354 static void ngx_quic_rbtree_insert_stream(ngx_rbtree_node_t *temp,
355 ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel);
356 static ngx_quic_stream_t *ngx_quic_find_stream(ngx_rbtree_t *rbtree,
357 uint64_t id);
358 static ngx_quic_stream_t *ngx_quic_create_client_stream(ngx_connection_t *c,
359 uint64_t id);
360 static ngx_quic_stream_t *ngx_quic_create_stream(ngx_connection_t *c,
361 uint64_t id, size_t rcvbuf_size);
362 static ssize_t ngx_quic_stream_recv(ngx_connection_t *c, u_char *buf,
363 size_t size);
364 static ssize_t ngx_quic_stream_send(ngx_connection_t *c, u_char *buf,
365 size_t size);
366 static ngx_chain_t *ngx_quic_stream_send_chain(ngx_connection_t *c,
367 ngx_chain_t *in, off_t limit);
368 static size_t ngx_quic_max_stream_flow(ngx_connection_t *c);
369 static void ngx_quic_stream_cleanup_handler(void *data);
370 static ngx_quic_frame_t *ngx_quic_alloc_frame(ngx_connection_t *c);
371 static void ngx_quic_free_frame(ngx_connection_t *c, ngx_quic_frame_t *frame);
372
373 static void ngx_quic_congestion_ack(ngx_connection_t *c,
374 ngx_quic_frame_t *frame);
375 static void ngx_quic_congestion_lost(ngx_connection_t *c,
376 ngx_quic_frame_t *frame);
377
378 static ngx_chain_t *ngx_quic_alloc_buf(ngx_connection_t *c);
379 static void ngx_quic_free_bufs(ngx_connection_t *c, ngx_chain_t *in);
380 static ngx_chain_t *ngx_quic_copy_buf(ngx_connection_t *c, u_char *data,
381 size_t len);
382 static ngx_chain_t *ngx_quic_copy_chain(ngx_connection_t *c, ngx_chain_t *in,
383 size_t limit);
384 static ngx_chain_t *ngx_quic_split_bufs(ngx_connection_t *c, ngx_chain_t *in,
385 size_t len);
386
387
388 static SSL_QUIC_METHOD quic_method = {
389 #if BORINGSSL_API_VERSION >= 10
390 ngx_quic_set_read_secret,
391 ngx_quic_set_write_secret,
392 #else
393 ngx_quic_set_encryption_secrets,
394 #endif
395 ngx_quic_add_handshake_data,
396 ngx_quic_flush_flight,
397 ngx_quic_send_alert,
398 };
399
400
401 #if (NGX_DEBUG)
402
403 static void
404 ngx_quic_log_frame(ngx_log_t *log, ngx_quic_frame_t *f, ngx_uint_t tx)
405 {
406 u_char *p, *last, *pos, *end;
407 ssize_t n;
408 uint64_t gap, range, largest, smallest;
409 ngx_uint_t i;
410 u_char buf[NGX_MAX_ERROR_STR];
411
412 p = buf;
413 last = buf + sizeof(buf);
414
415 switch (f->type) {
416
417 case NGX_QUIC_FT_CRYPTO:
418 p = ngx_slprintf(p, last, "CRYPTO len:%uL off:%uL",
419 f->u.crypto.length, f->u.crypto.offset);
420 break;
421
422 case NGX_QUIC_FT_PADDING:
423 p = ngx_slprintf(p, last, "PADDING");
424 break;
425
426 case NGX_QUIC_FT_ACK:
427 case NGX_QUIC_FT_ACK_ECN:
428
429 p = ngx_slprintf(p, last, "ACK n:%ui delay:%uL ",
430 f->u.ack.range_count, f->u.ack.delay);
431
432 if (f->data) {
433 pos = f->data->buf->pos;
434 end = f->data->buf->end;
435
436 } else {
437 pos = NULL;
438 end = NULL;
439 }
440
441 largest = f->u.ack.largest;
442 smallest = f->u.ack.largest - f->u.ack.first_range;
443
444 if (largest == smallest) {
445 p = ngx_slprintf(p, last, "%uL", largest);
446
447 } else {
448 p = ngx_slprintf(p, last, "%uL-%uL", largest, smallest);
449 }
450
451 for (i = 0; i < f->u.ack.range_count; i++) {
452 n = ngx_quic_parse_ack_range(log, pos, end, &gap, &range);
453 if (n == NGX_ERROR) {
454 break;
455 }
456
457 pos += n;
458
459 largest = smallest - gap - 2;
460 smallest = largest - range;
461
462 if (largest == smallest) {
463 p = ngx_slprintf(p, last, " %uL", largest);
464
465 } else {
466 p = ngx_slprintf(p, last, " %uL-%uL", largest, smallest);
467 }
468 }
469
470 if (f->type == NGX_QUIC_FT_ACK_ECN) {
471 p = ngx_slprintf(p, last, " ECN counters ect0:%uL ect1:%uL ce:%uL",
472 f->u.ack.ect0, f->u.ack.ect1, f->u.ack.ce);
473 }
474 break;
475
476 case NGX_QUIC_FT_PING:
477 p = ngx_slprintf(p, last, "PING");
478 break;
479
480 case NGX_QUIC_FT_NEW_CONNECTION_ID:
481 p = ngx_slprintf(p, last,
482 "NEW_CONNECTION_ID seq:%uL retire:%uL len:%ud",
483 f->u.ncid.seqnum, f->u.ncid.retire, f->u.ncid.len);
484 break;
485
486 case NGX_QUIC_FT_RETIRE_CONNECTION_ID:
487 p = ngx_slprintf(p, last, "RETIRE_CONNECTION_ID seqnum:%uL",
488 f->u.retire_cid.sequence_number);
489 break;
490
491 case NGX_QUIC_FT_CONNECTION_CLOSE:
492 case NGX_QUIC_FT_CONNECTION_CLOSE_APP:
493 p = ngx_slprintf(p, last, "CONNECTION_CLOSE%s err:%ui",
494 f->u.close.app ? "_APP" : "", f->u.close.error_code);
495
496 if (f->u.close.reason.len) {
497 p = ngx_slprintf(p, last, " %V", &f->u.close.reason);
498 }
499
500 if (f->type == NGX_QUIC_FT_CONNECTION_CLOSE) {
501 p = ngx_slprintf(p, last, " ft:%ui", f->u.close.frame_type);
502 }
503
504
505 break;
506
507 case NGX_QUIC_FT_STREAM0:
508 case NGX_QUIC_FT_STREAM1:
509 case NGX_QUIC_FT_STREAM2:
510 case NGX_QUIC_FT_STREAM3:
511 case NGX_QUIC_FT_STREAM4:
512 case NGX_QUIC_FT_STREAM5:
513 case NGX_QUIC_FT_STREAM6:
514 case NGX_QUIC_FT_STREAM7:
515
516 p = ngx_slprintf(p, last, "STREAM id:0x%xL", f->u.stream.stream_id);
517
518 if (f->u.stream.off) {
519 p = ngx_slprintf(p, last, " off:%uL", f->u.stream.offset);
520 }
521
522 if (f->u.stream.len) {
523 p = ngx_slprintf(p, last, " len:%uL", f->u.stream.length);
524 }
525
526 if (f->u.stream.fin) {
527 p = ngx_slprintf(p, last, " fin:1");
528 }
529
530 #ifdef NGX_QUIC_DEBUG_FRAMES
531 {
532 ngx_chain_t *cl;
533
534 p = ngx_slprintf(p, last, " data:");
535
536 for (cl = f->data; cl; cl = cl->next) {
537 p = ngx_slprintf(p, last, "%*xs",
538 cl->buf->last - cl->buf->pos, cl->buf->pos);
539 }
540 }
541 #endif
542
543 break;
544
545 case NGX_QUIC_FT_MAX_DATA:
546 p = ngx_slprintf(p, last, "MAX_DATA max_data:%uL on recv",
547 f->u.max_data.max_data);
548 break;
549
550 case NGX_QUIC_FT_RESET_STREAM:
551 p = ngx_slprintf(p, last, "RESET_STREAM"
552 " id:0x%xL error_code:0x%xL final_size:0x%xL",
553 f->u.reset_stream.id, f->u.reset_stream.error_code,
554 f->u.reset_stream.final_size);
555 break;
556
557 case NGX_QUIC_FT_STOP_SENDING:
558 p = ngx_slprintf(p, last, "STOP_SENDING id:0x%xL err:0x%xL",
559 f->u.stop_sending.id, f->u.stop_sending.error_code);
560 break;
561
562 case NGX_QUIC_FT_STREAMS_BLOCKED:
563 case NGX_QUIC_FT_STREAMS_BLOCKED2:
564 p = ngx_slprintf(p, last, "STREAMS_BLOCKED limit:%uL bidi:%ui",
565 f->u.streams_blocked.limit, f->u.streams_blocked.bidi);
566 break;
567
568 case NGX_QUIC_FT_MAX_STREAMS:
569 case NGX_QUIC_FT_MAX_STREAMS2:
570 p = ngx_slprintf(p, last, "MAX_STREAMS limit:%uL bidi:%ui",
571 f->u.max_streams.limit, f->u.max_streams.bidi);
572 break;
573
574 case NGX_QUIC_FT_MAX_STREAM_DATA:
575 p = ngx_slprintf(p, last, "MAX_STREAM_DATA id:0x%xL limit:%uL",
576 f->u.max_stream_data.id, f->u.max_stream_data.limit);
577 break;
578
579
580 case NGX_QUIC_FT_DATA_BLOCKED:
581 p = ngx_slprintf(p, last, "DATA_BLOCKED limit:%uL",
582 f->u.data_blocked.limit);
583 break;
584
585 case NGX_QUIC_FT_STREAM_DATA_BLOCKED:
586 p = ngx_slprintf(p, last, "STREAM_DATA_BLOCKED id:0x%xL limit:%uL",
587 f->u.stream_data_blocked.id,
588 f->u.stream_data_blocked.limit);
589 break;
590
591 case NGX_QUIC_FT_PATH_CHALLENGE:
592 p = ngx_slprintf(p, last, "PATH_CHALLENGE data:0x%*xs",
593 sizeof(f->u.path_challenge.data),
594 f->u.path_challenge.data);
595 break;
596
597 case NGX_QUIC_FT_PATH_RESPONSE:
598 p = ngx_slprintf(p, last, "PATH_RESPONSE data:0x%*xs",
599 sizeof(f->u.path_challenge.data),
600 f->u.path_challenge.data);
601 break;
602
603 case NGX_QUIC_FT_NEW_TOKEN:
604 p = ngx_slprintf(p, last, "NEW_TOKEN");
605 break;
606
607 case NGX_QUIC_FT_HANDSHAKE_DONE:
608 p = ngx_slprintf(p, last, "HANDSHAKE DONE");
609 break;
610
611 default:
612 p = ngx_slprintf(p, last, "unknown type 0x%xi", f->type);
613 break;
614 }
615
616 ngx_log_debug4(NGX_LOG_DEBUG_EVENT, log, 0, "quic frame %s %s %*s",
617 tx ? "tx" : "rx", ngx_quic_level_name(f->level),
618 p - buf, buf);
619 }
620
621
622 static void
623 ngx_quic_connstate_dbg(ngx_connection_t *c)
624 {
625 u_char *p, *last;
626 ngx_quic_connection_t *qc;
627 u_char buf[NGX_MAX_ERROR_STR];
628
629 p = buf;
630 last = p + sizeof(buf);
631
632 qc = ngx_quic_get_connection(c);
633
634 p = ngx_slprintf(p, last, "state:");
635
636 if (qc) {
637
638 if (qc->error) {
639 p = ngx_slprintf(p, last, "%s", qc->error_app ? " app" : "");
640 p = ngx_slprintf(p, last, " error:%ui", qc->error);
641
642 if (qc->error_reason) {
643 p = ngx_slprintf(p, last, " \"%s\"", qc->error_reason);
644 }
645 }
646
647 p = ngx_slprintf(p, last, "%s", qc->closing ? " closing" : "");
648 p = ngx_slprintf(p, last, "%s", qc->draining ? " draining" : "");
649 p = ngx_slprintf(p, last, "%s", qc->key_phase ? " kp" : "");
650 p = ngx_slprintf(p, last, "%s", qc->in_retry ? " retry" : "");
651 p = ngx_slprintf(p, last, "%s", qc->validated? " valid" : "");
652
653 } else {
654 p = ngx_slprintf(p, last, " early");
655 }
656
657 if (c->read->timer_set) {
658 p = ngx_slprintf(p, last,
659 qc && qc->send_timer_set ? " send:%M" : " read:%M",
660 c->read->timer.key - ngx_current_msec);
661 }
662
663 if (qc) {
664
665 if (qc->push.timer_set) {
666 p = ngx_slprintf(p, last, " push:%M",
667 qc->push.timer.key - ngx_current_msec);
668 }
669
670 if (qc->pto.timer_set) {
671 p = ngx_slprintf(p, last, " pto:%M",
672 qc->pto.timer.key - ngx_current_msec);
673 }
674
675 if (qc->close.timer_set) {
676 p = ngx_slprintf(p, last, " close:%M",
677 qc->close.timer.key - ngx_current_msec);
678 }
679 }
680
681 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
682 "quic %*s", p - buf, buf);
683 }
684
685 #else
686
687 #define ngx_quic_log_frame(log, f, tx)
688 #define ngx_quic_connstate_dbg(c)
689
690 #endif
691
692
693 #if BORINGSSL_API_VERSION >= 10
694
695 static int
696 ngx_quic_set_read_secret(ngx_ssl_conn_t *ssl_conn,
697 enum ssl_encryption_level_t level, const SSL_CIPHER *cipher,
698 const uint8_t *rsecret, size_t secret_len)
699 {
700 ngx_connection_t *c;
701 ngx_quic_connection_t *qc;
702
703 c = ngx_ssl_get_connection((ngx_ssl_conn_t *) ssl_conn);
704 qc = ngx_quic_get_connection(c);
705
706 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
707 "quic ngx_quic_set_read_secret() level:%d", level);
708 #ifdef NGX_QUIC_DEBUG_CRYPTO
709 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
710 "quic read secret len:%uz %*xs", secret_len,
711 secret_len, rsecret);
712 #endif
713
714 return ngx_quic_keys_set_encryption_secret(c->pool, 0, qc->keys, level,
715 cipher, rsecret, secret_len);
716 }
717
718
719 static int
720 ngx_quic_set_write_secret(ngx_ssl_conn_t *ssl_conn,
721 enum ssl_encryption_level_t level, const SSL_CIPHER *cipher,
722 const uint8_t *wsecret, size_t secret_len)
723 {
724 ngx_connection_t *c;
725 ngx_quic_connection_t *qc;
726
727 c = ngx_ssl_get_connection((ngx_ssl_conn_t *) ssl_conn);
728 qc = ngx_quic_get_connection(c);
729
730 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
731 "quic ngx_quic_set_write_secret() level:%d", level);
732 #ifdef NGX_QUIC_DEBUG_CRYPTO
733 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
734 "quic write secret len:%uz %*xs", secret_len,
735 secret_len, wsecret);
736 #endif
737
738 return ngx_quic_keys_set_encryption_secret(c->pool, 1, qc->keys, level,
739 cipher, wsecret, secret_len);
740 }
741
742 #else
743
744 static int
745 ngx_quic_set_encryption_secrets(ngx_ssl_conn_t *ssl_conn,
746 enum ssl_encryption_level_t level, const uint8_t *rsecret,
747 const uint8_t *wsecret, size_t secret_len)
748 {
749 ngx_connection_t *c;
750 const SSL_CIPHER *cipher;
751 ngx_quic_connection_t *qc;
752
753 c = ngx_ssl_get_connection((ngx_ssl_conn_t *) ssl_conn);
754 qc = ngx_quic_get_connection(c);
755
756 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
757 "quic ngx_quic_set_encryption_secrets() level:%d", level);
758 #ifdef NGX_QUIC_DEBUG_CRYPTO
759 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
760 "quic read secret len:%uz %*xs", secret_len,
761 secret_len, rsecret);
762 #endif
763
764 cipher = SSL_get_current_cipher(ssl_conn);
765
766 if (ngx_quic_keys_set_encryption_secret(c->pool, 0, qc->keys, level,
767 cipher, rsecret, secret_len)
768 != 1)
769 {
770 return 0;
771 }
772
773 if (level == ssl_encryption_early_data) {
774 return 1;
775 }
776
777 #ifdef NGX_QUIC_DEBUG_CRYPTO
778 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
779 "quic write secret len:%uz %*xs", secret_len,
780 secret_len, wsecret);
781 #endif
782
783 return ngx_quic_keys_set_encryption_secret(c->pool, 1, qc->keys, level,
784 cipher, wsecret, secret_len);
785 }
786
787 #endif
788
789
790 static int
791 ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn,
792 enum ssl_encryption_level_t level, const uint8_t *data, size_t len)
793 {
794 u_char *p, *end;
795 size_t client_params_len;
796 const uint8_t *client_params;
797 ngx_quic_frame_t *frame;
798 ngx_connection_t *c;
799 ngx_quic_connection_t *qc;
800 ngx_quic_frames_stream_t *fs;
801
802 c = ngx_ssl_get_connection((ngx_ssl_conn_t *) ssl_conn);
803 qc = ngx_quic_get_connection(c);
804
805 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
806 "quic ngx_quic_add_handshake_data");
807
808 if (!qc->client_tp_done) {
809 /*
810 * things to do once during handshake: check ALPN and transport
811 * parameters; we want to break handshake if something is wrong
812 * here;
813 */
814
815 #if defined(TLSEXT_TYPE_application_layer_protocol_negotiation)
816 if (qc->conf->require_alpn) {
817 unsigned int len;
818 const unsigned char *data;
819
820 SSL_get0_alpn_selected(ssl_conn, &data, &len);
821
822 if (len == 0) {
823 qc->error = 0x100 + SSL_AD_NO_APPLICATION_PROTOCOL;
824 qc->error_reason = "unsupported protocol in ALPN extension";
825
826 ngx_log_error(NGX_LOG_INFO, c->log, 0,
827 "quic unsupported protocol in ALPN extension");
828 return 0;
829 }
830 }
831 #endif
832
833 SSL_get_peer_quic_transport_params(ssl_conn, &client_params,
834 &client_params_len);
835
836 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
837 "quic SSL_get_peer_quic_transport_params():"
838 " params_len:%ui", client_params_len);
839
840 if (client_params_len == 0) {
841 /* quic-tls 8.2 */
842 qc->error = NGX_QUIC_ERR_CRYPTO(SSL_AD_MISSING_EXTENSION);
843 qc->error_reason = "missing transport parameters";
844
845 ngx_log_error(NGX_LOG_INFO, c->log, 0,
846 "missing transport parameters");
847 return 0;
848 }
849
850 p = (u_char *) client_params;
851 end = p + client_params_len;
852
853 if (ngx_quic_parse_transport_params(p, end, &qc->ctp, c->log)
854 != NGX_OK)
855 {
856 qc->error = NGX_QUIC_ERR_TRANSPORT_PARAMETER_ERROR;
857 qc->error_reason = "failed to process transport parameters";
858
859 return 0;
860 }
861
862 if (qc->ctp.max_idle_timeout > 0
863 && qc->ctp.max_idle_timeout < qc->tp.max_idle_timeout)
864 {
865 qc->tp.max_idle_timeout = qc->ctp.max_idle_timeout;
866 }
867
868 if (qc->ctp.max_udp_payload_size < NGX_QUIC_MIN_INITIAL_SIZE
869 || qc->ctp.max_udp_payload_size > NGX_QUIC_MAX_UDP_PAYLOAD_SIZE)
870 {
871 qc->error = NGX_QUIC_ERR_TRANSPORT_PARAMETER_ERROR;
872 qc->error_reason = "invalid maximum packet size";
873
874 ngx_log_error(NGX_LOG_INFO, c->log, 0,
875 "quic maximum packet size is invalid");
876 return 0;
877 }
878
879 if (qc->ctp.max_udp_payload_size > ngx_quic_max_udp_payload(c)) {
880 qc->ctp.max_udp_payload_size = ngx_quic_max_udp_payload(c);
881 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
882 "quic client maximum packet size truncated");
883 }
884
885 #if (NGX_QUIC_DRAFT_VERSION >= 28)
886 if (qc->scid.len != qc->ctp.initial_scid.len
887 || ngx_memcmp(qc->scid.data, qc->ctp.initial_scid.data,
888 qc->scid.len) != 0)
889 {
890 ngx_log_error(NGX_LOG_INFO, c->log, 0,
891 "quic client initial_source_connection_id "
892 "mismatch");
893 return 0;
894 }
895 #endif
896
897 qc->streams.server_max_streams_bidi = qc->ctp.initial_max_streams_bidi;
898 qc->streams.server_max_streams_uni = qc->ctp.initial_max_streams_uni;
899
900 qc->client_tp_done = 1;
901 }
902
903 fs = &qc->crypto[level];
904
905 frame = ngx_quic_alloc_frame(c);
906 if (frame == NULL) {
907 return 0;
908 }
909
910 frame->data = ngx_quic_copy_buf(c, (u_char *) data, len);
911 if (frame->data == NGX_CHAIN_ERROR) {
912 return 0;
913 }
914
915 frame->level = level;
916 frame->type = NGX_QUIC_FT_CRYPTO;
917 frame->u.crypto.offset = fs->sent;
918 frame->u.crypto.length = len;
919
920 fs->sent += len;
921
922 ngx_quic_queue_frame(qc, frame);
923
924 return 1;
925 }
926
927
928 static int
929 ngx_quic_flush_flight(ngx_ssl_conn_t *ssl_conn)
930 {
931 #if (NGX_DEBUG)
932 ngx_connection_t *c;
933
934 c = ngx_ssl_get_connection((ngx_ssl_conn_t *) ssl_conn);
935
936 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
937 "quic ngx_quic_flush_flight()");
938 #endif
939 return 1;
940 }
941
942
943 static int
944 ngx_quic_send_alert(ngx_ssl_conn_t *ssl_conn, enum ssl_encryption_level_t level,
945 uint8_t alert)
946 {
947 ngx_connection_t *c;
948 ngx_quic_connection_t *qc;
949
950 c = ngx_ssl_get_connection((ngx_ssl_conn_t *) ssl_conn);
951
952 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
953 "quic ngx_quic_send_alert() lvl:%d alert:%d",
954 (int) level, (int) alert);
955
956 qc = ngx_quic_get_connection(c);
957 if (qc == NULL) {
958 return 1;
959 }
960
961 qc->error_level = level;
962 qc->error = NGX_QUIC_ERR_CRYPTO(alert);
963 qc->error_reason = "TLS alert";
964 qc->error_app = 0;
965 qc->error_ftype = 0;
966
967 if (ngx_quic_send_cc(c) != NGX_OK) {
968 return 0;
969 }
970
971 return 1;
972 }
973
974
975 void
976 ngx_quic_run(ngx_connection_t *c, ngx_quic_conf_t *conf)
977 {
978 ngx_int_t rc;
979 ngx_quic_connection_t *qc;
980
981 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic run");
982
983 rc = ngx_quic_input(c, c->buffer, conf);
984 if (rc != NGX_OK) {
985 ngx_quic_close_connection(c, rc == NGX_DECLINED ? NGX_DONE : NGX_ERROR);
986 return;
987 }
988
989 qc = ngx_quic_get_connection(c);
990
991 ngx_add_timer(c->read, qc->in_retry ? NGX_QUIC_RETRY_TIMEOUT
992 : qc->tp.max_idle_timeout);
993
994 c->read->handler = ngx_quic_input_handler;
995
996 ngx_quic_connstate_dbg(c);
997 return;
998 }
999
1000
1001 static ngx_quic_connection_t *
1002 ngx_quic_new_connection(ngx_connection_t *c, ngx_quic_conf_t *conf,
1003 ngx_quic_header_t *pkt)
1004 {
1005 ngx_uint_t i;
1006 ngx_quic_tp_t *ctp;
1007 ngx_quic_client_id_t *cid;
1008 ngx_quic_connection_t *qc;
1009
1010 qc = ngx_pcalloc(c->pool, sizeof(ngx_quic_connection_t));
1011 if (qc == NULL) {
1012 return NULL;
1013 }
1014
1015 qc->keys = ngx_quic_keys_new(c->pool);
1016 if (qc->keys == NULL) {
1017 return NULL;
1018 }
1019
1020 qc->version = pkt->version;
1021
1022 ngx_rbtree_init(&qc->streams.tree, &qc->streams.sentinel,
1023 ngx_quic_rbtree_insert_stream);
1024
1025 for (i = 0; i < NGX_QUIC_SEND_CTX_LAST; i++) {
1026 ngx_queue_init(&qc->send_ctx[i].frames);
1027 ngx_queue_init(&qc->send_ctx[i].sent);
1028 qc->send_ctx[i].largest_pn = NGX_QUIC_UNSET_PN;
1029 qc->send_ctx[i].largest_ack = NGX_QUIC_UNSET_PN;
1030 qc->send_ctx[i].largest_range = NGX_QUIC_UNSET_PN;
1031 qc->send_ctx[i].pending_ack = NGX_QUIC_UNSET_PN;
1032 }
1033
1034 qc->send_ctx[0].level = ssl_encryption_initial;
1035 qc->send_ctx[1].level = ssl_encryption_handshake;
1036 qc->send_ctx[2].level = ssl_encryption_application;
1037
1038 for (i = 0; i < NGX_QUIC_ENCRYPTION_LAST; i++) {
1039 ngx_queue_init(&qc->crypto[i].frames);
1040 }
1041
1042 ngx_queue_init(&qc->free_frames);
1043 ngx_queue_init(&qc->client_ids);
1044 ngx_queue_init(&qc->server_ids);
1045 ngx_queue_init(&qc->free_client_ids);
1046 ngx_queue_init(&qc->free_server_ids);
1047
1048 qc->avg_rtt = NGX_QUIC_INITIAL_RTT;
1049 qc->rttvar = NGX_QUIC_INITIAL_RTT / 2;
1050 qc->min_rtt = NGX_TIMER_INFINITE;
1051
1052 /*
1053 * qc->latest_rtt = 0
1054 * qc->nclient_ids = 0
1055 * qc->nserver_ids = 0
1056 * qc->max_retired_seqnum = 0
1057 */
1058
1059 qc->received = pkt->raw->last - pkt->raw->start;
1060
1061 qc->pto.log = c->log;
1062 qc->pto.data = c;
1063 qc->pto.handler = ngx_quic_pto_handler;
1064 qc->pto.cancelable = 1;
1065
1066 qc->push.log = c->log;
1067 qc->push.data = c;
1068 qc->push.handler = ngx_quic_push_handler;
1069 qc->push.cancelable = 1;
1070
1071 qc->conf = conf;
1072 qc->tp = conf->tp;
1073
1074 if (qc->tp.disable_active_migration) {
1075 qc->sockaddr = ngx_palloc(c->pool, c->socklen);
1076 if (qc->sockaddr == NULL) {
1077 return NULL;
1078 }
1079
1080 ngx_memcpy(qc->sockaddr, c->sockaddr, c->socklen);
1081 qc->socklen = c->socklen;
1082 }
1083
1084 ctp = &qc->ctp;
1085 ctp->max_udp_payload_size = ngx_quic_max_udp_payload(c);
1086 ctp->ack_delay_exponent = NGX_QUIC_DEFAULT_ACK_DELAY_EXPONENT;
1087 ctp->max_ack_delay = NGX_QUIC_DEFAULT_MAX_ACK_DELAY;
1088
1089 qc->streams.recv_max_data = qc->tp.initial_max_data;
1090
1091 qc->streams.client_max_streams_uni = qc->tp.initial_max_streams_uni;
1092 qc->streams.client_max_streams_bidi = qc->tp.initial_max_streams_bidi;
1093
1094 qc->congestion.window = ngx_min(10 * qc->tp.max_udp_payload_size,
1095 ngx_max(2 * qc->tp.max_udp_payload_size,
1096 14720));
1097 qc->congestion.ssthresh = (size_t) -1;
1098 qc->congestion.recovery_start = ngx_current_msec;
1099
1100 qc->odcid.len = pkt->dcid.len;
1101 qc->odcid.data = ngx_pstrdup(c->pool, &pkt->dcid);
1102 if (qc->odcid.data == NULL) {
1103 return NULL;
1104 }
1105
1106 qc->dcid.len = NGX_QUIC_SERVER_CID_LEN;
1107 qc->dcid.data = ngx_pnalloc(c->pool, qc->dcid.len);
1108 if (qc->dcid.data == NULL) {
1109 return NULL;
1110 }
1111
1112 if (ngx_quic_create_server_id(c, qc->dcid.data) != NGX_OK) {
1113 return NULL;
1114 }
1115
1116 #if (NGX_QUIC_DRAFT_VERSION >= 28)
1117 qc->tp.original_dcid = qc->odcid;
1118 #endif
1119 qc->tp.initial_scid = qc->dcid;
1120
1121 qc->scid.len = pkt->scid.len;
1122 qc->scid.data = ngx_pnalloc(c->pool, qc->scid.len);
1123 if (qc->scid.data == NULL) {
1124 return NULL;
1125 }
1126 ngx_memcpy(qc->scid.data, pkt->scid.data, qc->scid.len);
1127
1128 cid = ngx_quic_alloc_client_id(c, qc);
1129 if (cid == NULL) {
1130 return NULL;
1131 }
1132
1133 cid->seqnum = 0;
1134 cid->len = pkt->scid.len;
1135 ngx_memcpy(cid->id, pkt->scid.data, pkt->scid.len);
1136
1137 ngx_queue_insert_tail(&qc->client_ids, &cid->queue);
1138 qc->nclient_ids++;
1139 qc->client_seqnum = 0;
1140
1141 qc->server_seqnum = NGX_QUIC_UNSET_PN;
1142
1143 return qc;
1144 }
1145
1146
1147 static ngx_int_t
1148 ngx_quic_send_stateless_reset(ngx_connection_t *c, ngx_quic_conf_t *conf,
1149 ngx_quic_header_t *pkt)
1150 {
1151 u_char *token;
1152 size_t len, max;
1153 uint16_t rndbytes;
1154 u_char buf[NGX_QUIC_MAX_SR_PACKET];
1155
1156 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
1157 "quic handle stateless reset output");
1158
1159 if (pkt->len <= NGX_QUIC_MIN_PKT_LEN) {
1160 return NGX_DECLINED;
1161 }
1162
1163 if (pkt->len <= NGX_QUIC_MIN_SR_PACKET) {
1164 len = pkt->len - 1;
1165
1166 } else {
1167 max = ngx_min(NGX_QUIC_MAX_SR_PACKET, pkt->len * 3);
1168
1169 if (RAND_bytes((u_char *) &rndbytes, sizeof(rndbytes)) != 1) {
1170 return NGX_ERROR;
1171 }
1172
1173 len = (rndbytes % (max - NGX_QUIC_MIN_SR_PACKET + 1))
1174 + NGX_QUIC_MIN_SR_PACKET;
1175 }
1176
1177 if (RAND_bytes(buf, len - NGX_QUIC_SR_TOKEN_LEN) != 1) {
1178 return NGX_ERROR;
1179 }
1180
1181 buf[0] &= ~NGX_QUIC_PKT_LONG;
1182 buf[0] |= NGX_QUIC_PKT_FIXED_BIT;
1183
1184 token = &buf[len - NGX_QUIC_SR_TOKEN_LEN];
1185
1186 if (ngx_quic_new_sr_token(c, &pkt->dcid, &conf->sr_token_key, token)
1187 != NGX_OK)
1188 {
1189 return NGX_ERROR;
1190 }
1191
1192 (void) ngx_quic_send(c, buf, len);
1193
1194 return NGX_DECLINED;
1195 }
1196
1197
1198 static ngx_int_t
1199 ngx_quic_process_stateless_reset(ngx_connection_t *c, ngx_quic_header_t *pkt)
1200 {
1201 u_char *tail, ch;
1202 ngx_uint_t i;
1203 ngx_queue_t *q;
1204 ngx_quic_client_id_t *cid;
1205 ngx_quic_connection_t *qc;
1206
1207 qc = ngx_quic_get_connection(c);
1208
1209 /* A stateless reset uses an entire UDP datagram */
1210 if (pkt->raw->start != pkt->data) {
1211 return NGX_DECLINED;
1212 }
1213
1214 tail = pkt->raw->last - NGX_QUIC_SR_TOKEN_LEN;
1215
1216 for (q = ngx_queue_head(&qc->client_ids);
1217 q != ngx_queue_sentinel(&qc->client_ids);
1218 q = ngx_queue_next(q))
1219 {
1220 cid = ngx_queue_data(q, ngx_quic_client_id_t, queue);
1221
1222 if (cid->seqnum == 0) {
1223 /* no stateless reset token in initial connection id */
1224 continue;
1225 }
1226
1227 /* constant time comparison */
1228
1229 for (ch = 0, i = 0; i < NGX_QUIC_SR_TOKEN_LEN; i++) {
1230 ch |= tail[i] ^ cid->sr_token[i];
1231 }
1232
1233 if (ch == 0) {
1234 return NGX_OK;
1235 }
1236 }
1237
1238 return NGX_DECLINED;
1239 }
1240
1241
1242 static ngx_int_t
1243 ngx_quic_negotiate_version(ngx_connection_t *c, ngx_quic_header_t *inpkt)
1244 {
1245 size_t len;
1246 ngx_quic_header_t pkt;
1247 static u_char buf[NGX_QUIC_MAX_UDP_PAYLOAD_SIZE];
1248
1249 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
1250 "sending version negotiation packet");
1251
1252 pkt.log = c->log;
1253 pkt.flags = NGX_QUIC_PKT_LONG | NGX_QUIC_PKT_FIXED_BIT;
1254 pkt.dcid = inpkt->scid;
1255 pkt.scid = inpkt->dcid;
1256
1257 len = ngx_quic_create_version_negotiation(&pkt, buf);
1258
1259 #ifdef NGX_QUIC_DEBUG_PACKETS
1260 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
1261 "quic vnego packet to send len:%uz %*xs", len, len, buf);
1262 #endif
1263
1264 (void) ngx_quic_send(c, buf, len);
1265
1266 return NGX_ERROR;
1267 }
1268
1269
1270 static ngx_int_t
1271 ngx_quic_create_server_id(ngx_connection_t *c, u_char *id)
1272 {
1273 if (RAND_bytes(id, NGX_QUIC_SERVER_CID_LEN) != 1) {
1274 return NGX_ERROR;
1275 }
1276
1277 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
1278 "quic create server id %*xs",
1279 (size_t) NGX_QUIC_SERVER_CID_LEN, id);
1280 return NGX_OK;
1281 }
1282
1283
1284 static ngx_int_t
1285 ngx_quic_send_retry(ngx_connection_t *c)
1286 {
1287 ssize_t len;
1288 ngx_str_t res, token;
1289 ngx_quic_header_t pkt;
1290 ngx_quic_connection_t *qc;
1291 u_char buf[NGX_QUIC_RETRY_BUFFER_SIZE];
1292
1293 qc = ngx_quic_get_connection(c);
1294
1295 if (ngx_quic_new_token(c, &token) != NGX_OK) {
1296 return NGX_ERROR;
1297 }
1298
1299 ngx_memzero(&pkt, sizeof(ngx_quic_header_t));
1300 pkt.flags = NGX_QUIC_PKT_FIXED_BIT | NGX_QUIC_PKT_LONG | NGX_QUIC_PKT_RETRY;
1301 pkt.version = qc->version;
1302 pkt.log = c->log;
1303 pkt.odcid = qc->odcid;
1304 pkt.dcid = qc->scid;
1305 pkt.scid = qc->dcid;
1306 pkt.token = token;
1307
1308 res.data = buf;
1309
1310 if (ngx_quic_encrypt(&pkt, &res) != NGX_OK) {
1311 return NGX_ERROR;
1312 }
1313
1314 #ifdef NGX_QUIC_DEBUG_PACKETS
1315 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
1316 "quic packet to send len:%uz %xV", res.len, &res);
1317 #endif
1318
1319 len = ngx_quic_send(c, res.data, res.len);
1320 if (len == NGX_ERROR) {
1321 return NGX_ERROR;
1322 }
1323
1324 qc->token = token;
1325 #if (NGX_QUIC_DRAFT_VERSION < 28)
1326 qc->tp.original_dcid = qc->odcid;
1327 #endif
1328 qc->tp.retry_scid = qc->dcid;
1329 qc->in_retry = 1;
1330
1331 if (ngx_quic_insert_server_id(c, &qc->dcid) == NULL) {
1332 return NGX_ERROR;
1333 }
1334
1335 return NGX_OK;
1336 }
1337
1338
1339 static ngx_int_t
1340 ngx_quic_new_token(ngx_connection_t *c, ngx_str_t *token)
1341 {
1342 int len, iv_len;
1343 u_char *data, *p, *key, *iv;
1344 ngx_msec_t now;
1345 EVP_CIPHER_CTX *ctx;
1346 const EVP_CIPHER *cipher;
1347 struct sockaddr_in *sin;
1348 #if (NGX_HAVE_INET6)
1349 struct sockaddr_in6 *sin6;
1350 #endif
1351 ngx_quic_connection_t *qc;
1352 u_char in[NGX_QUIC_MAX_TOKEN_SIZE];
1353
1354 switch (c->sockaddr->sa_family) {
1355
1356 #if (NGX_HAVE_INET6)
1357 case AF_INET6:
1358 sin6 = (struct sockaddr_in6 *) c->sockaddr;
1359
1360 len = sizeof(struct in6_addr);
1361 data = sin6->sin6_addr.s6_addr;
1362
1363 break;
1364 #endif
1365
1366 #if (NGX_HAVE_UNIX_DOMAIN)
1367 case AF_UNIX:
1368
1369 len = ngx_min(c->addr_text.len, NGX_QUIC_MAX_TOKEN_SIZE - sizeof(now));
1370 data = c->addr_text.data;
1371
1372 break;
1373 #endif
1374
1375 default: /* AF_INET */
1376 sin = (struct sockaddr_in *) c->sockaddr;
1377
1378 len = sizeof(in_addr_t);
1379 data = (u_char *) &sin->sin_addr;
1380
1381 break;
1382 }
1383
1384 p = ngx_cpymem(in, data, len);
1385
1386 now = ngx_current_msec;
1387 len += sizeof(now);
1388 ngx_memcpy(p, &now, sizeof(now));
1389
1390 cipher = EVP_aes_256_cbc();
1391 iv_len = EVP_CIPHER_iv_length(cipher);
1392
1393 token->len = iv_len + len + EVP_CIPHER_block_size(cipher);
1394 token->data = ngx_pnalloc(c->pool, token->len);
1395 if (token->data == NULL) {
1396 return NGX_ERROR;
1397 }
1398
1399 ctx = EVP_CIPHER_CTX_new();
1400 if (ctx == NULL) {
1401 return NGX_ERROR;
1402 }
1403
1404 qc = ngx_quic_get_connection(c);
1405 key = qc->conf->token_key;
1406 iv = token->data;
1407
1408 if (RAND_bytes(iv, iv_len) <= 0
1409 || !EVP_EncryptInit_ex(ctx, cipher, NULL, key, iv))
1410 {
1411 EVP_CIPHER_CTX_free(ctx);
1412 return NGX_ERROR;
1413 }
1414
1415 token->len = iv_len;
1416
1417 if (EVP_EncryptUpdate(ctx, token->data + token->len, &len, in, len) != 1) {
1418 EVP_CIPHER_CTX_free(ctx);
1419 return NGX_ERROR;
1420 }
1421
1422 token->len += len;
1423
1424 if (EVP_EncryptFinal_ex(ctx, token->data + token->len, &len) <= 0) {
1425 EVP_CIPHER_CTX_free(ctx);
1426 return NGX_ERROR;
1427 }
1428
1429 token->len += len;
1430
1431 EVP_CIPHER_CTX_free(ctx);
1432
1433 #ifdef NGX_QUIC_DEBUG_PACKETS
1434 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
1435 "quic new token len:%uz %xV", token->len, token);
1436 #endif
1437
1438 return NGX_OK;
1439 }
1440
1441
1442 static ngx_int_t
1443 ngx_quic_validate_token(ngx_connection_t *c, ngx_quic_header_t *pkt)
1444 {
1445 int len, tlen, iv_len;
1446 u_char *key, *iv, *p, *data;
1447 ngx_msec_t msec;
1448 EVP_CIPHER_CTX *ctx;
1449 const EVP_CIPHER *cipher;
1450 struct sockaddr_in *sin;
1451 #if (NGX_HAVE_INET6)
1452 struct sockaddr_in6 *sin6;
1453 #endif
1454 ngx_quic_connection_t *qc;
1455 u_char tdec[NGX_QUIC_MAX_TOKEN_SIZE];
1456
1457 qc = ngx_quic_get_connection(c);
1458
1459 /* Retry token */
1460
1461 if (qc->token.len) {
1462 if (pkt->token.len != qc->token.len) {
1463 goto bad_token;
1464 }
1465
1466 if (ngx_memcmp(pkt->token.data, qc->token.data, pkt->token.len) != 0) {
1467 goto bad_token;
1468 }
1469
1470 return NGX_OK;
1471 }
1472
1473 /* NEW_TOKEN in a previous connection */
1474
1475 cipher = EVP_aes_256_cbc();
1476 key = qc->conf->token_key;
1477 iv = pkt->token.data;
1478 iv_len = EVP_CIPHER_iv_length(cipher);
1479
1480 /* sanity checks */
1481
1482 if (pkt->token.len < (size_t) iv_len + EVP_CIPHER_block_size(cipher)) {
1483 goto bad_token;
1484 }
1485
1486 if (pkt->token.len > (size_t) iv_len + NGX_QUIC_MAX_TOKEN_SIZE) {
1487 goto bad_token;
1488 }
1489
1490 ctx = EVP_CIPHER_CTX_new();
1491 if (ctx == NULL) {
1492 return NGX_ERROR;
1493 }
1494
1495 if (!EVP_DecryptInit_ex(ctx, cipher, NULL, key, iv)) {
1496 EVP_CIPHER_CTX_free(ctx);
1497 return NGX_ERROR;
1498 }
1499
1500 p = pkt->token.data + iv_len;
1501 len = pkt->token.len - iv_len;
1502
1503 if (EVP_DecryptUpdate(ctx, tdec, &len, p, len) != 1) {
1504 EVP_CIPHER_CTX_free(ctx);
1505 goto bad_token;
1506 }
1507
1508 if (EVP_DecryptFinal_ex(ctx, tdec + len, &tlen) <= 0) {
1509 EVP_CIPHER_CTX_free(ctx);
1510 goto bad_token;
1511 }
1512
1513 EVP_CIPHER_CTX_free(ctx);
1514
1515 switch (c->sockaddr->sa_family) {
1516
1517 #if (NGX_HAVE_INET6)
1518 case AF_INET6:
1519 sin6 = (struct sockaddr_in6 *) c->sockaddr;
1520
1521 len = sizeof(struct in6_addr);
1522 data = sin6->sin6_addr.s6_addr;
1523
1524 break;
1525 #endif
1526
1527 #if (NGX_HAVE_UNIX_DOMAIN)
1528 case AF_UNIX:
1529
1530 len = ngx_min(c->addr_text.len, NGX_QUIC_MAX_TOKEN_SIZE - sizeof(msec));
1531 data = c->addr_text.data;
1532
1533 break;
1534 #endif
1535
1536 default: /* AF_INET */
1537 sin = (struct sockaddr_in *) c->sockaddr;
1538
1539 len = sizeof(in_addr_t);
1540 data = (u_char *) &sin->sin_addr;
1541
1542 break;
1543 }
1544
1545 if (ngx_memcmp(tdec, data, len) != 0) {
1546 goto bad_token;
1547 }
1548
1549 ngx_memcpy(&msec, tdec + len, sizeof(msec));
1550
1551 if (ngx_current_msec - msec > NGX_QUIC_RETRY_LIFETIME) {
1552 ngx_log_error(NGX_LOG_INFO, c->log, 0, "quic expired token");
1553 return NGX_DECLINED;
1554 }
1555
1556 return NGX_OK;
1557
1558 bad_token:
1559
1560 ngx_log_error(NGX_LOG_INFO, c->log, 0, "quic invalid token");
1561
1562 qc->error = NGX_QUIC_ERR_INVALID_TOKEN;
1563 qc->error_reason = "invalid_token";
1564
1565 return NGX_DECLINED;
1566 }
1567
1568
1569 static ngx_int_t
1570 ngx_quic_init_connection(ngx_connection_t *c)
1571 {
1572 u_char *p;
1573 size_t clen;
1574 ssize_t len;
1575 ngx_ssl_conn_t *ssl_conn;
1576 ngx_quic_connection_t *qc;
1577
1578 qc = ngx_quic_get_connection(c);
1579
1580 if (ngx_ssl_create_connection(qc->conf->ssl, c, NGX_SSL_BUFFER) != NGX_OK) {
1581 return NGX_ERROR;
1582 }
1583
1584 c->ssl->no_wait_shutdown = 1;
1585
1586 ssl_conn = c->ssl->connection;
1587
1588 if (SSL_set_quic_method(ssl_conn, &quic_method) == 0) {
1589 ngx_log_error(NGX_LOG_INFO, c->log, 0,
1590 "quic SSL_set_quic_method() failed");
1591 return NGX_ERROR;
1592 }
1593
1594 #ifdef SSL_READ_EARLY_DATA_SUCCESS
1595 if (SSL_CTX_get_max_early_data(qc->conf->ssl->ctx)) {
1596 SSL_set_quic_early_data_enabled(ssl_conn, 1);
1597 }
1598 #endif
1599
1600 if (ngx_quic_new_sr_token(c, &qc->dcid, &qc->conf->sr_token_key,
1601 qc->tp.sr_token)
1602 != NGX_OK)
1603 {
1604 return NGX_ERROR;
1605 }
1606
1607 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
1608 "quic stateless reset token %*xs",
1609 (size_t) NGX_QUIC_SR_TOKEN_LEN, qc->tp.sr_token);
1610
1611 len = ngx_quic_create_transport_params(NULL, NULL, &qc->tp, &clen);
1612 /* always succeeds */
1613
1614 p = ngx_pnalloc(c->pool, len);
1615 if (p == NULL) {
1616 return NGX_ERROR;
1617 }
1618
1619 len = ngx_quic_create_transport_params(p, p + len, &qc->tp, NULL);
1620 if (len < 0) {
1621 return NGX_ERROR;
1622 }
1623
1624 #ifdef NGX_QUIC_DEBUG_PACKETS
1625 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
1626 "quic transport parameters len:%uz %*xs", len, len, p);
1627 #endif
1628
1629 if (SSL_set_quic_transport_params(ssl_conn, p, len) == 0) {
1630 ngx_log_error(NGX_LOG_INFO, c->log, 0,
1631 "quic SSL_set_quic_transport_params() failed");
1632 return NGX_ERROR;
1633 }
1634
1635 #if NGX_OPENSSL_QUIC_ZRTT_CTX
1636 if (SSL_set_quic_early_data_context(ssl_conn, p, clen) == 0) {
1637 ngx_log_error(NGX_LOG_INFO, c->log, 0,
1638 "quic SSL_set_quic_early_data_context() failed");
1639 return NGX_ERROR;
1640 }
1641 #endif
1642
1643 return NGX_OK;
1644 }
1645
1646
1647 static ngx_inline size_t
1648 ngx_quic_max_udp_payload(ngx_connection_t *c)
1649 {
1650 /* TODO: path MTU discovery */
1651
1652 #if (NGX_HAVE_INET6)
1653 if (c->sockaddr->sa_family == AF_INET6) {
1654 return NGX_QUIC_MAX_UDP_PAYLOAD_OUT6;
1655 }
1656 #endif
1657
1658 return NGX_QUIC_MAX_UDP_PAYLOAD_OUT;
1659 }
1660
1661
1662 static void
1663 ngx_quic_input_handler(ngx_event_t *rev)
1664 {
1665 ssize_t n;
1666 ngx_int_t rc;
1667 ngx_buf_t b;
1668 ngx_connection_t *c;
1669 ngx_quic_connection_t *qc;
1670 static u_char buf[NGX_QUIC_MAX_UDP_PAYLOAD_SIZE];
1671
1672 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, rev->log, 0, "quic input handler");
1673
1674 ngx_memzero(&b, sizeof(ngx_buf_t));
1675 b.start = buf;
1676 b.end = buf + sizeof(buf);
1677 b.pos = b.last = b.start;
1678 b.memory = 1;
1679
1680 c = rev->data;
1681 qc = ngx_quic_get_connection(c);
1682
1683 c->log->action = "handling quic input";
1684
1685 if (rev->timedout) {
1686 ngx_log_error(NGX_LOG_INFO, c->log, NGX_ETIMEDOUT,
1687 "quic client timed out");
1688 ngx_quic_close_connection(c, NGX_DONE);
1689 return;
1690 }
1691
1692 if (c->close) {
1693 qc->error_reason = "graceful shutdown";
1694 ngx_quic_close_connection(c, NGX_OK);
1695 return;
1696 }
1697
1698 n = c->recv(c, b.start, b.end - b.start);
1699
1700 if (n == NGX_AGAIN) {
1701 if (qc->closing) {
1702 ngx_quic_close_connection(c, NGX_OK);
1703 }
1704 return;
1705 }
1706
1707 if (n == NGX_ERROR) {
1708 c->read->eof = 1;
1709 ngx_quic_close_connection(c, NGX_ERROR);
1710 return;
1711 }
1712
1713 if (qc->tp.disable_active_migration) {
1714 if (c->socklen != qc->socklen
1715 || ngx_memcmp(c->sockaddr, qc->sockaddr, c->socklen) != 0)
1716 {
1717 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
1718 "quic dropping packet from new address");
1719 return;
1720 }
1721 }
1722
1723 b.last += n;
1724 qc->received += n;
1725
1726 rc = ngx_quic_input(c, &b, NULL);
1727
1728 if (rc == NGX_ERROR) {
1729 ngx_quic_close_connection(c, NGX_ERROR);
1730 return;
1731 }
1732
1733 if (rc == NGX_DECLINED) {
1734 return;
1735 }
1736
1737 /* rc == NGX_OK */
1738
1739 qc->send_timer_set = 0;
1740 ngx_add_timer(rev, qc->tp.max_idle_timeout);
1741
1742 ngx_quic_connstate_dbg(c);
1743 }
1744
1745
1746 static void
1747 ngx_quic_close_connection(ngx_connection_t *c, ngx_int_t rc)
1748 {
1749 ngx_pool_t *pool;
1750 ngx_quic_connection_t *qc;
1751
1752 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
1753 "quic ngx_quic_close_connection rc:%i", rc);
1754
1755 qc = ngx_quic_get_connection(c);
1756
1757 if (qc == NULL) {
1758 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
1759 "quic close connection early error");
1760
1761 } else if (ngx_quic_close_quic(c, rc) == NGX_AGAIN) {
1762 return;
1763 }
1764
1765 if (c->ssl) {
1766 (void) ngx_ssl_shutdown(c);
1767 }
1768
1769 if (c->read->timer_set) {
1770 ngx_del_timer(c->read);
1771 }
1772
1773 #if (NGX_STAT_STUB)
1774 (void) ngx_atomic_fetch_add(ngx_stat_active, -1);
1775 #endif
1776
1777 c->destroyed = 1;
1778
1779 pool = c->pool;
1780
1781 ngx_close_connection(c);
1782
1783 ngx_destroy_pool(pool);
1784 }
1785
1786
1787 static ngx_int_t
1788 ngx_quic_close_quic(ngx_connection_t *c, ngx_int_t rc)
1789 {
1790 ngx_uint_t i;
1791 ngx_queue_t *q;
1792 ngx_quic_send_ctx_t *ctx;
1793 ngx_quic_server_id_t *sid;
1794 ngx_quic_connection_t *qc;
1795
1796 qc = ngx_quic_get_connection(c);
1797
1798 if (!qc->closing) {
1799
1800 /* drop packets from retransmit queues, no ack is expected */
1801 for (i = 0; i < NGX_QUIC_SEND_CTX_LAST; i++) {
1802 ngx_quic_free_frames(c, &qc->send_ctx[i].sent);
1803 }
1804
1805 if (rc == NGX_DONE) {
1806
1807 /*
1808 * 10.2. Idle Timeout
1809 *
1810 * If the idle timeout is enabled by either peer, a connection is
1811 * silently closed and its state is discarded when it remains idle
1812 */
1813
1814 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
1815 "quic closing %s connection",
1816 qc->draining ? "drained" : "idle");
1817
1818 } else {
1819
1820 /*
1821 * 10.3. Immediate Close
1822 *
1823 * An endpoint sends a CONNECTION_CLOSE frame (Section 19.19)
1824 * to terminate the connection immediately.
1825 */
1826
1827 qc->error_level = c->ssl ? SSL_quic_read_level(c->ssl->connection)
1828 : ssl_encryption_initial;
1829
1830 if (rc == NGX_OK) {
1831 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
1832 "quic immediate close drain:%d",
1833 qc->draining);
1834
1835 qc->close.log = c->log;
1836 qc->close.data = c;
1837 qc->close.handler = ngx_quic_close_timer_handler;
1838 qc->close.cancelable = 1;
1839
1840 ctx = ngx_quic_get_send_ctx(qc, qc->error_level);
1841
1842 ngx_add_timer(&qc->close, 3 * ngx_quic_pto(c, ctx));
1843
1844 qc->error = NGX_QUIC_ERR_NO_ERROR;
1845
1846 } else {
1847 if (qc->error == 0 && !qc->error_app) {
1848 qc->error = NGX_QUIC_ERR_INTERNAL_ERROR;
1849 }
1850
1851 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
1852 "quic immediate close due to %s error: %ui %s",
1853 qc->error_app ? "app " : "", qc->error,
1854 qc->error_reason ? qc->error_reason : "");
1855 }
1856
1857 (void) ngx_quic_send_cc(c);
1858
1859 if (qc->error_level == ssl_encryption_handshake) {
1860 /* for clients that might not have handshake keys */
1861 qc->error_level = ssl_encryption_initial;
1862 (void) ngx_quic_send_cc(c);
1863 }
1864 }
1865
1866 qc->closing = 1;
1867 }
1868
1869 if (rc == NGX_ERROR && qc->close.timer_set) {
1870 /* do not wait for timer in case of fatal error */
1871 ngx_del_timer(&qc->close);
1872 }
1873
1874 if (ngx_quic_close_streams(c, qc) == NGX_AGAIN) {
1875 return NGX_AGAIN;
1876 }
1877
1878 if (qc->push.timer_set) {
1879 ngx_del_timer(&qc->push);
1880 }
1881
1882 if (qc->pto.timer_set) {
1883 ngx_del_timer(&qc->pto);
1884 }
1885
1886 if (qc->push.posted) {
1887 ngx_delete_posted_event(&qc->push);
1888 }
1889
1890 while (!ngx_queue_empty(&qc->server_ids)) {
1891 q = ngx_queue_head(&qc->server_ids);
1892 sid = ngx_queue_data(q, ngx_quic_server_id_t, queue);
1893
1894 ngx_queue_remove(q);
1895 ngx_rbtree_delete(&c->listening->rbtree, &sid->udp.node);
1896 qc->nserver_ids--;
1897 }
1898
1899 if (qc->close.timer_set) {
1900 return NGX_AGAIN;
1901 }
1902
1903 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
1904 "quic part of connection is terminated");
1905
1906 /* may be tested from SSL callback during SSL shutdown */
1907 c->udp = NULL;
1908
1909 return NGX_OK;
1910 }
1911
1912
1913 void
1914 ngx_quic_finalize_connection(ngx_connection_t *c, ngx_uint_t err,
1915 const char *reason)
1916 {
1917 ngx_quic_connection_t *qc;
1918
1919 qc = ngx_quic_get_connection(c);
1920 qc->error = err;
1921 qc->error_reason = reason;
1922 qc->error_app = 1;
1923 qc->error_ftype = 0;
1924
1925 ngx_quic_close_connection(c, NGX_ERROR);
1926 }
1927
1928
1929 static void
1930 ngx_quic_close_timer_handler(ngx_event_t *ev)
1931 {
1932 ngx_connection_t *c;
1933
1934 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, "quic close timer");
1935
1936 c = ev->data;
1937 ngx_quic_close_connection(c, NGX_DONE);
1938 }
1939
1940
1941 static ngx_int_t
1942 ngx_quic_close_streams(ngx_connection_t *c, ngx_quic_connection_t *qc)
1943 {
1944 ngx_event_t *rev, *wev;
1945 ngx_rbtree_t *tree;
1946 ngx_rbtree_node_t *node;
1947 ngx_quic_stream_t *qs;
1948
1949 #if (NGX_DEBUG)
1950 ngx_uint_t ns;
1951 #endif
1952
1953 tree = &qc->streams.tree;
1954
1955 if (tree->root == tree->sentinel) {
1956 return NGX_OK;
1957 }
1958
1959 #if (NGX_DEBUG)
1960 ns = 0;
1961 #endif
1962
1963 for (node = ngx_rbtree_min(tree->root, tree->sentinel);
1964 node;
1965 node = ngx_rbtree_next(tree, node))
1966 {
1967 qs = (ngx_quic_stream_t *) node;
1968
1969 rev = qs->c->read;
1970 rev->error = 1;
1971 rev->ready = 1;
1972
1973 wev = qs->c->write;
1974 wev->error = 1;
1975 wev->ready = 1;
1976
1977 ngx_post_event(rev, &ngx_posted_events);
1978
1979 if (rev->timer_set) {
1980 ngx_del_timer(rev);
1981 }
1982
1983 #if (NGX_DEBUG)
1984 ns++;
1985 #endif
1986 }
1987
1988 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
1989 "quic connection has %ui active streams", ns);
1990
1991 return NGX_AGAIN;
1992 }
1993
1994
1995 static ngx_int_t
1996 ngx_quic_input(ngx_connection_t *c, ngx_buf_t *b, ngx_quic_conf_t *conf)
1997 {
1998 u_char *p;
1999 ngx_int_t rc;
2000 ngx_uint_t good;
2001 ngx_quic_header_t pkt;
2002 ngx_quic_connection_t *qc;
2003
2004 good = 0;
2005
2006 p = b->pos;
2007
2008 while (p < b->last) {
2009
2010 ngx_memzero(&pkt, sizeof(ngx_quic_header_t));
2011 pkt.raw = b;
2012 pkt.data = p;
2013 pkt.len = b->last - p;
2014 pkt.log = c->log;
2015 pkt.flags = p[0];
2016 pkt.raw->pos++;
2017
2018 qc = ngx_quic_get_connection(c);
2019 if (qc) {
2020 qc->error = 0;
2021 qc->error_reason = 0;
2022 }
2023
2024 rc = ngx_quic_process_packet(c, conf, &pkt);
2025
2026 #if (NGX_DEBUG)
2027 if (pkt.parsed) {
2028 ngx_log_debug5(NGX_LOG_DEBUG_EVENT, c->log, 0,
2029 "quic packet %s done decr:%d pn:%L perr:%ui rc:%i",
2030 ngx_quic_level_name(pkt.level), pkt.decrypted,
2031 pkt.pn, pkt.error, rc);
2032 } else {
2033 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
2034 "quic packet done parse failed rc:%i", rc);
2035 }
2036 #endif
2037
2038 if (rc == NGX_ERROR) {
2039 return NGX_ERROR;
2040 }
2041
2042 if (rc == NGX_OK) {
2043 good = 1;
2044 }
2045
2046 /* NGX_OK || NGX_DECLINED */
2047
2048 /*
2049 * we get NGX_DECLINED when there are no keys [yet] available
2050 * to decrypt packet.
2051 * Instead of queueing it, we ignore it and rely on the sender's
2052 * retransmission:
2053 *
2054 * 12.2. Coalescing Packets:
2055 *
2056 * For example, if decryption fails (because the keys are
2057 * not available or any other reason), the receiver MAY either
2058 * discard or buffer the packet for later processing and MUST
2059 * attempt to process the remaining packets.
2060 *
2061 * We also skip packets that don't match connection state
2062 * or cannot be parsed properly.
2063 */
2064
2065 /* b->pos is at header end, adjust by actual packet length */
2066 b->pos = pkt.data + pkt.len;
2067
2068 /* firefox workaround: skip zero padding at the end of quic packet */
2069 while (b->pos < b->last && *(b->pos) == 0) {
2070 b->pos++;
2071 }
2072
2073 p = b->pos;
2074 }
2075
2076 return good ? NGX_OK : NGX_DECLINED;
2077 }
2078
2079
2080 static ngx_int_t
2081 ngx_quic_process_packet(ngx_connection_t *c, ngx_quic_conf_t *conf,
2082 ngx_quic_header_t *pkt)
2083 {
2084 ngx_int_t rc;
2085 ngx_quic_send_ctx_t *ctx;
2086 ngx_quic_connection_t *qc;
2087
2088 static u_char buf[NGX_QUIC_MAX_UDP_PAYLOAD_SIZE];
2089
2090 c->log->action = "parsing quic packet";
2091
2092 rc = ngx_quic_parse_packet(pkt);
2093
2094 if (rc == NGX_DECLINED || rc == NGX_ERROR) {
2095 return rc;
2096 }
2097
2098 pkt->parsed = 1;
2099
2100 c->log->action = "processing quic packet";
2101
2102 qc = ngx_quic_get_connection(c);
2103
2104 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
2105 "quic packet rx dcid len:%uz %xV",
2106 pkt->dcid.len, &pkt->dcid);
2107
2108 #if (NGX_DEBUG)
2109 if (pkt->level != ssl_encryption_application) {
2110 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
2111 "quic packet rx scid len:%uz %xV",
2112 pkt->scid.len, &pkt->scid);
2113 }
2114
2115 if (pkt->level == ssl_encryption_initial) {
2116 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
2117 "quic token len:%uz %xV",
2118 pkt->token.len, &pkt->token);
2119 }
2120 #endif
2121
2122 if (qc) {
2123
2124 if (rc == NGX_ABORT) {
2125 ngx_log_error(NGX_LOG_INFO, c->log, 0,
2126 "quic unsupported version: 0x%xD", pkt->version);
2127 return NGX_DECLINED;
2128 }
2129
2130 if (pkt->level != ssl_encryption_application) {
2131 if (pkt->version != qc->version) {
2132 ngx_log_error(NGX_LOG_INFO, c->log, 0,
2133 "quic version mismatch: 0x%xD", pkt->version);
2134 return NGX_DECLINED;
2135 }
2136 }
2137
2138 if (ngx_quic_check_peer(qc, pkt) != NGX_OK) {
2139
2140 if (pkt->level == ssl_encryption_application) {
2141 if (ngx_quic_process_stateless_reset(c, pkt) == NGX_OK) {
2142 ngx_log_error(NGX_LOG_INFO, c->log, 0,
2143 "quic stateless reset packet detected");
2144
2145 qc->draining = 1;
2146 ngx_quic_close_connection(c, NGX_OK);
2147
2148 return NGX_OK;
2149 }
2150
2151 return ngx_quic_send_stateless_reset(c, qc->conf, pkt);
2152 }
2153
2154 return NGX_DECLINED;
2155 }
2156
2157 if (qc->in_retry) {
2158
2159 c->log->action = "retrying quic connection";
2160
2161 if (pkt->level != ssl_encryption_initial) {
2162 ngx_log_error(NGX_LOG_INFO, c->log, 0,
2163 "quic discard late retry packet");
2164 return NGX_DECLINED;
2165 }
2166
2167 if (!pkt->token.len) {
2168 ngx_log_error(NGX_LOG_INFO, c->log, 0,
2169 "quic discard retry packet without token");
2170 return NGX_DECLINED;
2171 }
2172
2173 qc->odcid.len = pkt->dcid.len;
2174 qc->odcid.data = ngx_pstrdup(c->pool, &pkt->dcid);
2175 if (qc->odcid.data == NULL) {
2176 return NGX_ERROR;
2177 }
2178
2179 ngx_quic_clear_temp_server_ids(c);
2180
2181 qc->dcid.len = NGX_QUIC_SERVER_CID_LEN;
2182 qc->dcid.data = ngx_pnalloc(c->pool, qc->dcid.len);
2183 if (qc->dcid.data == NULL) {
2184 return NGX_ERROR;
2185 }
2186
2187 if (ngx_quic_create_server_id(c, qc->dcid.data) != NGX_OK) {
2188 return NGX_ERROR;
2189 }
2190
2191 qc->server_seqnum = 0;
2192
2193 if (ngx_quic_insert_server_id(c, &qc->dcid) == NULL) {
2194 return NGX_ERROR;
2195 }
2196
2197 qc->tp.initial_scid = qc->dcid;
2198 qc->in_retry = 0;
2199
2200 if (ngx_quic_init_secrets(c) != NGX_OK) {
2201 return NGX_ERROR;
2202 }
2203
2204 if (ngx_quic_validate_token(c, pkt) != NGX_OK) {
2205 return NGX_ERROR;
2206 }
2207
2208 qc->validated = 1;
2209 }
2210
2211 } else {
2212
2213 if (rc == NGX_ABORT) {
2214 return ngx_quic_negotiate_version(c, pkt);
2215 }
2216
2217 if (pkt->level == ssl_encryption_initial) {
2218
2219 c->log->action = "creating quic connection";
2220
2221 if (pkt->dcid.len < NGX_QUIC_CID_LEN_MIN) {
2222 /* 7.2. Negotiating Connection IDs */
2223 ngx_log_error(NGX_LOG_INFO, c->log, 0,
2224 "quic too short dcid in initial"
2225 " packet: len:%i", pkt->dcid.len);
2226 return NGX_ERROR;
2227 }
2228
2229 qc = ngx_quic_new_connection(c, conf, pkt);
2230 if (qc == NULL) {
2231 return NGX_ERROR;
2232 }
2233
2234 c->udp = &qc->udp;
2235
2236 if (ngx_terminate || ngx_exiting) {
2237 qc->error = NGX_QUIC_ERR_CONNECTION_REFUSED;
2238 return NGX_ERROR;
2239 }
2240
2241 if (pkt->token.len) {
2242 rc = ngx_quic_validate_token(c, pkt);
2243
2244 if (rc == NGX_OK) {
2245 qc->validated = 1;
2246
2247 } else if (rc == NGX_ERROR) {
2248 return NGX_ERROR;
2249
2250 } else {
2251 /* NGX_DECLINED */
2252 if (conf->retry) {
2253 return ngx_quic_send_retry(c);
2254 }
2255 }
2256
2257 } else if (conf->retry) {
2258 return ngx_quic_send_retry(c);
2259 }
2260
2261 if (ngx_quic_init_secrets(c) != NGX_OK) {
2262 return NGX_ERROR;
2263 }
2264
2265 if (ngx_quic_insert_server_id(c, &qc->odcid) == NULL) {
2266 return NGX_ERROR;
2267 }
2268
2269 qc->server_seqnum = 0;
2270
2271 if (ngx_quic_insert_server_id(c, &qc->dcid) == NULL) {
2272 return NGX_ERROR;
2273 }
2274
2275 } else if (pkt->level == ssl_encryption_application) {
2276 return ngx_quic_send_stateless_reset(c, conf, pkt);
2277
2278 } else {
2279 return NGX_ERROR;
2280 }
2281 }
2282
2283 c->log->action = "decrypting packet";
2284
2285 if (!ngx_quic_keys_available(qc->keys, pkt->level)) {
2286 ngx_log_error(NGX_LOG_INFO, c->log, 0,
2287 "quic no level %d keys yet, ignoring packet", pkt->level);
2288 return NGX_DECLINED;
2289 }
2290
2291 pkt->keys = qc->keys;
2292 pkt->key_phase = qc->key_phase;
2293 pkt->plaintext = buf;
2294
2295 ctx = ngx_quic_get_send_ctx(qc, pkt->level);
2296
2297 rc = ngx_quic_decrypt(pkt, &ctx->largest_pn);
2298 if (rc != NGX_OK) {
2299 qc->error = pkt->error;
2300 qc->error_reason = "failed to decrypt packet";
2301 return rc;
2302 }
2303
2304 pkt->decrypted = 1;
2305
2306 if (c->ssl == NULL) {
2307 if (ngx_quic_init_connection(c) != NGX_OK) {
2308 return NGX_ERROR;
2309 }
2310 }
2311
2312 if (pkt->level == ssl_encryption_handshake) {
2313 /*
2314 * 4.10.1. The successful use of Handshake packets indicates
2315 * that no more Initial packets need to be exchanged
2316 */
2317 ngx_quic_discard_ctx(c, ssl_encryption_initial);
2318
2319 if (qc->validated == 0) {
2320 qc->validated = 1;
2321 ngx_post_event(&qc->push, &ngx_posted_events);
2322 }
2323 }
2324
2325 pkt->received = ngx_current_msec;
2326
2327 c->log->action = "handling payload";
2328
2329 if (pkt->level != ssl_encryption_application) {
2330 return ngx_quic_payload_handler(c, pkt);
2331 }
2332
2333 if (!pkt->key_update) {
2334 return ngx_quic_payload_handler(c, pkt);
2335 }
2336
2337 /* switch keys and generate next on Key Phase change */
2338
2339 qc->key_phase ^= 1;
2340 ngx_quic_keys_switch(c, qc->keys);
2341
2342 rc = ngx_quic_payload_handler(c, pkt);
2343 if (rc != NGX_OK) {
2344 return rc;
2345 }
2346
2347 return ngx_quic_keys_update(c, qc->keys);
2348 }
2349
2350
2351 static ngx_int_t
2352 ngx_quic_init_secrets(ngx_connection_t *c)
2353 {
2354 ngx_quic_connection_t *qc;
2355
2356 qc = ngx_quic_get_connection(c);
2357
2358 if (ngx_quic_keys_set_initial_secret(c->pool, qc->keys, &qc->odcid)
2359 != NGX_OK)
2360 {
2361 return NGX_ERROR;
2362 }
2363
2364 qc->initialized = 1;
2365
2366 return NGX_OK;
2367 }
2368
2369
2370 static void
2371 ngx_quic_discard_ctx(ngx_connection_t *c, enum ssl_encryption_level_t level)
2372 {
2373 ngx_queue_t *q;
2374 ngx_quic_frame_t *f;
2375 ngx_quic_send_ctx_t *ctx;
2376 ngx_quic_connection_t *qc;
2377
2378 qc = ngx_quic_get_connection(c);
2379
2380 if (!ngx_quic_keys_available(qc->keys, level)) {
2381 return;
2382 }
2383
2384 ngx_quic_keys_discard(qc->keys, level);
2385
2386 qc->pto_count = 0;
2387
2388 ctx = ngx_quic_get_send_ctx(qc, level);
2389
2390 while (!ngx_queue_empty(&ctx->sent)) {
2391 q = ngx_queue_head(&ctx->sent);
2392 ngx_queue_remove(q);
2393
2394 f = ngx_queue_data(q, ngx_quic_frame_t, queue);
2395 ngx_quic_congestion_ack(c, f);
2396 ngx_quic_free_frame(c, f);
2397 }
2398
2399 while (!ngx_queue_empty(&ctx->frames)) {
2400 q = ngx_queue_head(&ctx->frames);
2401 ngx_queue_remove(q);
2402
2403 f = ngx_queue_data(q, ngx_quic_frame_t, queue);
2404 ngx_quic_congestion_ack(c, f);
2405 ngx_quic_free_frame(c, f);
2406 }
2407
2408 if (level == ssl_encryption_initial) {
2409 ngx_quic_clear_temp_server_ids(c);
2410 }
2411
2412 ctx->send_ack = 0;
2413 }
2414
2415
2416 static ngx_int_t
2417 ngx_quic_check_peer(ngx_quic_connection_t *qc, ngx_quic_header_t *pkt)
2418 {
2419 ngx_queue_t *q;
2420 ngx_quic_client_id_t *cid;
2421
2422 if (pkt->level == ssl_encryption_application) {
2423 return NGX_OK;
2424 }
2425
2426 for (q = ngx_queue_head(&qc->client_ids);
2427 q != ngx_queue_sentinel(&qc->client_ids);
2428 q = ngx_queue_next(q))
2429 {
2430 cid = ngx_queue_data(q, ngx_quic_client_id_t, queue);
2431
2432 if (pkt->scid.len == cid->len
2433 && ngx_memcmp(pkt->scid.data, cid->id, cid->len) == 0)
2434 {
2435 return NGX_OK;
2436 }
2437 }
2438
2439 ngx_log_error(NGX_LOG_INFO, pkt->log, 0, "quic unexpected quic scid");
2440 return NGX_ERROR;
2441 }
2442
2443
2444 static ngx_int_t
2445 ngx_quic_payload_handler(ngx_connection_t *c, ngx_quic_header_t *pkt)
2446 {
2447 u_char *end, *p;
2448 ssize_t len;
2449 ngx_buf_t buf;
2450 ngx_uint_t do_close;
2451 ngx_chain_t chain;
2452 ngx_quic_frame_t frame;
2453 ngx_quic_connection_t *qc;
2454
2455 qc = ngx_quic_get_connection(c);
2456
2457 if (qc->closing) {
2458 /*
2459 * 10.1 Closing and Draining Connection States
2460 * ... delayed or reordered packets are properly discarded.
2461 *
2462 * An endpoint retains only enough information to generate
2463 * a packet containing a CONNECTION_CLOSE frame and to identify
2464 * packets as belonging to the connection.
2465 */
2466
2467 qc->error_level = pkt->level;
2468 qc->error = NGX_QUIC_ERR_NO_ERROR;
2469 qc->error_reason = "connection is closing, packet discarded";
2470 qc->error_ftype = 0;
2471 qc->error_app = 0;
2472
2473 return ngx_quic_send_cc(c);
2474 }
2475
2476 p = pkt->payload.data;
2477 end = p + pkt->payload.len;
2478
2479 do_close = 0;
2480
2481 while (p < end) {
2482
2483 c->log->action = "parsing frames";
2484
2485 ngx_memzero(&buf, sizeof(ngx_buf_t));
2486 buf.temporary = 1;
2487
2488 chain.buf = &buf;
2489 chain.next = NULL;
2490 frame.data = &chain;
2491
2492 len = ngx_quic_parse_frame(pkt, p, end, &frame);
2493
2494 if (len < 0) {
2495 qc->error = pkt->error;
2496 return NGX_ERROR;
2497 }
2498
2499 ngx_quic_log_frame(c->log, &frame, 0);
2500
2501 c->log->action = "handling frames";
2502
2503 p += len;
2504
2505 switch (frame.type) {
2506
2507 case NGX_QUIC_FT_ACK:
2508 if (ngx_quic_handle_ack_frame(c, pkt, &frame) != NGX_OK) {
2509 return NGX_ERROR;
2510 }
2511
2512 continue;
2513
2514 case NGX_QUIC_FT_PADDING:
2515 /* no action required */
2516 continue;
2517
2518 case NGX_QUIC_FT_CONNECTION_CLOSE:
2519 case NGX_QUIC_FT_CONNECTION_CLOSE_APP:
2520 do_close = 1;
2521 continue;
2522 }
2523
2524 /* got there with ack-eliciting packet */
2525 pkt->need_ack = 1;
2526
2527 switch (frame.type) {
2528
2529 case NGX_QUIC_FT_CRYPTO:
2530
2531 if (ngx_quic_handle_crypto_frame(c, pkt, &frame) != NGX_OK) {
2532 return NGX_ERROR;
2533 }
2534
2535 break;
2536
2537 case NGX_QUIC_FT_PING:
2538 break;
2539
2540 case NGX_QUIC_FT_STREAM0:
2541 case NGX_QUIC_FT_STREAM1:
2542 case NGX_QUIC_FT_STREAM2:
2543 case NGX_QUIC_FT_STREAM3:
2544 case NGX_QUIC_FT_STREAM4:
2545 case NGX_QUIC_FT_STREAM5:
2546 case NGX_QUIC_FT_STREAM6:
2547 case NGX_QUIC_FT_STREAM7:
2548
2549 if (ngx_quic_handle_stream_frame(c, pkt, &frame) != NGX_OK) {
2550 return NGX_ERROR;
2551 }
2552
2553 break;
2554
2555 case NGX_QUIC_FT_MAX_DATA:
2556
2557 if (ngx_quic_handle_max_data_frame(c, &frame.u.max_data) != NGX_OK)
2558 {
2559 return NGX_ERROR;
2560 }
2561
2562 break;
2563
2564 case NGX_QUIC_FT_STREAMS_BLOCKED:
2565 case NGX_QUIC_FT_STREAMS_BLOCKED2:
2566
2567 if (ngx_quic_handle_streams_blocked_frame(c, pkt,
2568 &frame.u.streams_blocked)
2569 != NGX_OK)
2570 {
2571 return NGX_ERROR;
2572 }
2573
2574 break;
2575
2576 case NGX_QUIC_FT_STREAM_DATA_BLOCKED:
2577
2578 if (ngx_quic_handle_stream_data_blocked_frame(c, pkt,
2579 &frame.u.stream_data_blocked)
2580 != NGX_OK)
2581 {
2582 return NGX_ERROR;
2583 }
2584
2585 break;
2586
2587 case NGX_QUIC_FT_MAX_STREAM_DATA:
2588
2589 if (ngx_quic_handle_max_stream_data_frame(c, pkt,
2590 &frame.u.max_stream_data)
2591 != NGX_OK)
2592 {
2593 return NGX_ERROR;
2594 }
2595
2596 break;
2597
2598 case NGX_QUIC_FT_RESET_STREAM:
2599
2600 if (ngx_quic_handle_reset_stream_frame(c, pkt,
2601 &frame.u.reset_stream)
2602 != NGX_OK)
2603 {
2604 return NGX_ERROR;
2605 }
2606
2607 break;
2608
2609 case NGX_QUIC_FT_STOP_SENDING:
2610
2611 if (ngx_quic_handle_stop_sending_frame(c, pkt,
2612 &frame.u.stop_sending)
2613 != NGX_OK)
2614 {
2615 return NGX_ERROR;
2616 }
2617
2618 break;
2619
2620 case NGX_QUIC_FT_MAX_STREAMS:
2621 case NGX_QUIC_FT_MAX_STREAMS2:
2622
2623 if (ngx_quic_handle_max_streams_frame(c, pkt, &frame.u.max_streams)
2624 != NGX_OK)
2625 {
2626 return NGX_ERROR;
2627 }
2628
2629 break;
2630
2631 case NGX_QUIC_FT_PATH_CHALLENGE:
2632
2633 if (ngx_quic_handle_path_challenge_frame(c, pkt,
2634 &frame.u.path_challenge)
2635 != NGX_OK)
2636 {
2637 return NGX_ERROR;
2638 }
2639
2640 break;
2641
2642 case NGX_QUIC_FT_NEW_CONNECTION_ID:
2643
2644 if (ngx_quic_handle_new_connection_id_frame(c, pkt, &frame.u.ncid)
2645 != NGX_OK)
2646 {
2647 return NGX_ERROR;
2648 }
2649
2650 break;
2651
2652 case NGX_QUIC_FT_RETIRE_CONNECTION_ID:
2653
2654 if (ngx_quic_handle_retire_connection_id_frame(c, pkt,
2655 &frame.u.retire_cid)
2656 != NGX_OK)
2657 {
2658 return NGX_ERROR;
2659 }
2660
2661 break;
2662
2663 case NGX_QUIC_FT_PATH_RESPONSE:
2664
2665 /* TODO: handle */
2666 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
2667 "quic frame handler not implemented");
2668 break;
2669
2670 default:
2671 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
2672 "quic missing frame handler");
2673 return NGX_ERROR;
2674 }
2675 }
2676
2677 if (p != end) {
2678 ngx_log_error(NGX_LOG_INFO, c->log, 0,
2679 "quic trailing garbage in payload:%ui bytes", end - p);
2680
2681 qc->error = NGX_QUIC_ERR_FRAME_ENCODING_ERROR;
2682 return NGX_ERROR;
2683 }
2684
2685 if (do_close) {
2686 qc->draining = 1;
2687 ngx_quic_close_connection(c, NGX_OK);
2688 }
2689
2690 if (ngx_quic_ack_packet(c, pkt) != NGX_OK) {
2691 return NGX_ERROR;
2692 }
2693
2694 return NGX_OK;
2695 }
2696
2697
2698 static ngx_int_t
2699 ngx_quic_ack_packet(ngx_connection_t *c, ngx_quic_header_t *pkt)
2700 {
2701 uint64_t base, largest, smallest, gs, ge, gap, range, pn;
2702 uint64_t prev_pending;
2703 ngx_uint_t i, nr;
2704 ngx_quic_send_ctx_t *ctx;
2705 ngx_quic_ack_range_t *r;
2706 ngx_quic_connection_t *qc;
2707
2708 c->log->action = "preparing ack";
2709
2710 qc = ngx_quic_get_connection(c);
2711
2712 ctx = ngx_quic_get_send_ctx(qc, pkt->level);
2713
2714 ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
2715 "quic ngx_quic_ack_packet pn:%uL largest %L fr:%uL"
2716 " nranges:%ui", pkt->pn, (int64_t) ctx->largest_range,
2717 ctx->first_range, ctx->nranges);
2718
2719 prev_pending = ctx->pending_ack;
2720
2721 if (pkt->need_ack) {
2722
2723 ngx_post_event(&qc->push, &ngx_posted_events);
2724
2725 if (ctx->send_ack == 0) {
2726 ctx->ack_delay_start = ngx_current_msec;
2727 }
2728
2729 ctx->send_ack++;
2730
2731 if (ctx->pending_ack == NGX_QUIC_UNSET_PN
2732 || ctx->pending_ack < pkt->pn)
2733 {
2734 ctx->pending_ack = pkt->pn;
2735 }
2736 }
2737
2738 base = ctx->largest_range;
2739 pn = pkt->pn;
2740
2741 if (base == NGX_QUIC_UNSET_PN) {
2742 ctx->largest_range = pn;
2743 ctx->largest_received = pkt->received;
2744 return NGX_OK;
2745 }
2746
2747 if (base == pn) {
2748 return NGX_OK;
2749 }
2750
2751 largest = base;
2752 smallest = largest - ctx->first_range;
2753
2754 if (pn > base) {
2755
2756 if (pn - base == 1) {
2757 ctx->first_range++;
2758 ctx->largest_range = pn;
2759 ctx->largest_received = pkt->received;
2760
2761 return NGX_OK;
2762
2763 } else {
2764 /* new gap in front of current largest */
2765
2766 /* no place for new range, send current range as is */
2767 if (ctx->nranges == NGX_QUIC_MAX_RANGES) {
2768
2769 if (prev_pending != NGX_QUIC_UNSET_PN) {
2770 if (ngx_quic_send_ack(c, ctx) != NGX_OK) {
2771 return NGX_ERROR;
2772 }
2773 }
2774
2775 if (prev_pending == ctx->pending_ack || !pkt->need_ack) {
2776 ctx->pending_ack = NGX_QUIC_UNSET_PN;
2777 }
2778 }
2779
2780 gap = pn - base - 2;
2781 range = ctx->first_range;
2782
2783 ctx->first_range = 0;
2784 ctx->largest_range = pn;
2785 ctx->largest_received = pkt->received;
2786
2787 /* packet is out of order, force send */
2788 if (pkt->need_ack) {
2789 ctx->send_ack = NGX_QUIC_MAX_ACK_GAP;
2790 }
2791
2792 i = 0;
2793
2794 goto insert;
2795 }
2796 }
2797
2798 /* pn < base, perform lookup in existing ranges */
2799
2800 /* packet is out of order */
2801 if (pkt->need_ack) {
2802 ctx->send_ack = NGX_QUIC_MAX_ACK_GAP;
2803 }
2804
2805 if (pn >= smallest && pn <= largest) {
2806 return NGX_OK;
2807 }
2808
2809 #if (NGX_SUPPRESS_WARN)
2810 r = NULL;
2811 #endif
2812
2813 for (i = 0; i < ctx->nranges; i++) {
2814 r = &ctx->ranges[i];
2815
2816 ge = smallest - 1;
2817 gs = ge - r->gap;
2818
2819 if (pn >= gs && pn <= ge) {
2820
2821 if (gs == ge) {
2822 /* gap size is exactly one packet, now filled */
2823
2824 /* data moves to previous range, current is removed */
2825
2826 if (i == 0) {
2827 ctx->first_range += r->range + 2;
2828
2829 } else {
2830 ctx->ranges[i - 1].range += r->range + 2;
2831 }
2832
2833 nr = ctx->nranges - i - 1;
2834 if (nr) {
2835 ngx_memmove(&ctx->ranges[i], &ctx->ranges[i + 1],
2836 sizeof(ngx_quic_ack_range_t) * nr);
2837 }
2838
2839 ctx->nranges--;
2840
2841 } else if (pn == gs) {
2842 /* current gap shrinks from tail (current range grows) */
2843 r->gap--;
2844 r->range++;
2845
2846 } else if (pn == ge) {
2847 /* current gap shrinks from head (previous range grows) */
2848 r->gap--;
2849
2850 if (i == 0) {
2851 ctx->first_range++;
2852
2853 } else {
2854 ctx->ranges[i - 1].range++;
2855 }
2856
2857 } else {
2858 /* current gap is split into two parts */
2859
2860 gap = ge - pn - 1;
2861 range = 0;
2862
2863 if (ctx->nranges == NGX_QUIC_MAX_RANGES) {
2864 if (prev_pending != NGX_QUIC_UNSET_PN) {
2865 if (ngx_quic_send_ack(c, ctx) != NGX_OK) {
2866 return NGX_ERROR;
2867 }
2868 }
2869
2870 if (prev_pending == ctx->pending_ack || !pkt->need_ack) {
2871 ctx->pending_ack = NGX_QUIC_UNSET_PN;
2872 }
2873 }
2874
2875 r->gap = pn - gs - 1;
2876 goto insert;
2877 }
2878
2879 return NGX_OK;
2880 }
2881
2882 largest = smallest - r->gap - 2;
2883 smallest = largest - r->range;
2884
2885 if (pn >= smallest && pn <= largest) {
2886 /* this packet number is already known */
2887 return NGX_OK;
2888 }
2889
2890 }
2891
2892 if (pn == smallest - 1) {
2893 /* extend first or last range */
2894
2895 if (i == 0) {
2896 ctx->first_range++;
2897
2898 } else {
2899 r->range++;
2900 }
2901
2902 return NGX_OK;
2903 }
2904
2905 /* nothing found, add new range at the tail */
2906
2907 if (ctx->nranges == NGX_QUIC_MAX_RANGES) {
2908 /* packet is too old to keep it */
2909
2910 if (pkt->need_ack) {
2911 return ngx_quic_send_ack_range(c, ctx, pn, pn);
2912 }
2913
2914 return NGX_OK;
2915 }
2916
2917 gap = smallest - 2 - pn;
2918 range = 0;
2919
2920 insert:
2921
2922 if (ctx->nranges < NGX_QUIC_MAX_RANGES) {
2923 ctx->nranges++;
2924 }
2925
2926 ngx_memmove(&ctx->ranges[i + 1], &ctx->ranges[i],
2927 sizeof(ngx_quic_ack_range_t) * (ctx->nranges - i - 1));
2928
2929 ctx->ranges[i].gap = gap;
2930 ctx->ranges[i].range = range;
2931
2932 return NGX_OK;
2933 }
2934
2935
2936 static ngx_int_t
2937 ngx_quic_send_ack_range(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx,
2938 uint64_t smallest, uint64_t largest)
2939 {
2940 ngx_quic_frame_t *frame;
2941
2942 frame = ngx_quic_alloc_frame(c);
2943 if (frame == NULL) {
2944 return NGX_ERROR;
2945 }
2946
2947 frame->level = ctx->level;
2948 frame->type = NGX_QUIC_FT_ACK;
2949 frame->u.ack.largest = largest;
2950 frame->u.ack.delay = 0;
2951 frame->u.ack.range_count = 0;
2952 frame->u.ack.first_range = largest - smallest;
2953
2954 return NGX_OK;
2955 }
2956
2957
2958 static void
2959 ngx_quic_drop_ack_ranges(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx,
2960 uint64_t pn)
2961 {
2962 uint64_t base;
2963 ngx_uint_t i, smallest, largest;
2964 ngx_quic_ack_range_t *r;
2965
2966 ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
2967 "quic ngx_quic_drop_ack_ranges pn:%uL largest:%uL"
2968 " fr:%uL nranges:%ui", pn, ctx->largest_range,
2969 ctx->first_range, ctx->nranges);
2970
2971 base = ctx->largest_range;
2972
2973 if (base == NGX_QUIC_UNSET_PN) {
2974 return;
2975 }
2976
2977 if (ctx->pending_ack != NGX_QUIC_UNSET_PN && pn >= ctx->pending_ack) {
2978 ctx->pending_ack = NGX_QUIC_UNSET_PN;
2979 }
2980
2981 largest = base;
2982 smallest = largest - ctx->first_range;
2983
2984 if (pn >= largest) {
2985 ctx->largest_range = NGX_QUIC_UNSET_PN;
2986 ctx->first_range = 0;
2987 ctx->nranges = 0;
2988 return;
2989 }
2990
2991 if (pn >= smallest) {
2992 ctx->first_range = largest - pn - 1;
2993 ctx->nranges = 0;
2994 return;
2995 }
2996
2997 for (i = 0; i < ctx->nranges; i++) {
2998 r = &ctx->ranges[i];
2999
3000 largest = smallest - r->gap - 2;
3001 smallest = largest - r->range;
3002
3003 if (pn >= largest) {
3004 ctx->nranges = i;
3005 return;
3006 }
3007 if (pn >= smallest) {
3008 r->range = largest - pn - 1;
3009 ctx->nranges = i + 1;
3010 return;
3011 }
3012 }
3013 }
3014
3015
3016 static ngx_int_t
3017 ngx_quic_send_ack(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx)
3018 {
3019 size_t len, left;
3020 uint64_t ack_delay;
3021 ngx_buf_t *b;
3022 ngx_uint_t i;
3023 ngx_chain_t *cl, **ll;
3024 ngx_quic_frame_t *frame;
3025 ngx_quic_connection_t *qc;
3026
3027 qc = ngx_quic_get_connection(c);
3028
3029 ack_delay = ngx_current_msec - ctx->largest_received;
3030 ack_delay *= 1000;
3031 ack_delay >>= qc->ctp.ack_delay_exponent;
3032
3033 frame = ngx_quic_alloc_frame(c);
3034 if (frame == NULL) {
3035 return NGX_ERROR;
3036 }
3037
3038 ll = &frame->data;
3039 b = NULL;
3040
3041 for (i = 0; i < ctx->nranges; i++) {
3042 len = ngx_quic_create_ack_range(NULL, ctx->ranges[i].gap,
3043 ctx->ranges[i].range);
3044
3045 left = b ? b->end - b->last : 0;
3046
3047 if (left < len) {
3048 cl = ngx_quic_alloc_buf(c);
3049 if (cl == NULL) {
3050 return NGX_ERROR;
3051 }
3052
3053 *ll = cl;
3054 ll = &cl->next;
3055
3056 b = cl->buf;
3057 left = b->end - b->last;
3058
3059 if (left < len) {
3060 return NGX_ERROR;
3061 }
3062 }
3063
3064 b->last += ngx_quic_create_ack_range(b->last, ctx->ranges[i].gap,
3065 ctx->ranges[i].range);
3066
3067 frame->u.ack.ranges_length += len;
3068 }
3069
3070 *ll = NULL;
3071
3072 frame->level = ctx->level;
3073 frame->type = NGX_QUIC_FT_ACK;
3074 frame->u.ack.largest = ctx->largest_range;
3075 frame->u.ack.delay = ack_delay;
3076 frame->u.ack.range_count = ctx->nranges;
3077 frame->u.ack.first_range = ctx->first_range;
3078
3079 ngx_quic_queue_frame(qc, frame);
3080
3081 return NGX_OK;
3082 }
3083
3084
3085 static ngx_int_t
3086 ngx_quic_send_cc(ngx_connection_t *c)
3087 {
3088 ngx_quic_frame_t *frame;
3089 ngx_quic_connection_t *qc;
3090
3091 qc = ngx_quic_get_connection(c);
3092
3093 if (qc->draining) {
3094 return NGX_OK;
3095 }
3096
3097 if (!qc->initialized) {
3098 /* try to initialize secrets to send an early error */
3099 if (ngx_quic_init_secrets(c) != NGX_OK) {
3100 return NGX_OK;
3101 }
3102 }
3103
3104 if (qc->closing
3105 && ngx_current_msec - qc->last_cc < NGX_QUIC_CC_MIN_INTERVAL)
3106 {
3107 /* dot not send CC too often */
3108 return NGX_OK;
3109 }
3110
3111 frame = ngx_quic_alloc_frame(c);
3112 if (frame == NULL) {
3113 return NGX_ERROR;
3114 }
3115
3116 frame->level = qc->error_level;
3117 frame->type = NGX_QUIC_FT_CONNECTION_CLOSE;
3118 frame->u.close.error_code = qc->error;
3119 frame->u.close.frame_type = qc->error_ftype;
3120 frame->u.close.app = qc->error_app;
3121
3122 if (qc->error_reason) {
3123 frame->u.close.reason.len = ngx_strlen(qc->error_reason);
3124 frame->u.close.reason.data = (u_char *) qc->error_reason;
3125 }
3126
3127 ngx_quic_queue_frame(qc, frame);
3128
3129 qc->last_cc = ngx_current_msec;
3130
3131 return ngx_quic_output(c);
3132 }
3133
3134
3135 static ngx_int_t
3136 ngx_quic_send_new_token(ngx_connection_t *c)
3137 {
3138 ngx_str_t token;
3139 ngx_quic_frame_t *frame;
3140 ngx_quic_connection_t *qc;
3141
3142 qc = ngx_quic_get_connection(c);
3143
3144 if (!qc->conf->retry) {
3145 return NGX_OK;
3146 }
3147
3148 if (ngx_quic_new_token(c, &token) != NGX_OK) {
3149 return NGX_ERROR;
3150 }
3151
3152 frame = ngx_quic_alloc_frame(c);
3153 if (frame == NULL) {
3154 return NGX_ERROR;
3155 }
3156
3157 frame->level = ssl_encryption_application;
3158 frame->type = NGX_QUIC_FT_NEW_TOKEN;
3159 frame->u.token.length = token.len;
3160 frame->u.token.data = token.data;
3161
3162 ngx_quic_queue_frame(qc, frame);
3163
3164 return NGX_OK;
3165 }
3166
3167
3168 static ngx_int_t
3169 ngx_quic_handle_ack_frame(ngx_connection_t *c, ngx_quic_header_t *pkt,
3170 ngx_quic_frame_t *f)
3171 {
3172 ssize_t n;
3173 u_char *pos, *end;
3174 uint64_t min, max, gap, range;
3175 ngx_msec_t send_time;
3176 ngx_uint_t i;
3177 ngx_quic_send_ctx_t *ctx;
3178 ngx_quic_ack_frame_t *ack;
3179 ngx_quic_connection_t *qc;
3180
3181 qc = ngx_quic_get_connection(c);
3182
3183 ctx = ngx_quic_get_send_ctx(qc, pkt->level);
3184
3185 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
3186 "quic ngx_quic_handle_ack_frame level:%d", pkt->level);
3187
3188 ack = &f->u.ack;
3189
3190 /*
3191 * If any computed packet number is negative, an endpoint MUST
3192 * generate a connection error of type FRAME_ENCODING_ERROR.
3193 * (19.3.1)
3194 */
3195
3196 if (ack->first_range > ack->largest) {
3197 qc->error = NGX_QUIC_ERR_FRAME_ENCODING_ERROR;
3198 ngx_log_error(NGX_LOG_INFO, c->log, 0,
3199 "quic invalid first range in ack frame");
3200 return NGX_ERROR;
3201 }
3202
3203 min = ack->largest - ack->first_range;
3204 max = ack->largest;
3205
3206 if (ngx_quic_handle_ack_frame_range(c, ctx, min, max, &send_time)
3207 != NGX_OK)
3208 {
3209 return NGX_ERROR;
3210 }
3211
3212 /* 13.2.3. Receiver Tracking of ACK Frames */
3213 if (ctx->largest_ack < max || ctx->largest_ack == NGX_QUIC_UNSET_PN) {
3214 ctx->largest_ack = max;
3215 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
3216 "quic updated largest received ack:%uL", max);
3217
3218 /*
3219 * An endpoint generates an RTT sample on receiving an
3220 * ACK frame that meets the following two conditions:
3221 *
3222 * - the largest acknowledged packet number is newly acknowledged
3223 * - at least one of the newly acknowledged packets was ack-eliciting.
3224 */
3225
3226 if (send_time != NGX_TIMER_INFINITE) {
3227 ngx_quic_rtt_sample(c, ack, pkt->level, send_time);
3228 }
3229 }
3230
3231 if (f->data) {
3232 pos = f->data->buf->pos;
3233 end = f->data->buf->last;
3234
3235 } else {
3236 pos = NULL;
3237 end = NULL;
3238 }
3239
3240 for (i = 0; i < ack->range_count; i++) {
3241
3242 n = ngx_quic_parse_ack_range(pkt->log, pos, end, &gap, &range);
3243 if (n == NGX_ERROR) {
3244 return NGX_ERROR;
3245 }
3246 pos += n;
3247
3248 if (gap + 2 > min) {
3249 qc->error = NGX_QUIC_ERR_FRAME_ENCODING_ERROR;
3250 ngx_log_error(NGX_LOG_INFO, c->log, 0,
3251 "quic invalid range:%ui in ack frame", i);
3252 return NGX_ERROR;
3253 }
3254
3255 max = min - gap - 2;
3256
3257 if (range > max) {
3258 qc->error = NGX_QUIC_ERR_FRAME_ENCODING_ERROR;
3259 ngx_log_error(NGX_LOG_INFO, c->log, 0,
3260 "quic invalid range:%ui in ack frame", i);
3261 return NGX_ERROR;
3262 }
3263
3264 min = max - range;
3265
3266 if (ngx_quic_handle_ack_frame_range(c, ctx, min, max, &send_time)
3267 != NGX_OK)
3268 {
3269 return NGX_ERROR;
3270 }
3271 }
3272
3273 return ngx_quic_detect_lost(c);
3274 }
3275
3276
3277 static ngx_int_t
3278 ngx_quic_handle_ack_frame_range(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx,
3279 uint64_t min, uint64_t max, ngx_msec_t *send_time)
3280 {
3281 ngx_uint_t found;
3282 ngx_queue_t *q;
3283 ngx_quic_frame_t *f;
3284 ngx_quic_connection_t *qc;
3285
3286 qc = ngx_quic_get_connection(c);
3287
3288 *send_time = NGX_TIMER_INFINITE;
3289 found = 0;
3290
3291 q = ngx_queue_last(&ctx->sent);
3292
3293 while (q != ngx_queue_sentinel(&ctx->sent)) {
3294
3295 f = ngx_queue_data(q, ngx_quic_frame_t, queue);
3296 q = ngx_queue_prev(q);
3297
3298 if (f->pnum >= min && f->pnum <= max) {
3299 ngx_quic_congestion_ack(c, f);
3300
3301 switch (f->type) {
3302 case NGX_QUIC_FT_ACK:
3303 case NGX_QUIC_FT_ACK_ECN:
3304 ngx_quic_drop_ack_ranges(c, ctx, f->u.ack.largest);
3305 break;
3306
3307 case NGX_QUIC_FT_STREAM0:
3308 case NGX_QUIC_FT_STREAM1:
3309 case NGX_QUIC_FT_STREAM2:
3310 case NGX_QUIC_FT_STREAM3:
3311 case NGX_QUIC_FT_STREAM4:
3312 case NGX_QUIC_FT_STREAM5:
3313 case NGX_QUIC_FT_STREAM6:
3314 case NGX_QUIC_FT_STREAM7:
3315 ngx_quic_handle_stream_ack(c, f);
3316 break;
3317 }
3318
3319 if (f->pnum == max) {
3320 *send_time = f->last;
3321 }
3322
3323 ngx_queue_remove(&f->queue);
3324 ngx_quic_free_frame(c, f);
3325 found = 1;
3326 }
3327 }
3328
3329 if (!found) {
3330
3331 if (max < ctx->pnum) {
3332 /* duplicate ACK or ACK for non-ack-eliciting frame */
3333 return NGX_OK;
3334 }
3335
3336 ngx_log_error(NGX_LOG_INFO, c->log, 0,
3337 "quic ACK for the packet not sent");
3338
3339 qc->error = NGX_QUIC_ERR_PROTOCOL_VIOLATION;
3340 qc->error_ftype = NGX_QUIC_FT_ACK;
3341 qc->error_reason = "unknown packet number";
3342
3343 return NGX_ERROR;
3344 }
3345
3346 if (!qc->push.timer_set) {
3347 ngx_post_event(&qc->push, &ngx_posted_events);
3348 }
3349
3350 qc->pto_count = 0;
3351
3352 return NGX_OK;
3353 }
3354
3355
3356 static void
3357 ngx_quic_rtt_sample(ngx_connection_t *c, ngx_quic_ack_frame_t *ack,
3358 enum ssl_encryption_level_t level, ngx_msec_t send_time)
3359 {
3360 ngx_msec_t latest_rtt, ack_delay, adjusted_rtt, rttvar_sample;
3361 ngx_quic_connection_t *qc;
3362
3363 qc = ngx_quic_get_connection(c);
3364
3365 latest_rtt = ngx_current_msec - send_time;
3366 qc->latest_rtt = latest_rtt;
3367
3368 if (qc->min_rtt == NGX_TIMER_INFINITE) {
3369 qc->min_rtt = latest_rtt;
3370 qc->avg_rtt = latest_rtt;
3371 qc->rttvar = latest_rtt / 2;
3372
3373 } else {
3374 qc->min_rtt = ngx_min(qc->min_rtt, latest_rtt);
3375
3376 ack_delay = ack->delay * (1 << qc->ctp.ack_delay_exponent) / 1000;
3377
3378 if (c->ssl->handshaked) {
3379 ack_delay = ngx_min(ack_delay, qc->ctp.max_ack_delay);
3380 }
3381
3382 adjusted_rtt = latest_rtt;
3383
3384 if (qc->min_rtt + ack_delay < latest_rtt) {
3385 adjusted_rtt -= ack_delay;
3386 }
3387
3388 qc->avg_rtt = 0.875 * qc->avg_rtt + 0.125 * adjusted_rtt;
3389 rttvar_sample = ngx_abs((ngx_msec_int_t) (qc->avg_rtt - adjusted_rtt));
3390 qc->rttvar = 0.75 * qc->rttvar + 0.25 * rttvar_sample;
3391 }
3392
3393 ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
3394 "quic rtt sample latest:%M min:%M avg:%M var:%M",
3395 latest_rtt, qc->min_rtt, qc->avg_rtt, qc->rttvar);
3396 }
3397
3398
3399 static ngx_inline ngx_msec_t
3400 ngx_quic_pto(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx)
3401 {
3402 ngx_msec_t duration;
3403 ngx_quic_connection_t *qc;
3404
3405 qc = ngx_quic_get_connection(c);
3406
3407 /* PTO calculation: quic-recovery, Appendix 8 */
3408 duration = qc->avg_rtt;
3409
3410 duration += ngx_max(4 * qc->rttvar, NGX_QUIC_TIME_GRANULARITY);
3411 duration <<= qc->pto_count;
3412
3413 if (qc->congestion.in_flight == 0) { /* no in-flight packets */
3414 return duration;
3415 }
3416
3417 if (ctx->level == ssl_encryption_application && c->ssl->handshaked) {
3418 duration += qc->ctp.max_ack_delay << qc->pto_count;
3419 }
3420
3421 return duration;
3422 }
3423
3424
3425 static void
3426 ngx_quic_handle_stream_ack(ngx_connection_t *c, ngx_quic_frame_t *f)
3427 {
3428 uint64_t sent, unacked;
3429 ngx_event_t *wev;
3430 ngx_quic_stream_t *sn;
3431 ngx_quic_connection_t *qc;
3432
3433 qc = ngx_quic_get_connection(c);
3434
3435 sn = ngx_quic_find_stream(&qc->streams.tree, f->u.stream.stream_id);
3436 if (sn == NULL) {
3437 return;
3438 }
3439
3440 wev = sn->c->write;
3441 sent = sn->c->sent;
3442 unacked = sent - sn->acked;
3443
3444 if (unacked >= NGX_QUIC_STREAM_BUFSIZE && wev->active) {
3445 wev->ready = 1;
3446 ngx_post_event(wev, &ngx_posted_events);
3447 }
3448
3449 sn->acked += f->u.stream.length;
3450
3451 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, sn->c->log, 0,
3452 "quic stream ack len:%uL acked:%uL unacked:%uL",
3453 f->u.stream.length, sn->acked, sent - sn->acked);
3454 }
3455
3456
3457 static ngx_int_t
3458 ngx_quic_handle_ordered_frame(ngx_connection_t *c, ngx_quic_frames_stream_t *fs,
3459 ngx_quic_frame_t *frame, ngx_quic_frame_handler_pt handler, void *data)
3460 {
3461 size_t full_len;
3462 ngx_int_t rc;
3463 ngx_queue_t *q;
3464 ngx_quic_ordered_frame_t *f;
3465
3466 f = &frame->u.ord;
3467
3468 if (f->offset > fs->received) {
3469 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
3470 "quic out-of-order frame: expecting:%uL got:%uL",
3471 fs->received, f->offset);
3472
3473 return ngx_quic_buffer_frame(c, fs, frame);
3474 }
3475
3476 if (f->offset < fs->received) {
3477
3478 if (ngx_quic_adjust_frame_offset(c, frame, fs->received)
3479 == NGX_DONE)
3480 {
3481 /* old/duplicate data range */
3482 return handler == ngx_quic_crypto_input ? NGX_DECLINED : NGX_OK;
3483 }
3484
3485 /* intersecting data range, frame modified */
3486 }
3487
3488 /* f->offset == fs->received */
3489
3490 rc = handler(c, frame, data);
3491 if (rc == NGX_ERROR) {
3492 return NGX_ERROR;
3493
3494 } else if (rc == NGX_DONE) {
3495 /* handler destroyed stream, queue no longer exists */
3496 return NGX_OK;
3497 }
3498
3499 /* rc == NGX_OK */
3500
3501 fs->received += f->length;
3502
3503 /* now check the queue if we can continue with buffered frames */
3504
3505 do {
3506 q = ngx_queue_head(&fs->frames);
3507 if (q == ngx_queue_sentinel(&fs->frames)) {
3508 break;
3509 }
3510
3511 frame = ngx_queue_data(q, ngx_quic_frame_t, queue);
3512 f = &frame->u.ord;
3513
3514 if (f->offset > fs->received) {
3515 /* gap found, nothing more to do */
3516 break;
3517 }
3518
3519 full_len = f->length;
3520
3521 if (f->offset < fs->received) {
3522
3523 if (ngx_quic_adjust_frame_offset(c, frame, fs->received)
3524 == NGX_DONE)
3525 {
3526 /* old/duplicate data range */
3527 ngx_queue_remove(q);
3528 fs->total -= f->length;
3529
3530 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
3531 "quic skipped buffered frame, total:%ui",
3532 fs->total);
3533 ngx_quic_free_frame(c, frame);
3534 continue;
3535 }
3536
3537 /* frame was adjusted, proceed to input */
3538 }
3539
3540 /* f->offset == fs->received */
3541
3542 rc = handler(c, frame, data);
3543
3544 if (rc == NGX_ERROR) {
3545 return NGX_ERROR;
3546
3547 } else if (rc == NGX_DONE) {
3548 /* handler destroyed stream, queue no longer exists */
3549 return NGX_OK;
3550 }
3551
3552 fs->received += f->length;
3553 fs->total -= full_len;
3554
3555 ngx_queue_remove(q);
3556
3557 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
3558 "quic consumed buffered frame, total:%ui", fs->total);
3559
3560 ngx_quic_free_frame(c, frame);
3561
3562 } while (1);
3563
3564 return NGX_OK;
3565 }
3566
3567
3568 static ngx_int_t
3569 ngx_quic_adjust_frame_offset(ngx_connection_t *c, ngx_quic_frame_t *frame,
3570 uint64_t offset_in)
3571 {
3572 size_t tail, n;
3573 ngx_buf_t *b;
3574 ngx_chain_t *cl;
3575 ngx_quic_ordered_frame_t *f;
3576
3577 f = &frame->u.ord;
3578
3579 tail = offset_in - f->offset;
3580
3581 if (tail >= f->length) {
3582 /* range preceeding already received data or duplicate, ignore */
3583
3584 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
3585 "quic old or duplicate data in ordered frame, ignored");
3586 return NGX_DONE;
3587 }
3588
3589 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
3590 "quic adjusted ordered frame data start to expected offset");
3591
3592 /* intersecting range: adjust data size */
3593
3594 f->offset += tail;
3595 f->length -= tail;
3596
3597 for (cl = frame->data; cl; cl = cl->next) {
3598 b = cl->buf;
3599 n = ngx_buf_size(b);
3600
3601 if (n >= tail) {
3602 b->pos += tail;
3603 break;
3604 }
3605
3606 cl->buf->pos = cl->buf->last;
3607 tail -= n;
3608 }
3609
3610 return NGX_OK;
3611 }
3612
3613
3614 static ngx_int_t
3615 ngx_quic_buffer_frame(ngx_connection_t *c, ngx_quic_frames_stream_t *fs,
3616 ngx_quic_frame_t *frame)
3617 {
3618 ngx_queue_t *q;
3619 ngx_quic_frame_t *dst, *item;
3620 ngx_quic_ordered_frame_t *f, *df;
3621
3622 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
3623 "quic ngx_quic_buffer_frame");
3624
3625 f = &frame->u.ord;
3626
3627 /* frame start offset is in the future, buffer it */
3628
3629 dst = ngx_quic_alloc_frame(c);
3630 if (dst == NULL) {
3631 return NGX_ERROR;
3632 }
3633
3634 ngx_memcpy(dst, frame, sizeof(ngx_quic_frame_t));
3635
3636 dst->data = ngx_quic_copy_chain(c, frame->data, 0);
3637 if (dst->data == NGX_CHAIN_ERROR) {
3638 return NGX_ERROR;
3639 }
3640
3641 df = &dst->u.ord;
3642
3643 fs->total += f->length;
3644
3645 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
3646 "quic ordered frame with unexpected offset:"
3647 " buffered total:%ui", fs->total);
3648
3649 if (ngx_queue_empty(&fs->frames)) {
3650 ngx_queue_insert_after(&fs->frames, &dst->queue);
3651 return NGX_OK;
3652 }
3653
3654 for (q = ngx_queue_last(&fs->frames);
3655 q != ngx_queue_sentinel(&fs->frames);
3656 q = ngx_queue_prev(q))
3657 {
3658 item = ngx_queue_data(q, ngx_quic_frame_t, queue);
3659 f = &item->u.ord;
3660
3661 if (f->offset < df->offset) {
3662 ngx_queue_insert_after(q, &dst->queue);
3663 return NGX_OK;
3664 }
3665 }
3666
3667 ngx_queue_insert_after(&fs->frames, &dst->queue);
3668
3669 return NGX_OK;
3670 }
3671
3672
3673 static ngx_int_t
3674 ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt,
3675 ngx_quic_frame_t *frame)
3676 {
3677 uint64_t last;
3678 ngx_int_t rc;
3679 ngx_quic_send_ctx_t *ctx;
3680 ngx_quic_connection_t *qc;
3681 ngx_quic_crypto_frame_t *f;
3682 ngx_quic_frames_stream_t *fs;
3683
3684 qc = ngx_quic_get_connection(c);
3685 fs = &qc->crypto[pkt->level];
3686 f = &frame->u.crypto;
3687
3688 /* no overflow since both values are 62-bit */
3689 last = f->offset + f->length;
3690
3691 if (last > fs->received && last - fs->received > NGX_QUIC_MAX_BUFFERED) {
3692 qc->error = NGX_QUIC_ERR_CRYPTO_BUFFER_EXCEEDED;
3693 return NGX_ERROR;
3694 }
3695
3696 rc = ngx_quic_handle_ordered_frame(c, fs, frame, ngx_quic_crypto_input,
3697 NULL);
3698 if (rc != NGX_DECLINED) {
3699 return rc;
3700 }
3701
3702 /* speeding up handshake completion */
3703
3704 if (pkt->level == ssl_encryption_initial) {
3705 ctx = ngx_quic_get_send_ctx(qc, pkt->level);
3706
3707 if (!ngx_queue_empty(&ctx->sent)) {
3708 ngx_quic_resend_frames(c, ctx);
3709
3710 ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_handshake);
3711 while (!ngx_queue_empty(&ctx->sent)) {
3712 ngx_quic_resend_frames(c, ctx);
3713 }
3714 }
3715 }
3716
3717 return NGX_OK;
3718 }
3719
3720
3721 static ngx_int_t
3722 ngx_quic_crypto_input(ngx_connection_t *c, ngx_quic_frame_t *frame, void *data)
3723 {
3724 int n, sslerr;
3725 ngx_buf_t *b;
3726 ngx_chain_t *cl;
3727 ngx_ssl_conn_t *ssl_conn;
3728 ngx_quic_connection_t *qc;
3729
3730 qc = ngx_quic_get_connection(c);
3731
3732 ssl_conn = c->ssl->connection;
3733
3734 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
3735 "quic SSL_quic_read_level:%d SSL_quic_write_level:%d",
3736 (int) SSL_quic_read_level(ssl_conn),
3737 (int) SSL_quic_write_level(ssl_conn));
3738
3739 for (cl = frame->data; cl; cl = cl->next) {
3740 b = cl->buf;
3741
3742 if (!SSL_provide_quic_data(ssl_conn, SSL_quic_read_level(ssl_conn),
3743 b->pos, b->last - b->pos))
3744 {
3745 ngx_ssl_error(NGX_LOG_INFO, c->log, 0,
3746 "SSL_provide_quic_data() failed");
3747 return NGX_ERROR;
3748 }
3749 }
3750
3751 n = SSL_do_handshake(ssl_conn);
3752
3753 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
3754 "quic SSL_quic_read_level:%d SSL_quic_write_level:%d",
3755 (int) SSL_quic_read_level(ssl_conn),
3756 (int) SSL_quic_write_level(ssl_conn));
3757
3758 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_do_handshake: %d", n);
3759
3760 if (n <= 0) {
3761 sslerr = SSL_get_error(ssl_conn, n);
3762
3763 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_get_error: %d",
3764 sslerr);
3765
3766 if (sslerr != SSL_ERROR_WANT_READ) {
3767 ngx_ssl_error(NGX_LOG_ERR, c->log, 0, "SSL_do_handshake() failed");
3768 return NGX_ERROR;
3769 }
3770
3771 return NGX_OK;
3772 }
3773
3774 if (SSL_in_init(ssl_conn)) {
3775 return NGX_OK;
3776 }
3777
3778 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
3779 "quic ssl cipher:%s", SSL_get_cipher(ssl_conn));
3780
3781 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
3782 "quic handshake completed successfully");
3783
3784 c->ssl->handshaked = 1;
3785
3786 frame = ngx_quic_alloc_frame(c);
3787 if (frame == NULL) {
3788 return NGX_ERROR;
3789 }
3790
3791 /* 12.4 Frames and frame types, figure 8 */
3792 frame->level = ssl_encryption_application;
3793 frame->type = NGX_QUIC_FT_HANDSHAKE_DONE;
3794 ngx_quic_queue_frame(qc, frame);
3795
3796 if (ngx_quic_send_new_token(c) != NGX_OK) {
3797 return NGX_ERROR;
3798 }
3799
3800 /*
3801 * Generating next keys before a key update is received.
3802 * See quic-tls 9.4 Header Protection Timing Side-Channels.
3803 */
3804
3805 if (ngx_quic_keys_update(c, qc->keys) != NGX_OK) {
3806 return NGX_ERROR;
3807 }
3808
3809 /*
3810 * 4.10.2 An endpoint MUST discard its handshake keys
3811 * when the TLS handshake is confirmed
3812 */
3813 ngx_quic_discard_ctx(c, ssl_encryption_handshake);
3814
3815 if (ngx_quic_issue_server_ids(c) != NGX_OK) {
3816 return NGX_ERROR;
3817 }
3818
3819 return NGX_OK;
3820 }
3821
3822
3823 static ngx_int_t
3824 ngx_quic_handle_stream_frame(ngx_connection_t *c, ngx_quic_header_t *pkt,
3825 ngx_quic_frame_t *frame)
3826 {
3827 size_t window;
3828 uint64_t last;
3829 ngx_buf_t *b;
3830 ngx_pool_t *pool;
3831 ngx_connection_t *sc;
3832 ngx_quic_stream_t *sn;
3833 ngx_quic_connection_t *qc;
3834 ngx_quic_stream_frame_t *f;
3835 ngx_quic_frames_stream_t *fs;
3836
3837 qc = ngx_quic_get_connection(c);
3838 f = &frame->u.stream;
3839
3840 if ((f->stream_id & NGX_QUIC_STREAM_UNIDIRECTIONAL)
3841 && (f->stream_id & NGX_QUIC_STREAM_SERVER_INITIATED))
3842 {
3843 qc->error = NGX_QUIC_ERR_STREAM_STATE_ERROR;
3844 return NGX_ERROR;
3845 }
3846
3847 /* no overflow since both values are 62-bit */
3848 last = f->offset + f->length;
3849
3850 sn = ngx_quic_find_stream(&qc->streams.tree, f->stream_id);
3851
3852 if (sn == NULL) {
3853 sn = ngx_quic_create_client_stream(c, f->stream_id);
3854
3855 if (sn == NULL) {
3856 return NGX_ERROR;
3857 }
3858
3859 if (sn == NGX_QUIC_STREAM_GONE) {
3860 return NGX_OK;
3861 }
3862
3863 sc = sn->c;
3864 fs = &sn->fs;
3865 b = sn->b;
3866 window = b->end - b->last;
3867
3868 if (last > window) {
3869 qc->error = NGX_QUIC_ERR_FLOW_CONTROL_ERROR;
3870 goto cleanup;
3871 }
3872
3873 if (ngx_quic_handle_ordered_frame(c, fs, frame, ngx_quic_stream_input,
3874 sn)
3875 != NGX_OK)
3876 {
3877 goto cleanup;
3878 }
3879
3880 sc->listening->handler(sc);
3881
3882 return NGX_OK;
3883 }
3884
3885 fs = &sn->fs;
3886 b = sn->b;
3887 window = (b->pos - b->start) + (b->end - b->last);
3888
3889 if (last > fs->received && last - fs->received > window) {
3890 qc->error = NGX_QUIC_ERR_FLOW_CONTROL_ERROR;
3891 return NGX_ERROR;
3892 }
3893
3894 return ngx_quic_handle_ordered_frame(c, fs, frame, ngx_quic_stream_input,
3895 sn);
3896
3897 cleanup:
3898
3899 pool = sc->pool;
3900
3901 ngx_close_connection(sc);
3902 ngx_destroy_pool(pool);
3903
3904 return NGX_ERROR;
3905 }
3906
3907
3908 static ngx_int_t
3909 ngx_quic_stream_input(ngx_connection_t *c, ngx_quic_frame_t *frame, void *data)
3910 {
3911 uint64_t id;
3912 ngx_buf_t *b;
3913 ngx_event_t *rev;
3914 ngx_chain_t *cl;
3915 ngx_quic_stream_t *sn;
3916 ngx_quic_connection_t *qc;
3917 ngx_quic_stream_frame_t *f;
3918
3919 qc = ngx_quic_get_connection(c);
3920 sn = data;
3921
3922 f = &frame->u.stream;
3923 id = f->stream_id;
3924
3925 b = sn->b;
3926
3927 if ((size_t) ((b->pos - b->start) + (b->end - b->last)) < f->length) {
3928 ngx_log_error(NGX_LOG_INFO, c->log, 0,
3929 "quic no space in stream buffer");
3930 return NGX_ERROR;
3931 }
3932
3933 if ((size_t) (b->end - b->last) < f->length) {
3934 b->last = ngx_movemem(b->start, b->pos, b->last - b->pos);
3935 b->pos = b->start;
3936 }
3937
3938 for (cl = frame->data; cl; cl = cl->next) {
3939 b->last = ngx_cpymem(b->last, cl->buf->pos,
3940 cl->buf->last - cl->buf->pos);
3941 }
3942
3943 rev = sn->c->read;
3944 rev->ready = 1;
3945
3946 if (f->fin) {
3947 rev->pending_eof = 1;
3948 }
3949
3950 if (rev->active) {
3951 rev->handler(rev);
3952 }
3953
3954 /* check if stream was destroyed by handler */
3955 if (ngx_quic_find_stream(&qc->streams.tree, id) == NULL) {
3956 return NGX_DONE;
3957 }
3958
3959 return NGX_OK;
3960 }
3961
3962
3963 static ngx_int_t
3964 ngx_quic_handle_max_data_frame(ngx_connection_t *c,
3965 ngx_quic_max_data_frame_t *f)
3966 {
3967 ngx_event_t *wev;
3968 ngx_rbtree_t *tree;
3969 ngx_rbtree_node_t *node;
3970 ngx_quic_stream_t *qs;
3971 ngx_quic_connection_t *qc;
3972
3973 qc = ngx_quic_get_connection(c);
3974 tree = &qc->streams.tree;
3975
3976 if (f->max_data <= qc->streams.send_max_data) {
3977 return NGX_OK;
3978 }
3979
3980 if (qc->streams.sent >= qc->streams.send_max_data) {
3981
3982 for (node = ngx_rbtree_min(tree->root, tree->sentinel);
3983 node;
3984 node = ngx_rbtree_next(tree, node))
3985 {
3986 qs = (ngx_quic_stream_t *) node;
3987 wev = qs->c->write;
3988
3989 if (wev->active) {
3990 wev->ready = 1;
3991 ngx_post_event(wev, &ngx_posted_events);
3992 }
3993 }
3994 }
3995
3996 qc->streams.send_max_data = f->max_data;
3997
3998 return NGX_OK;
3999 }
4000
4001
4002 static ngx_int_t
4003 ngx_quic_handle_streams_blocked_frame(ngx_connection_t *c,
4004 ngx_quic_header_t *pkt, ngx_quic_streams_blocked_frame_t *f)
4005 {
4006 return NGX_OK;
4007 }
4008
4009
4010 static ngx_int_t
4011 ngx_quic_handle_stream_data_blocked_frame(ngx_connection_t *c,
4012 ngx_quic_header_t *pkt, ngx_quic_stream_data_blocked_frame_t *f)
4013 {
4014 size_t n;
4015 ngx_buf_t *b;
4016 ngx_quic_frame_t *frame;
4017 ngx_quic_stream_t *sn;
4018 ngx_quic_connection_t *qc;
4019
4020 qc = ngx_quic_get_connection(c);
4021
4022 if ((f->id & NGX_QUIC_STREAM_UNIDIRECTIONAL)
4023 && (f->id & NGX_QUIC_STREAM_SERVER_INITIATED))
4024 {
4025 qc->error = NGX_QUIC_ERR_STREAM_STATE_ERROR;
4026 return NGX_ERROR;
4027 }
4028
4029 sn = ngx_quic_find_stream(&qc->streams.tree, f->id);
4030
4031 if (sn == NULL) {
4032 sn = ngx_quic_create_client_stream(c, f->id);
4033
4034 if (sn == NULL) {
4035 return NGX_ERROR;
4036 }
4037
4038 if (sn == NGX_QUIC_STREAM_GONE) {
4039 return NGX_OK;
4040 }
4041
4042 b = sn->b;
4043 n = b->end - b->last;
4044
4045 sn->c->listening->handler(sn->c);
4046
4047 } else {
4048 b = sn->b;
4049 n = sn->fs.received + (b->pos - b->start) + (b->end - b->last);
4050 }
4051
4052 frame = ngx_quic_alloc_frame(c);
4053 if (frame == NULL) {
4054 return NGX_ERROR;
4055 }
4056
4057 frame->level = pkt->level;
4058 frame->type = NGX_QUIC_FT_MAX_STREAM_DATA;
4059 frame->u.max_stream_data.id = f->id;
4060 frame->u.max_stream_data.limit = n;
4061
4062 ngx_quic_queue_frame(qc, frame);
4063
4064 return NGX_OK;
4065 }
4066
4067
4068 static ngx_int_t
4069 ngx_quic_handle_max_stream_data_frame(ngx_connection_t *c,
4070 ngx_quic_header_t *pkt, ngx_quic_max_stream_data_frame_t *f)
4071 {
4072 uint64_t sent;
4073 ngx_event_t *wev;
4074 ngx_quic_stream_t *sn;
4075 ngx_quic_connection_t *qc;
4076
4077 qc = ngx_quic_get_connection(c);
4078
4079 if ((f->id & NGX_QUIC_STREAM_UNIDIRECTIONAL)
4080 && (f->id & NGX_QUIC_STREAM_SERVER_INITIATED) == 0)
4081 {
4082 qc->error = NGX_QUIC_ERR_STREAM_STATE_ERROR;
4083 return NGX_ERROR;
4084 }
4085
4086 sn = ngx_quic_find_stream(&qc->streams.tree, f->id);
4087
4088 if (sn == NULL) {
4089 sn = ngx_quic_create_client_stream(c, f->id);
4090
4091 if (sn == NULL) {
4092 return NGX_ERROR;
4093 }
4094
4095 if (sn == NGX_QUIC_STREAM_GONE) {
4096 return NGX_OK;
4097 }
4098
4099 if (f->limit > sn->send_max_data) {
4100 sn->send_max_data = f->limit;
4101 }
4102
4103 sn->c->listening->handler(sn->c);
4104
4105 return NGX_OK;
4106 }
4107
4108 if (f->limit <= sn->send_max_data) {
4109 return NGX_OK;
4110 }
4111
4112 sent = sn->c->sent;
4113
4114 if (sent >= sn->send_max_data) {
4115 wev = sn->c->write;
4116
4117 if (wev->active) {
4118 wev->ready = 1;
4119 ngx_post_event(wev, &ngx_posted_events);
4120 }
4121 }
4122
4123 sn->send_max_data = f->limit;
4124
4125 return NGX_OK;
4126 }
4127
4128
4129 static ngx_int_t
4130 ngx_quic_handle_reset_stream_frame(ngx_connection_t *c,
4131 ngx_quic_header_t *pkt, ngx_quic_reset_stream_frame_t *f)
4132 {
4133 ngx_event_t *rev;
4134 ngx_connection_t *sc;
4135 ngx_quic_stream_t *sn;
4136 ngx_quic_connection_t *qc;
4137
4138 qc = ngx_quic_get_connection(c);
4139
4140 if ((f->id & NGX_QUIC_STREAM_UNIDIRECTIONAL)
4141 && (f->id & NGX_QUIC_STREAM_SERVER_INITIATED))
4142 {
4143 qc->error = NGX_QUIC_ERR_STREAM_STATE_ERROR;
4144 return NGX_ERROR;
4145 }
4146
4147 sn = ngx_quic_find_stream(&qc->streams.tree, f->id);
4148
4149 if (sn == NULL) {
4150 sn = ngx_quic_create_client_stream(c, f->id);
4151
4152 if (sn == NULL) {
4153 return NGX_ERROR;
4154 }
4155
4156 if (sn == NGX_QUIC_STREAM_GONE) {
4157 return NGX_OK;
4158 }
4159
4160 sc = sn->c;
4161
4162 rev = sc->read;
4163 rev->error = 1;
4164 rev->ready = 1;
4165
4166 sc->listening->handler(sc);
4167
4168 return NGX_OK;
4169 }
4170
4171 rev = sn->c->read;
4172 rev->error = 1;
4173 rev->ready = 1;
4174
4175 if (rev->active) {
4176 rev->handler(rev);
4177 }
4178
4179 return NGX_OK;
4180 }
4181
4182
4183 static ngx_int_t
4184 ngx_quic_handle_stop_sending_frame(ngx_connection_t *c,
4185 ngx_quic_header_t *pkt, ngx_quic_stop_sending_frame_t *f)
4186 {
4187 ngx_event_t *wev;
4188 ngx_connection_t *sc;
4189 ngx_quic_stream_t *sn;
4190 ngx_quic_connection_t *qc;
4191
4192 qc = ngx_quic_get_connection(c);
4193
4194 if ((f->id & NGX_QUIC_STREAM_UNIDIRECTIONAL)
4195 && (f->id & NGX_QUIC_STREAM_SERVER_INITIATED) == 0)
4196 {
4197 qc->error = NGX_QUIC_ERR_STREAM_STATE_ERROR;
4198 return NGX_ERROR;
4199 }
4200
4201 sn = ngx_quic_find_stream(&qc->streams.tree, f->id);
4202
4203 if (sn == NULL) {
4204 sn = ngx_quic_create_client_stream(c, f->id);
4205
4206 if (sn == NULL) {
4207 return NGX_ERROR;
4208 }
4209
4210 if (sn == NGX_QUIC_STREAM_GONE) {
4211 return NGX_OK;
4212 }
4213
4214 sc = sn->c;
4215
4216 wev = sc->write;
4217 wev->error = 1;
4218 wev->ready = 1;
4219
4220 sc->listening->handler(sc);
4221
4222 return NGX_OK;
4223 }
4224
4225 wev = sn->c->write;
4226 wev->error = 1;
4227 wev->ready = 1;
4228
4229 if (wev->active) {
4230 wev->handler(wev);
4231 }
4232
4233 return NGX_OK;
4234 }
4235
4236
4237 static ngx_int_t
4238 ngx_quic_handle_max_streams_frame(ngx_connection_t *c,
4239 ngx_quic_header_t *pkt, ngx_quic_max_streams_frame_t *f)
4240 {
4241 ngx_quic_connection_t *qc;
4242
4243 qc = ngx_quic_get_connection(c);
4244
4245 if (f->bidi) {
4246 if (qc->streams.server_max_streams_bidi < f->limit) {
4247 qc->streams.server_max_streams_bidi = f->limit;
4248
4249 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
4250 "quic max_streams_bidi:%uL", f->limit);
4251 }
4252
4253 } else {
4254 if (qc->streams.server_max_streams_uni < f->limit) {
4255 qc->streams.server_max_streams_uni = f->limit;
4256
4257 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
4258 "quic max_streams_uni:%uL", f->limit);
4259 }
4260 }
4261
4262 return NGX_OK;
4263 }
4264
4265
4266 static ngx_int_t
4267 ngx_quic_handle_path_challenge_frame(ngx_connection_t *c,
4268 ngx_quic_header_t *pkt, ngx_quic_path_challenge_frame_t *f)
4269 {
4270 ngx_quic_frame_t *frame;
4271 ngx_quic_connection_t *qc;
4272
4273 qc = ngx_quic_get_connection(c);
4274
4275 frame = ngx_quic_alloc_frame(c);
4276 if (frame == NULL) {
4277 return NGX_ERROR;
4278 }
4279
4280 frame->level = pkt->level;
4281 frame->type = NGX_QUIC_FT_PATH_RESPONSE;
4282 frame->u.path_response = *f;
4283
4284 ngx_quic_queue_frame(qc, frame);
4285
4286 return NGX_OK;
4287 }
4288
4289
4290 static ngx_int_t
4291 ngx_quic_handle_new_connection_id_frame(ngx_connection_t *c,
4292 ngx_quic_header_t *pkt, ngx_quic_new_conn_id_frame_t *f)
4293 {
4294 ngx_queue_t *q;
4295 ngx_quic_client_id_t *cid, *item;
4296 ngx_quic_connection_t *qc;
4297
4298 qc = ngx_quic_get_connection(c);
4299
4300 if (f->seqnum < qc->max_retired_seqnum) {
4301 /*
4302 * An endpoint that receives a NEW_CONNECTION_ID frame with
4303 * a sequence number smaller than the Retire Prior To field
4304 * of a previously received NEW_CONNECTION_ID frame MUST send
4305 * a corresponding RETIRE_CONNECTION_ID frame that retires
4306 * the newly received connection ID, unless it has already
4307 * done so for that sequence number.
4308 */
4309
4310 if (ngx_quic_retire_connection_id(c, pkt->level, f->seqnum) != NGX_OK) {
4311 return NGX_ERROR;
4312 }
4313
4314 goto retire;
4315 }
4316
4317 cid = NULL;
4318
4319 for (q = ngx_queue_head(&qc->client_ids);
4320 q != ngx_queue_sentinel(&qc->client_ids);
4321 q = ngx_queue_next(q))
4322 {
4323 item = ngx_queue_data(q, ngx_quic_client_id_t, queue);
4324
4325 if (item->seqnum == f->seqnum) {
4326 cid = item;
4327 break;
4328 }
4329 }
4330
4331 if (cid) {
4332 /*
4333 * Transmission errors, timeouts and retransmissions might cause the
4334 * same NEW_CONNECTION_ID frame to be received multiple times
4335 */
4336
4337 if (cid->len != f->len
4338 || ngx_strncmp(cid->id, f->cid, f->len) != 0
4339 || ngx_strncmp(cid->sr_token, f->srt, NGX_QUIC_SR_TOKEN_LEN) != 0)
4340 {
4341 /*
4342 * ..a sequence number is used for different connection IDs,
4343 * the endpoint MAY treat that receipt as a connection error
4344 * of type PROTOCOL_VIOLATION.
4345 */
4346 qc->error = NGX_QUIC_ERR_PROTOCOL_VIOLATION;
4347 qc->error_reason = "seqnum refers to different connection id/token";
4348 return NGX_ERROR;
4349 }
4350
4351 } else {
4352
4353 cid = ngx_quic_alloc_client_id(c, qc);
4354 if (cid == NULL) {
4355 return NGX_ERROR;
4356 }
4357
4358 cid->seqnum = f->seqnum;
4359 cid->len = f->len;
4360 ngx_memcpy(cid->id, f->cid, f->len);
4361
4362 ngx_memcpy(cid->sr_token, f->srt, NGX_QUIC_SR_TOKEN_LEN);
4363
4364 ngx_queue_insert_tail(&qc->client_ids, &cid->queue);
4365 qc->nclient_ids++;
4366
4367 /* always use latest available connection id */
4368 if (f->seqnum > qc->client_seqnum) {
4369 qc->scid.len = cid->len;
4370 qc->scid.data = cid->id;
4371 qc->client_seqnum = f->seqnum;
4372 }
4373 }
4374
4375 retire:
4376
4377 if (qc->max_retired_seqnum && f->retire <= qc->max_retired_seqnum) {
4378 /*
4379 * Once a sender indicates a Retire Prior To value, smaller values sent
4380 * in subsequent NEW_CONNECTION_ID frames have no effect. A receiver
4381 * MUST ignore any Retire Prior To fields that do not increase the
4382 * largest received Retire Prior To value.
4383 */
4384 goto done;
4385 }
4386
4387 qc->max_retired_seqnum = f->retire;
4388
4389 q = ngx_queue_head(&qc->client_ids);
4390
4391 while (q != ngx_queue_sentinel(&qc->client_ids)) {
4392
4393 cid = ngx_queue_data(q, ngx_quic_client_id_t, queue);
4394 q = ngx_queue_next(q);
4395
4396 if (cid->seqnum >= f->retire) {
4397 continue;
4398 }
4399
4400 /* this connection id must be retired */
4401
4402 if (ngx_quic_retire_connection_id(c, pkt->level, cid->seqnum)
4403 != NGX_OK)
4404 {
4405 return NGX_ERROR;
4406 }
4407
4408 ngx_queue_remove(&cid->queue);
4409 ngx_queue_insert_head(&qc->free_client_ids, &cid->queue);
4410 qc->nclient_ids--;
4411 }
4412
4413 done:
4414
4415 if (qc->nclient_ids > qc->tp.active_connection_id_limit) {
4416 /*
4417 * After processing a NEW_CONNECTION_ID frame and
4418 * adding and retiring active connection IDs, if the number of active
4419 * connection IDs exceeds the value advertised in its
4420 * active_connection_id_limit transport parameter, an endpoint MUST
4421 * close the connection with an error of type CONNECTION_ID_LIMIT_ERROR.
4422 */
4423 qc->error = NGX_QUIC_ERR_CONNECTION_ID_LIMIT_ERROR;
4424 qc->error_reason = "too many connection ids received";
4425 return NGX_ERROR;
4426 }
4427
4428 return NGX_OK;
4429 }
4430
4431
4432 static ngx_int_t
4433 ngx_quic_retire_connection_id(ngx_connection_t *c,
4434 enum ssl_encryption_level_t level, uint64_t seqnum)
4435 {
4436 ngx_quic_frame_t *frame;
4437 ngx_quic_connection_t *qc;
4438
4439 qc = ngx_quic_get_connection(c);
4440
4441 frame = ngx_quic_alloc_frame(c);
4442 if (frame == NULL) {
4443 return NGX_ERROR;
4444 }
4445
4446 frame->level = level;
4447 frame->type = NGX_QUIC_FT_RETIRE_CONNECTION_ID;
4448 frame->u.retire_cid.sequence_number = seqnum;
4449
4450 ngx_quic_queue_frame(qc, frame);
4451
4452 return NGX_OK;
4453 }
4454
4455
4456 static ngx_int_t
4457 ngx_quic_handle_retire_connection_id_frame(ngx_connection_t *c,
4458 ngx_quic_header_t *pkt, ngx_quic_retire_cid_frame_t *f)
4459 {
4460 ngx_queue_t *q;
4461 ngx_quic_server_id_t *sid;
4462 ngx_quic_connection_t *qc;
4463
4464 qc = ngx_quic_get_connection(c);
4465
4466 for (q = ngx_queue_head(&qc->server_ids);
4467 q != ngx_queue_sentinel(&qc->server_ids);
4468 q = ngx_queue_next(q))
4469 {
4470 sid = ngx_queue_data(q, ngx_quic_server_id_t, queue);
4471
4472 if (sid->seqnum == f->sequence_number) {
4473 ngx_queue_remove(q);
4474 ngx_queue_insert_tail(&qc->free_server_ids, &sid->queue);
4475 ngx_rbtree_delete(&c->listening->rbtree, &sid->udp.node);
4476 qc->nserver_ids--;
4477 break;
4478 }
4479 }
4480
4481 return ngx_quic_issue_server_ids(c);
4482 }
4483
4484
4485 static ngx_int_t
4486 ngx_quic_issue_server_ids(ngx_connection_t *c)
4487 {
4488 ngx_str_t dcid;
4489 ngx_uint_t n;
4490 ngx_quic_frame_t *frame;
4491 ngx_quic_server_id_t *sid;
4492 ngx_quic_connection_t *qc;
4493 u_char id[NGX_QUIC_SERVER_CID_LEN];
4494
4495 qc = ngx_quic_get_connection(c);
4496
4497 n = ngx_min(NGX_QUIC_MAX_SERVER_IDS, qc->ctp.active_connection_id_limit);
4498
4499 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
4500 "quic issue server ids has:%ui max:%ui", qc->nserver_ids, n);
4501
4502 while (qc->nserver_ids < n) {
4503 if (ngx_quic_create_server_id(c, id) != NGX_OK) {
4504 return NGX_ERROR;
4505 }
4506
4507 dcid.len = NGX_QUIC_SERVER_CID_LEN;
4508 dcid.data = id;
4509
4510 sid = ngx_quic_insert_server_id(c, &dcid);
4511 if (sid == NULL) {
4512 return NGX_ERROR;
4513 }
4514
4515 frame = ngx_quic_alloc_frame(c);
4516 if (frame == NULL) {
4517 return NGX_ERROR;
4518 }
4519
4520 frame->level = ssl_encryption_application;
4521 frame->type = NGX_QUIC_FT_NEW_CONNECTION_ID;
4522 frame->u.ncid.seqnum = sid->seqnum;
4523 frame->u.ncid.retire = 0;
4524 frame->u.ncid.len = NGX_QUIC_SERVER_CID_LEN;
4525 ngx_memcpy(frame->u.ncid.cid, id, NGX_QUIC_SERVER_CID_LEN);
4526
4527 if (ngx_quic_new_sr_token(c, &dcid, &qc->conf->sr_token_key,
4528 frame->u.ncid.srt)
4529 != NGX_OK)
4530 {
4531 return NGX_ERROR;
4532 }
4533
4534 ngx_quic_queue_frame(qc, frame);
4535 }
4536
4537 return NGX_OK;
4538 }
4539
4540
4541 static void
4542 ngx_quic_clear_temp_server_ids(ngx_connection_t *c)
4543 {
4544 ngx_queue_t *q, *next;
4545 ngx_quic_server_id_t *sid;
4546 ngx_quic_connection_t *qc;
4547
4548 qc = ngx_quic_get_connection(c);
4549
4550 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
4551 "quic clear temp server ids");
4552
4553 for (q = ngx_queue_head(&qc->server_ids);
4554 q != ngx_queue_sentinel(&qc->server_ids);
4555 q = next)
4556 {
4557 next = ngx_queue_next(q);
4558 sid = ngx_queue_data(q, ngx_quic_server_id_t, queue);
4559
4560 if (sid->seqnum != NGX_QUIC_UNSET_PN) {
4561 continue;
4562 }
4563
4564 ngx_queue_remove(q);
4565 ngx_queue_insert_tail(&qc->free_server_ids, &sid->queue);
4566 ngx_rbtree_delete(&c->listening->rbtree, &sid->udp.node);
4567 qc->nserver_ids--;
4568 }
4569 }
4570
4571
4572 static ngx_quic_server_id_t *
4573 ngx_quic_insert_server_id(ngx_connection_t *c, ngx_str_t *id)
4574 {
4575 ngx_str_t dcid;
4576 ngx_quic_server_id_t *sid;
4577 ngx_quic_connection_t *qc;
4578
4579 qc = ngx_quic_get_connection(c);
4580
4581 sid = ngx_quic_alloc_server_id(c, qc);
4582 if (sid == NULL) {
4583 return NULL;
4584 }
4585
4586 sid->seqnum = qc->server_seqnum;
4587
4588 if (qc->server_seqnum != NGX_QUIC_UNSET_PN) {
4589 qc->server_seqnum++;
4590 }
4591
4592 sid->len = id->len;
4593 ngx_memcpy(sid->id, id->data, id->len);
4594
4595 ngx_queue_insert_tail(&qc->server_ids, &sid->queue);
4596 qc->nserver_ids++;
4597
4598 dcid.data = sid->id;
4599 dcid.len = sid->len;
4600
4601 ngx_insert_udp_connection(c, &sid->udp, &dcid);
4602
4603 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
4604 "quic insert server id seqnum:%uL id len:%uz %xV",
4605 sid->seqnum, id->len, id);
4606
4607 return sid;
4608 }
4609
4610
4611 static ngx_quic_client_id_t *
4612 ngx_quic_alloc_client_id(ngx_connection_t *c, ngx_quic_connection_t *qc)
4613 {
4614 ngx_queue_t *q;
4615 ngx_quic_client_id_t *cid;
4616
4617 if (!ngx_queue_empty(&qc->free_client_ids)) {
4618
4619 q = ngx_queue_head(&qc->free_client_ids);
4620 cid = ngx_queue_data(q, ngx_quic_client_id_t, queue);
4621
4622 ngx_queue_remove(&cid->queue);
4623
4624 ngx_memzero(cid, sizeof(ngx_quic_client_id_t));
4625
4626 } else {
4627
4628 cid = ngx_pcalloc(c->pool, sizeof(ngx_quic_client_id_t));
4629 if (cid == NULL) {
4630 return NULL;
4631 }
4632 }
4633
4634 return cid;
4635 }
4636
4637
4638 static ngx_quic_server_id_t *
4639 ngx_quic_alloc_server_id(ngx_connection_t *c, ngx_quic_connection_t *qc)
4640 {
4641 ngx_queue_t *q;
4642 ngx_quic_server_id_t *sid;
4643
4644 if (!ngx_queue_empty(&qc->free_server_ids)) {
4645
4646 q = ngx_queue_head(&qc->free_server_ids);
4647 sid = ngx_queue_data(q, ngx_quic_server_id_t, queue);
4648
4649 ngx_queue_remove(&sid->queue);
4650
4651 ngx_memzero(sid, sizeof(ngx_quic_server_id_t));
4652
4653 } else {
4654
4655 sid = ngx_pcalloc(c->pool, sizeof(ngx_quic_server_id_t));
4656 if (sid == NULL) {
4657 return NULL;
4658 }
4659 }
4660
4661 return sid;
4662 }
4663
4664
4665 static void
4666 ngx_quic_queue_frame(ngx_quic_connection_t *qc, ngx_quic_frame_t *frame)
4667 {
4668 ngx_quic_send_ctx_t *ctx;
4669
4670 ctx = ngx_quic_get_send_ctx(qc, frame->level);
4671
4672 ngx_queue_insert_tail(&ctx->frames, &frame->queue);
4673
4674 frame->len = ngx_quic_create_frame(NULL, frame);
4675 /* always succeeds */
4676
4677 if (qc->closing) {
4678 return;
4679 }
4680
4681 ngx_post_event(&qc->push, &ngx_posted_events);
4682 }
4683
4684
4685 static ngx_int_t
4686 ngx_quic_output(ngx_connection_t *c)
4687 {
4688 off_t max;
4689 size_t len, min;
4690 ssize_t n;
4691 u_char *p;
4692 ngx_uint_t i, pad;
4693 ngx_quic_send_ctx_t *ctx;
4694 ngx_quic_connection_t *qc;
4695 static u_char dst[NGX_QUIC_MAX_UDP_PAYLOAD_SIZE];
4696
4697 c->log->action = "sending frames";
4698
4699 qc = ngx_quic_get_connection(c);
4700
4701 for ( ;; ) {
4702 p = dst;
4703
4704 len = ngx_min(qc->ctp.max_udp_payload_size,
4705 NGX_QUIC_MAX_UDP_PAYLOAD_SIZE);
4706
4707 if (!qc->validated) {
4708 max = qc->received * 3;
4709 max = (c->sent >= max) ? 0 : max - c->sent;
4710 len = ngx_min(len, (size_t) max);
4711 }
4712
4713 pad = ngx_quic_get_padding_level(c);
4714
4715 for (i = 0; i < NGX_QUIC_SEND_CTX_LAST; i++) {
4716
4717 ctx = &qc->send_ctx[i];
4718
4719 if (ngx_quic_generate_ack(c, ctx) != NGX_OK) {
4720 return NGX_ERROR;
4721 }
4722
4723 min = (i == pad && p - dst < NGX_QUIC_MIN_INITIAL_SIZE)
4724 ? NGX_QUIC_MIN_INITIAL_SIZE - (p - dst) : 0;
4725
4726 n = ngx_quic_output_packet(c, ctx, p, len, min);
4727 if (n == NGX_ERROR) {
4728 return NGX_ERROR;
4729 }
4730
4731 p += n;
4732 len -= n;
4733 }
4734
4735 len = p - dst;
4736 if (len == 0) {
4737 break;
4738 }
4739
4740 n = ngx_quic_send(c, dst, len);
4741 if (n == NGX_ERROR) {
4742 return NGX_ERROR;
4743 }
4744
4745 if (!qc->send_timer_set && !qc->closing) {
4746 qc->send_timer_set = 1;
4747 ngx_add_timer(c->read, qc->tp.max_idle_timeout);
4748 }
4749 }
4750
4751 return NGX_OK;
4752 }
4753
4754
4755 static ngx_uint_t
4756 ngx_quic_get_padding_level(ngx_connection_t *c)
4757 {
4758 ngx_queue_t *q;
4759 ngx_quic_frame_t *f;
4760 ngx_quic_send_ctx_t *ctx;
4761 ngx_quic_connection_t *qc;
4762
4763 /*
4764 * 14.1. Initial Datagram Size
4765 *
4766 * Similarly, a server MUST expand the payload of all UDP datagrams
4767 * carrying ack-eliciting Initial packets to at least the smallest
4768 * allowed maximum datagram size of 1200 bytes
4769 */
4770
4771 qc = ngx_quic_get_connection(c);
4772 ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_initial);
4773
4774 for (q = ngx_queue_head(&ctx->frames);
4775 q != ngx_queue_sentinel(&ctx->frames);
4776 q = ngx_queue_next(q))
4777 {
4778 f = ngx_queue_data(q, ngx_quic_frame_t, queue);
4779
4780 if (f->need_ack) {
4781 ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_handshake);
4782
4783 if (ngx_queue_empty(&ctx->frames)) {
4784 return 0;
4785 }
4786
4787 return 1;
4788 }
4789 }
4790
4791 return NGX_QUIC_SEND_CTX_LAST;
4792 }
4793
4794
4795 static ngx_int_t
4796 ngx_quic_generate_ack(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx)
4797 {
4798 ngx_msec_t delay;
4799 ngx_quic_connection_t *qc;
4800
4801 if (!ctx->send_ack) {
4802 return NGX_OK;
4803 }
4804
4805 if (ctx->level == ssl_encryption_application) {
4806
4807 delay = ngx_current_msec - ctx->ack_delay_start;
4808 qc = ngx_quic_get_connection(c);
4809
4810 if (ctx->send_ack < NGX_QUIC_MAX_ACK_GAP
4811 && delay < qc->tp.max_ack_delay)
4812 {
4813 if (!qc->push.timer_set && !qc->closing) {
4814 ngx_add_timer(&qc->push,
4815 qc->tp.max_ack_delay - delay);
4816 }
4817
4818 return NGX_OK;
4819 }
4820 }
4821
4822 if (ngx_quic_send_ack(c, ctx) != NGX_OK) {
4823 return NGX_ERROR;
4824 }
4825
4826 ctx->send_ack = 0;
4827
4828 return NGX_OK;
4829 }
4830
4831
4832 static ssize_t
4833 ngx_quic_output_packet(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx,
4834 u_char *data, size_t max, size_t min)
4835 {
4836 size_t len, hlen, pad_len;
4837 u_char *p;
4838 ssize_t flen;
4839 ngx_str_t out, res;
4840 ngx_int_t rc;
4841 ngx_uint_t nframes;
4842 ngx_msec_t now;
4843 ngx_queue_t *q;
4844 ngx_quic_frame_t *f;
4845 ngx_quic_header_t pkt;
4846 ngx_quic_congestion_t *cg;
4847 ngx_quic_connection_t *qc;
4848 static u_char src[NGX_QUIC_MAX_UDP_PAYLOAD_SIZE];
4849
4850 if (ngx_queue_empty(&ctx->frames)) {
4851 return 0;
4852 }
4853
4854 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
4855 "quic output %s packet max:%uz min:%uz",
4856 ngx_quic_level_name(ctx->level), max, min);
4857
4858 qc = ngx_quic_get_connection(c);
4859 cg = &qc->congestion;
4860
4861 hlen = (ctx->level == ssl_encryption_application)
4862 ? NGX_QUIC_MAX_SHORT_HEADER
4863 : NGX_QUIC_MAX_LONG_HEADER;
4864
4865 hlen += EVP_GCM_TLS_TAG_LEN;
4866 hlen -= NGX_QUIC_MAX_CID_LEN - qc->scid.len;
4867
4868 ngx_memzero(&pkt, sizeof(ngx_quic_header_t));
4869
4870 now = ngx_current_msec;
4871 nframes = 0;
4872 p = src;
4873 len = 0;
4874
4875 for (q = ngx_queue_head(&ctx->frames);
4876 q != ngx_queue_sentinel(&ctx->frames);
4877 q = ngx_queue_next(q))
4878 {
4879 f = ngx_queue_data(q, ngx_quic_frame_t, queue);
4880
4881 if (!pkt.need_ack && f->need_ack && max > cg->window) {
4882 max = cg->window;
4883 }
4884
4885 if (hlen + len >= max) {
4886 break;
4887 }
4888
4889 if (hlen + len + f->len > max) {
4890 rc = ngx_quic_split_frame(c, f, max - hlen - len);
4891
4892 if (rc == NGX_ERROR) {
4893 return NGX_ERROR;
4894 }
4895
4896 if (rc == NGX_DECLINED) {
4897 break;
4898 }
4899 }
4900
4901 if (f->need_ack) {
4902 pkt.need_ack = 1;
4903 }
4904
4905 ngx_quic_log_frame(c->log, f, 1);
4906
4907 flen = ngx_quic_create_frame(p, f);
4908 if (flen == -1) {
4909 return NGX_ERROR;
4910 }
4911
4912 len += flen;
4913 p += flen;
4914
4915 f->pnum = ctx->pnum;
4916 f->first = now;
4917 f->last = now;
4918 f->plen = 0;
4919
4920 nframes++;
4921 }
4922
4923 if (nframes == 0) {
4924 return 0;
4925 }
4926
4927 out.data = src;
4928 out.len = len;
4929
4930 pkt.keys = qc->keys;
4931 pkt.flags = NGX_QUIC_PKT_FIXED_BIT;
4932
4933 if (ctx->level == ssl_encryption_initial) {
4934 pkt.flags |= NGX_QUIC_PKT_LONG | NGX_QUIC_PKT_INITIAL;
4935
4936 } else if (ctx->level == ssl_encryption_handshake) {
4937 pkt.flags |= NGX_QUIC_PKT_LONG | NGX_QUIC_PKT_HANDSHAKE;
4938
4939 } else {
4940 if (qc->key_phase) {
4941 pkt.flags |= NGX_QUIC_PKT_KPHASE;
4942 }
4943 }
4944
4945 ngx_quic_set_packet_number(&pkt, ctx);
4946
4947 pkt.version = qc->version;
4948 pkt.log = c->log;
4949 pkt.level = ctx->level;
4950 pkt.dcid = qc->scid;
4951 pkt.scid = qc->dcid;
4952
4953 pad_len = 4;
4954
4955 if (min) {
4956 hlen = EVP_GCM_TLS_TAG_LEN
4957 + ngx_quic_create_header(&pkt, NULL, out.len, NULL);
4958
4959 if (min > hlen + pad_len) {
4960 pad_len = min - hlen;
4961 }
4962 }
4963
4964 if (out.len < pad_len) {
4965 ngx_memset(p, NGX_QUIC_FT_PADDING, pad_len - out.len);
4966 out.len = pad_len;
4967 }
4968
4969 pkt.payload = out;
4970
4971 res.data = data;
4972
4973 ngx_log_debug6(NGX_LOG_DEBUG_EVENT, c->log, 0,
4974 "quic packet tx %s bytes:%ui"
4975 " need_ack:%d number:%L encoded nl:%d trunc:0x%xD",
4976 ngx_quic_level_name(ctx->level), out.len, pkt.need_ack,
4977 pkt.number, pkt.num_len, pkt.trunc);
4978
4979 if (ngx_quic_encrypt(&pkt, &res) != NGX_OK) {
4980 return NGX_ERROR;
4981 }
4982
4983 ctx->pnum++;
4984
4985 if (pkt.need_ack) {
4986 /* move frames into the sent queue to wait for ack */
4987
4988 if (!qc->closing) {
4989 q = ngx_queue_head(&ctx->frames);
4990 f = ngx_queue_data(q, ngx_quic_frame_t, queue);
4991 f->plen = res.len;
4992
4993 do {
4994 q = ngx_queue_head(&ctx->frames);
4995 ngx_queue_remove(q);
4996 ngx_queue_insert_tail(&ctx->sent, q);
4997 } while (--nframes);
4998
4999 if (qc->pto.timer_set) {
5000 ngx_del_timer(&qc->pto);
5001 }
5002
5003 ngx_add_timer(&qc->pto, ngx_quic_pto(c, ctx));
5004 }
5005
5006 cg->in_flight += res.len;
5007
5008 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
5009 "quic congestion send if:%uz", cg->in_flight);
5010 }
5011
5012 while (nframes--) {
5013 q = ngx_queue_head(&ctx->frames);
5014 f = ngx_queue_data(q, ngx_quic_frame_t, queue);
5015
5016 ngx_queue_remove(q);
5017 ngx_quic_free_frame(c, f);
5018 }
5019
5020 return res.len;
5021 }
5022
5023
5024 static ngx_int_t
5025 ngx_quic_split_frame(ngx_connection_t *c, ngx_quic_frame_t *f, size_t len)
5026 {
5027 size_t shrink;
5028 ngx_quic_frame_t *nf;
5029 ngx_quic_ordered_frame_t *of, *onf;
5030
5031 switch (f->type) {
5032 case NGX_QUIC_FT_CRYPTO:
5033 case NGX_QUIC_FT_STREAM0:
5034 case NGX_QUIC_FT_STREAM1:
5035 case NGX_QUIC_FT_STREAM2:
5036 case NGX_QUIC_FT_STREAM3:
5037 case NGX_QUIC_FT_STREAM4:
5038 case NGX_QUIC_FT_STREAM5:
5039 case NGX_QUIC_FT_STREAM6:
5040 case NGX_QUIC_FT_STREAM7:
5041 break;
5042
5043 default:
5044 return NGX_DECLINED;
5045 }
5046
5047 if ((size_t) f->len <= len) {
5048 return NGX_OK;
5049 }
5050
5051 shrink = f->len - len;
5052
5053 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
5054 "quic split frame now:%uz need:%uz shrink:%uz",
5055 f->len, len, shrink);
5056
5057 of = &f->u.ord;
5058
5059 if (of->length <= shrink) {
5060 return NGX_DECLINED;
5061 }
5062
5063 of->length -= shrink;
5064 f->len = ngx_quic_create_frame(NULL, f);
5065
5066 if ((size_t) f->len > len) {
5067 ngx_log_error(NGX_LOG_ERR, c->log, 0, "could not split QUIC frame");
5068 return NGX_ERROR;
5069 }
5070
5071 nf = ngx_quic_alloc_frame(c);
5072 if (nf == NULL) {
5073 return NGX_ERROR;
5074 }
5075
5076 *nf = *f;
5077 onf = &nf->u.ord;
5078 onf->offset += of->length;
5079 onf->length = shrink;
5080 nf->len = ngx_quic_create_frame(NULL, nf);
5081
5082 nf->data = ngx_quic_split_bufs(c, f->data, of->length);
5083 if (nf->data == NGX_CHAIN_ERROR) {
5084 return NGX_ERROR;
5085 }
5086
5087 ngx_queue_insert_after(&f->queue, &nf->queue);
5088
5089 return NGX_OK;
5090 }
5091
5092
5093 static void
5094 ngx_quic_free_frames(ngx_connection_t *c, ngx_queue_t *frames)
5095 {
5096 ngx_queue_t *q;
5097 ngx_quic_frame_t *f;
5098
5099 do {
5100 q = ngx_queue_head(frames);
5101
5102 if (q == ngx_queue_sentinel(frames)) {
5103 break;
5104 }
5105
5106 ngx_queue_remove(q);
5107
5108 f = ngx_queue_data(q, ngx_quic_frame_t, queue);
5109
5110 ngx_quic_free_frame(c, f);
5111 } while (1);
5112 }
5113
5114
5115 static ssize_t
5116 ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len)
5117 {
5118 ngx_buf_t b;
5119 ngx_chain_t cl, *res;
5120
5121 ngx_memzero(&b, sizeof(ngx_buf_t));
5122
5123 b.pos = b.start = buf;
5124 b.last = b.end = buf + len;
5125 b.last_buf = 1;
5126 b.temporary = 1;
5127
5128 cl.buf = &b;
5129 cl.next= NULL;
5130
5131 res = c->send_chain(c, &cl, 0);
5132 if (res == NGX_CHAIN_ERROR) {
5133 return NGX_ERROR;
5134 }
5135
5136 return len;
5137 }
5138
5139
5140 static void
5141 ngx_quic_set_packet_number(ngx_quic_header_t *pkt, ngx_quic_send_ctx_t *ctx)
5142 {
5143 uint64_t delta;
5144
5145 delta = ctx->pnum - ctx->largest_ack;
5146 pkt->number = ctx->pnum;
5147
5148 if (delta <= 0x7F) {
5149 pkt->num_len = 1;
5150 pkt->trunc = ctx->pnum & 0xff;
5151
5152 } else if (delta <= 0x7FFF) {
5153 pkt->num_len = 2;
5154 pkt->flags |= 0x1;
5155 pkt->trunc = ctx->pnum & 0xffff;
5156
5157 } else if (delta <= 0x7FFFFF) {
5158 pkt->num_len = 3;
5159 pkt->flags |= 0x2;
5160 pkt->trunc = ctx->pnum & 0xffffff;
5161
5162 } else {
5163 pkt->num_len = 4;
5164 pkt->flags |= 0x3;
5165 pkt->trunc = ctx->pnum & 0xffffffff;
5166 }
5167 }
5168
5169
5170 static void
5171 ngx_quic_pto_handler(ngx_event_t *ev)
5172 {
5173 ngx_uint_t i;
5174 ngx_queue_t *q;
5175 ngx_connection_t *c;
5176 ngx_quic_frame_t *start;
5177 ngx_quic_send_ctx_t *ctx;
5178 ngx_quic_connection_t *qc;
5179
5180 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, "quic pto timer");
5181
5182 c = ev->data;
5183 qc = ngx_quic_get_connection(c);
5184
5185 qc->pto_count++;
5186
5187 for (i = 0; i < NGX_QUIC_SEND_CTX_LAST; i++) {
5188
5189 ctx = &qc->send_ctx[i];
5190
5191 if (ngx_queue_empty(&ctx->sent)) {
5192 continue;
5193 }
5194
5195 q = ngx_queue_head(&ctx->sent);
5196 start = ngx_queue_data(q, ngx_quic_frame_t, queue);
5197
5198 if (start->pnum <= ctx->largest_ack
5199 && ctx->largest_ack != NGX_QUIC_UNSET_PN)
5200 {
5201 continue;
5202 }
5203
5204 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
5205 "quic pto pnum:%uL pto_count:%ui level:%d",
5206 start->pnum, qc->pto_count, start->level);
5207
5208 ngx_quic_resend_frames(c, ctx);
5209 }
5210
5211 ngx_quic_connstate_dbg(c);
5212 }
5213
5214
5215 static void
5216 ngx_quic_push_handler(ngx_event_t *ev)
5217 {
5218 ngx_connection_t *c;
5219
5220 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, "quic push timer");
5221
5222 c = ev->data;
5223
5224 if (ngx_quic_output(c) != NGX_OK) {
5225 ngx_quic_close_connection(c, NGX_ERROR);
5226 return;
5227 }
5228
5229 ngx_quic_connstate_dbg(c);
5230 }
5231
5232
5233 static
5234 void ngx_quic_lost_handler(ngx_event_t *ev)
5235 {
5236 ngx_connection_t *c;
5237
5238 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, "quic lost timer");
5239
5240 c = ev->data;
5241
5242 if (ngx_quic_detect_lost(c) != NGX_OK) {
5243 ngx_quic_close_connection(c, NGX_ERROR);
5244 }
5245
5246 ngx_quic_connstate_dbg(c);
5247 }
5248
5249
5250 static ngx_int_t
5251 ngx_quic_detect_lost(ngx_connection_t *c)
5252 {
5253 ngx_uint_t i;
5254 ngx_msec_t now, wait, min_wait, thr;
5255 ngx_queue_t *q;
5256 ngx_quic_frame_t *start;
5257 ngx_quic_send_ctx_t *ctx;
5258 ngx_quic_connection_t *qc;
5259
5260 qc = ngx_quic_get_connection(c);
5261 now = ngx_current_msec;
5262
5263 min_wait = 0;
5264
5265 thr = NGX_QUIC_TIME_THR * ngx_max(qc->latest_rtt, qc->avg_rtt);
5266 thr = ngx_max(thr, NGX_QUIC_TIME_GRANULARITY);
5267
5268 for (i = 0; i < NGX_QUIC_SEND_CTX_LAST; i++) {
5269
5270 ctx = &qc->send_ctx[i];
5271
5272 if (ctx->largest_ack == NGX_QUIC_UNSET_PN) {
5273 continue;
5274 }
5275
5276 while (!ngx_queue_empty(&ctx->sent)) {
5277
5278 q = ngx_queue_head(&ctx->sent);
5279 start = ngx_queue_data(q, ngx_quic_frame_t, queue);
5280
5281 if (start->pnum > ctx->largest_ack) {
5282 break;
5283 }
5284
5285 wait = start->last + thr - now;
5286
5287 ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
5288 "quic detect_lost pnum:%uL thr:%M wait:%i level:%d",
5289 start->pnum, thr, (ngx_int_t) wait, start->level);
5290
5291 if ((ngx_msec_int_t) wait > 0
5292 && ctx->largest_ack - start->pnum < NGX_QUIC_PKT_THR)
5293 {
5294
5295 if (min_wait == 0 || wait < min_wait) {
5296 min_wait = wait;
5297 }
5298
5299 break;
5300 }
5301
5302 ngx_quic_resend_frames(c, ctx);
5303 }
5304 }
5305
5306 /* no more preceeding packets */
5307
5308 if (min_wait == 0) {
5309 qc->pto.handler = ngx_quic_pto_handler;
5310 return NGX_OK;
5311 }
5312
5313 qc->pto.handler = ngx_quic_lost_handler;
5314
5315 if (qc->pto.timer_set) {
5316 ngx_del_timer(&qc->pto);
5317 }
5318
5319 ngx_add_timer(&qc->pto, min_wait);
5320
5321 return NGX_OK;
5322 }
5323
5324
5325 static void
5326 ngx_quic_resend_frames(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx)
5327 {
5328 size_t n;
5329 ngx_buf_t *b;
5330 ngx_queue_t *q;
5331 ngx_quic_frame_t *f, *start;
5332 ngx_quic_stream_t *sn;
5333 ngx_quic_connection_t *qc;
5334
5335 qc = ngx_quic_get_connection(c);
5336 q = ngx_queue_head(&ctx->sent);
5337 start = ngx_queue_data(q, ngx_quic_frame_t, queue);
5338
5339 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
5340 "quic resend packet pnum:%uL", start->pnum);
5341
5342 ngx_quic_congestion_lost(c, start);
5343
5344 do {
5345 f = ngx_queue_data(q, ngx_quic_frame_t, queue);
5346
5347 if (f->pnum != start->pnum) {
5348 break;
5349 }
5350
5351 q = ngx_queue_next(q);
5352
5353 ngx_queue_remove(&f->queue);
5354
5355 switch (f->type) {
5356 case NGX_QUIC_FT_ACK:
5357 case NGX_QUIC_FT_ACK_ECN:
5358 /* force generation of most recent acknowledgment */
5359 ctx->send_ack = NGX_QUIC_MAX_ACK_GAP;
5360 ngx_quic_free_frame(c, f);
5361 break;
5362
5363 case NGX_QUIC_FT_PING:
5364 case NGX_QUIC_FT_PATH_RESPONSE:
5365 case NGX_QUIC_FT_CONNECTION_CLOSE:
5366 ngx_quic_free_frame(c, f);
5367 break;
5368
5369 case NGX_QUIC_FT_MAX_DATA:
5370 f->u.max_data.max_data = qc->streams.recv_max_data;
5371 ngx_quic_queue_frame(qc, f);
5372 break;
5373
5374 case NGX_QUIC_FT_MAX_STREAMS:
5375 case NGX_QUIC_FT_MAX_STREAMS2:
5376 f->u.max_streams.limit = f->u.max_streams.bidi
5377 ? qc->streams.client_max_streams_bidi
5378 : qc->streams.client_max_streams_uni;
5379 ngx_quic_queue_frame(qc, f);
5380 break;
5381
5382 case NGX_QUIC_FT_MAX_STREAM_DATA:
5383 sn = ngx_quic_find_stream(&qc->streams.tree,
5384 f->u.max_stream_data.id);
5385 if (sn == NULL) {
5386 ngx_quic_free_frame(c, f);
5387 break;
5388 }
5389
5390 b = sn->b;
5391 n = sn->fs.received + (b->pos - b->start) + (b->end - b->last);
5392
5393 if (f->u.max_stream_data.limit < n) {
5394 f->u.max_stream_data.limit = n;
5395 }
5396
5397 ngx_quic_queue_frame(qc, f);
5398 break;
5399
5400 case NGX_QUIC_FT_STREAM0:
5401 case NGX_QUIC_FT_STREAM1:
5402 case NGX_QUIC_FT_STREAM2:
5403 case NGX_QUIC_FT_STREAM3:
5404 case NGX_QUIC_FT_STREAM4:
5405 case NGX_QUIC_FT_STREAM5:
5406 case NGX_QUIC_FT_STREAM6:
5407 case NGX_QUIC_FT_STREAM7:
5408 sn = ngx_quic_find_stream(&qc->streams.tree, f->u.stream.stream_id);
5409
5410 if (sn && sn->c->write->error) {
5411 /* RESET_STREAM was sent */
5412 ngx_quic_free_frame(c, f);
5413 break;
5414 }
5415
5416 /* fall through */
5417
5418 default:
5419 ngx_queue_insert_tail(&ctx->frames, &f->queue);
5420 }
5421
5422 } while (q != ngx_queue_sentinel(&ctx->sent));
5423
5424 if (qc->closing) {
5425 return;
5426 }
5427
5428 ngx_post_event(&qc->push, &ngx_posted_events);
5429 }
5430
5431
5432 ngx_connection_t *
5433 ngx_quic_open_stream(ngx_connection_t *c, ngx_uint_t bidi)
5434 {
5435 size_t rcvbuf_size;
5436 uint64_t id;
5437 ngx_quic_stream_t *qs, *sn;
5438 ngx_quic_connection_t *qc;
5439
5440 qs = c->quic;
5441 qc = ngx_quic_get_connection(qs->parent);
5442
5443 if (bidi) {
5444 if (qc->streams.server_streams_bidi
5445 >= qc->streams.server_max_streams_bidi)
5446 {
5447 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
5448 "quic too many server bidi streams:%uL",
5449 qc->streams.server_streams_bidi);
5450 return NULL;
5451 }
5452
5453 id = (qc->streams.server_streams_bidi << 2)
5454 | NGX_QUIC_STREAM_SERVER_INITIATED;
5455
5456 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
5457 "quic creating server bidi stream"
5458 " streams:%uL max:%uL id:0x%xL",
5459 qc->streams.server_streams_bidi,
5460 qc->streams.server_max_streams_bidi, id);
5461
5462 qc->streams.server_streams_bidi++;
5463 rcvbuf_size = qc->tp.initial_max_stream_data_bidi_local;
5464
5465 } else {
5466 if (qc->streams.server_streams_uni
5467 >= qc->streams.server_max_streams_uni)
5468 {
5469 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
5470 "quic too many server uni streams:%uL",
5471 qc->streams.server_streams_uni);
5472 return NULL;
5473 }
5474
5475 id = (qc->streams.server_streams_uni << 2)
5476 | NGX_QUIC_STREAM_SERVER_INITIATED
5477 | NGX_QUIC_STREAM_UNIDIRECTIONAL;
5478
5479 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
5480 "quic creating server uni stream"
5481 " streams:%uL max:%uL id:0x%xL",
5482 qc->streams.server_streams_uni,
5483 qc->streams.server_max_streams_uni, id);
5484
5485 qc->streams.server_streams_uni++;
5486 rcvbuf_size = 0;
5487 }
5488
5489 sn = ngx_quic_create_stream(qs->parent, id, rcvbuf_size);
5490 if (sn == NULL) {
5491 return NULL;
5492 }
5493
5494 return sn->c;
5495 }
5496
5497
5498 static void
5499 ngx_quic_rbtree_insert_stream(ngx_rbtree_node_t *temp,
5500 ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel)
5501 {
5502 ngx_rbtree_node_t **p;
5503 ngx_quic_stream_t *qn, *qnt;
5504
5505 for ( ;; ) {
5506 qn = (ngx_quic_stream_t *) node;
5507 qnt = (ngx_quic_stream_t *) temp;
5508
5509 p = (qn->id < qnt->id) ? &temp->left : &temp->right;
5510
5511 if (*p == sentinel) {
5512 break;
5513 }
5514
5515 temp = *p;
5516 }
5517
5518 *p = node;
5519 node->parent = temp;
5520 node->left = sentinel;
5521 node->right = sentinel;
5522 ngx_rbt_red(node);
5523 }
5524
5525
5526 static ngx_quic_stream_t *
5527 ngx_quic_find_stream(ngx_rbtree_t *rbtree, uint64_t id)
5528 {
5529 ngx_rbtree_node_t *node, *sentinel;
5530 ngx_quic_stream_t *qn;
5531
5532 node = rbtree->root;
5533 sentinel = rbtree->sentinel;
5534
5535 while (node != sentinel) {
5536 qn = (ngx_quic_stream_t *) node;
5537
5538 if (id == qn->id) {
5539 return qn;
5540 }
5541
5542 node = (id < qn->id) ? node->left : node->right;
5543 }
5544
5545 return NULL;
5546 }
5547
5548
5549 static ngx_quic_stream_t *
5550 ngx_quic_create_client_stream(ngx_connection_t *c, uint64_t id)
5551 {
5552 size_t n;
5553 uint64_t min_id;
5554 ngx_quic_stream_t *sn;
5555 ngx_quic_connection_t *qc;
5556
5557 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
5558 "quic stream id:0x%xL is new", id);
5559
5560 qc = ngx_quic_get_connection(c);
5561
5562 if (id & NGX_QUIC_STREAM_UNIDIRECTIONAL) {
5563
5564 if (id & NGX_QUIC_STREAM_SERVER_INITIATED) {
5565 if ((id >> 2) < qc->streams.server_streams_uni) {
5566 return NGX_QUIC_STREAM_GONE;
5567 }
5568
5569 qc->error = NGX_QUIC_ERR_STREAM_STATE_ERROR;
5570 return NULL;
5571 }
5572
5573 if ((id >> 2) < qc->streams.client_streams_uni) {
5574 return NGX_QUIC_STREAM_GONE;
5575 }
5576
5577 if ((id >> 2) >= qc->streams.client_max_streams_uni) {
5578 qc->error = NGX_QUIC_ERR_STREAM_LIMIT_ERROR;
5579 return NULL;
5580 }
5581
5582 min_id = (qc->streams.client_streams_uni << 2)
5583 | NGX_QUIC_STREAM_UNIDIRECTIONAL;
5584 qc->streams.client_streams_uni = (id >> 2) + 1;
5585 n = qc->tp.initial_max_stream_data_uni;
5586
5587 } else {
5588
5589 if (id & NGX_QUIC_STREAM_SERVER_INITIATED) {
5590 if ((id >> 2) < qc->streams.server_streams_bidi) {
5591 return NGX_QUIC_STREAM_GONE;
5592 }
5593
5594 qc->error = NGX_QUIC_ERR_STREAM_STATE_ERROR;
5595 return NULL;
5596 }
5597
5598 if ((id >> 2) < qc->streams.client_streams_bidi) {
5599 return NGX_QUIC_STREAM_GONE;
5600 }
5601
5602 if ((id >> 2) >= qc->streams.client_max_streams_bidi) {
5603 qc->error = NGX_QUIC_ERR_STREAM_LIMIT_ERROR;
5604 return NULL;
5605 }
5606
5607 min_id = (qc->streams.client_streams_bidi << 2);
5608 qc->streams.client_streams_bidi = (id >> 2) + 1;
5609 n = qc->tp.initial_max_stream_data_bidi_remote;
5610 }
5611
5612 if (n < NGX_QUIC_STREAM_BUFSIZE) {
5613 n = NGX_QUIC_STREAM_BUFSIZE;
5614 }
5615
5616 /*
5617 * 2.1. Stream Types and Identifiers
5618 *
5619 * Within each type, streams are created with numerically increasing
5620 * stream IDs. A stream ID that is used out of order results in all
5621 * streams of that type with lower-numbered stream IDs also being
5622 * opened.
5623 */
5624
5625 for ( /* void */ ; min_id < id; min_id += 0x04) {
5626
5627 sn = ngx_quic_create_stream(c, min_id, n);
5628 if (sn == NULL) {
5629 return NULL;
5630 }
5631
5632 sn->c->listening->handler(sn->c);
5633 }
5634
5635 return ngx_quic_create_stream(c, id, n);
5636 }
5637
5638
5639 static ngx_quic_stream_t *
5640 ngx_quic_create_stream(ngx_connection_t *c, uint64_t id, size_t rcvbuf_size)
5641 {
5642 ngx_log_t *log;
5643 ngx_pool_t *pool;
5644 ngx_quic_stream_t *sn;
5645 ngx_pool_cleanup_t *cln;
5646 ngx_quic_connection_t *qc;
5647
5648 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
5649 "quic stream id:0x%xL create", id);
5650
5651 qc = ngx_quic_get_connection(c);
5652
5653 pool = ngx_create_pool(NGX_DEFAULT_POOL_SIZE, c->log);
5654 if (pool == NULL) {
5655 return NULL;
5656 }
5657
5658 sn = ngx_pcalloc(pool, sizeof(ngx_quic_stream_t));
5659 if (sn == NULL) {
5660 ngx_destroy_pool(pool);
5661 return NULL;
5662 }
5663
5664 sn->node.key = id;
5665 sn->parent = c;
5666 sn->id = id;
5667
5668 sn->b = ngx_create_temp_buf(pool, rcvbuf_size);
5669 if (sn->b == NULL) {
5670 ngx_destroy_pool(pool);
5671 return NULL;
5672 }
5673
5674 ngx_queue_init(&sn->fs.frames);
5675
5676 log = ngx_palloc(pool, sizeof(ngx_log_t));
5677 if (log == NULL) {
5678 ngx_destroy_pool(pool);
5679 return NULL;
5680 }
5681
5682 *log = *c->log;
5683 pool->log = log;
5684
5685 sn->c = ngx_get_connection(-1, log);
5686 if (sn->c == NULL) {
5687 ngx_destroy_pool(pool);
5688 return NULL;
5689 }
5690
5691 sn->c->quic = sn;
5692 sn->c->type = SOCK_STREAM;
5693 sn->c->pool = pool;
5694 sn->c->ssl = c->ssl;
5695 sn->c->sockaddr = c->sockaddr;
5696 sn->c->listening = c->listening;
5697 sn->c->addr_text = c->addr_text;
5698 sn->c->local_sockaddr = c->local_sockaddr;
5699 sn->c->local_socklen = c->local_socklen;
5700 sn->c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1);
5701
5702 sn->c->recv = ngx_quic_stream_recv;
5703 sn->c->send = ngx_quic_stream_send;
5704 sn->c->send_chain = ngx_quic_stream_send_chain;
5705
5706 sn->c->read->log = log;
5707 sn->c->write->log = log;
5708
5709 log->connection = sn->c->number;
5710
5711 if ((id & NGX_QUIC_STREAM_UNIDIRECTIONAL) == 0
5712 || (id & NGX_QUIC_STREAM_SERVER_INITIATED))
5713 {
5714 sn->c->write->ready = 1;
5715 }
5716
5717 if (id & NGX_QUIC_STREAM_UNIDIRECTIONAL) {
5718 if (id & NGX_QUIC_STREAM_SERVER_INITIATED) {
5719 sn->send_max_data = qc->ctp.initial_max_stream_data_uni;
5720 }
5721
5722 } else {
5723 if (id & NGX_QUIC_STREAM_SERVER_INITIATED) {
5724 sn->send_max_data = qc->ctp.initial_max_stream_data_bidi_remote;
5725 } else {
5726 sn->send_max_data = qc->ctp.initial_max_stream_data_bidi_local;
5727 }
5728 }
5729
5730 cln = ngx_pool_cleanup_add(pool, 0);
5731 if (cln == NULL) {
5732 ngx_close_connection(sn->c);
5733 ngx_destroy_pool(pool);
5734 return NULL;
5735 }
5736
5737 cln->handler = ngx_quic_stream_cleanup_handler;
5738 cln->data = sn->c;
5739
5740 ngx_rbtree_insert(&qc->streams.tree, &sn->node);
5741
5742 return sn;
5743 }
5744
5745
5746 static ssize_t
5747 ngx_quic_stream_recv(ngx_connection_t *c, u_char *buf, size_t size)
5748 {
5749 ssize_t len;
5750 ngx_buf_t *b;
5751 ngx_event_t *rev;
5752 ngx_connection_t *pc;
5753 ngx_quic_frame_t *frame;
5754 ngx_quic_stream_t *qs;
5755 ngx_quic_connection_t *qc;
5756
5757 qs = c->quic;
5758 b = qs->b;
5759 pc = qs->parent;
5760 qc = ngx_quic_get_connection(pc);
5761 rev = c->read;
5762
5763 if (rev->error) {
5764 return NGX_ERROR;
5765 }
5766
5767 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
5768 "quic stream recv id:0x%xL eof:%d avail:%z",
5769 qs->id, rev->pending_eof, b->last - b->pos);
5770
5771 if (b->pos == b->last) {
5772 rev->ready = 0;
5773
5774 if (rev->pending_eof) {
5775 rev->eof = 1;
5776 return 0;
5777 }
5778
5779 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
5780 "quic stream id:0x%xL recv() not ready", qs->id);
5781 return NGX_AGAIN;
5782 }
5783
5784 len = ngx_min(b->last - b->pos, (ssize_t) size);
5785
5786 ngx_memcpy(buf, b->pos, len);
5787
5788 b->pos += len;
5789 qc->streams.received += len;
5790
5791 if (b->pos == b->last) {
5792 b->pos = b->start;
5793 b->last = b->start;
5794 rev->ready = rev->pending_eof;
5795 }
5796
5797 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
5798 "quic stream id:0x%xL recv len:%z of size:%uz",
5799 qs->id, len, size);
5800
5801 if (!rev->pending_eof) {
5802 frame = ngx_quic_alloc_frame(pc);
5803 if (frame == NULL) {
5804 return NGX_ERROR;
5805 }
5806
5807 frame->level = ssl_encryption_application;
5808 frame->type = NGX_QUIC_FT_MAX_STREAM_DATA;
5809 frame->u.max_stream_data.id = qs->id;
5810 frame->u.max_stream_data.limit = qs->fs.received + (b->pos - b->start)
5811 + (b->end - b->last);
5812
5813 ngx_quic_queue_frame(qc, frame);
5814 }
5815
5816 if ((qc->streams.recv_max_data / 2) < qc->streams.received) {
5817
5818 frame = ngx_quic_alloc_frame(pc);
5819
5820 if (frame == NULL) {
5821 return NGX_ERROR;
5822 }
5823
5824 qc->streams.recv_max_data *= 2;
5825
5826 frame->level = ssl_encryption_application;
5827 frame->type = NGX_QUIC_FT_MAX_DATA;
5828 frame->u.max_data.max_data = qc->streams.recv_max_data;
5829
5830 ngx_quic_queue_frame(qc, frame);
5831
5832 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
5833 "quic stream id:0x%xL recv: increased max_data:%uL",
5834 qs->id, qc->streams.recv_max_data);
5835 }
5836
5837 return len;
5838 }
5839
5840
5841 static ssize_t
5842 ngx_quic_stream_send(ngx_connection_t *c, u_char *buf, size_t size)
5843 {
5844 ngx_buf_t b;
5845 ngx_chain_t cl;
5846
5847 ngx_memzero(&b, sizeof(ngx_buf_t));
5848
5849 b.memory = 1;
5850 b.pos = buf;
5851 b.last = buf + size;
5852
5853 cl.buf = &b;
5854 cl.next = NULL;
5855
5856 if (ngx_quic_stream_send_chain(c, &cl, 0) == NGX_CHAIN_ERROR) {
5857 return NGX_ERROR;
5858 }
5859
5860 if (b.pos == buf) {
5861 return NGX_AGAIN;
5862 }
5863
5864 return b.pos - buf;
5865 }
5866
5867
5868 static ngx_chain_t *
5869 ngx_quic_stream_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
5870 {
5871 size_t n, flow;
5872 ngx_event_t *wev;
5873 ngx_chain_t *cl;
5874 ngx_connection_t *pc;
5875 ngx_quic_frame_t *frame;
5876 ngx_quic_stream_t *qs;
5877 ngx_quic_connection_t *qc;
5878
5879 qs = c->quic;
5880 pc = qs->parent;
5881 qc = ngx_quic_get_connection(pc);
5882 wev = c->write;
5883
5884 if (wev->error) {
5885 return NGX_CHAIN_ERROR;
5886 }
5887
5888 flow = ngx_quic_max_stream_flow(c);
5889 if (flow == 0) {
5890 wev->ready = 0;
5891 return in;
5892 }
5893
5894 n = (limit && (size_t) limit < flow) ? (size_t) limit : flow;
5895
5896 frame = ngx_quic_alloc_frame(pc);
5897 if (frame == NULL) {
5898 return NGX_CHAIN_ERROR;
5899 }
5900
5901 frame->data = ngx_quic_copy_chain(pc, in, n);
5902 if (frame->data == NGX_CHAIN_ERROR) {
5903 return NGX_CHAIN_ERROR;
5904 }
5905
5906 for (n = 0, cl = frame->data; cl; cl = cl->next) {
5907 n += ngx_buf_size(cl->buf);
5908 }
5909
5910 while (in && ngx_buf_size(in->buf) == 0) {
5911 in = in->next;
5912 }
5913
5914 frame->level = ssl_encryption_application;
5915 frame->type = NGX_QUIC_FT_STREAM6; /* OFF=1 LEN=1 FIN=0 */
5916 frame->u.stream.off = 1;
5917 frame->u.stream.len = 1;
5918 frame->u.stream.fin = 0;
5919
5920 frame->u.stream.type = frame->type;
5921 frame->u.stream.stream_id = qs->id;
5922 frame->u.stream.offset = c->sent;
5923 frame->u.stream.length = n;
5924
5925 c->sent += n;
5926 qc->streams.sent += n;
5927
5928 ngx_quic_queue_frame(qc, frame);
5929
5930 wev->ready = (n < flow) ? 1 : 0;
5931
5932 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
5933 "quic send_chain sent:%uz", n);
5934
5935 return in;
5936 }
5937
5938
5939 static size_t
5940 ngx_quic_max_stream_flow(ngx_connection_t *c)
5941 {
5942 size_t size;
5943 uint64_t sent, unacked;
5944 ngx_quic_stream_t *qs;
5945 ngx_quic_connection_t *qc;
5946
5947 qs = c->quic;
5948 qc = ngx_quic_get_connection(qs->parent);
5949
5950 size = NGX_QUIC_STREAM_BUFSIZE;
5951 sent = c->sent;
5952 unacked = sent - qs->acked;
5953
5954 if (qc->streams.send_max_data == 0) {
5955 qc->streams.send_max_data = qc->ctp.initial_max_data;
5956 }
5957
5958 if (unacked >= NGX_QUIC_STREAM_BUFSIZE) {
5959 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
5960 "quic send flow hit buffer size");
5961 return 0;
5962 }
5963
5964 if (unacked + size > NGX_QUIC_STREAM_BUFSIZE) {
5965 size = NGX_QUIC_STREAM_BUFSIZE - unacked;
5966 }
5967
5968 if (qc->streams.sent >= qc->streams.send_max_data) {
5969 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
5970 "quic send flow hit MAX_DATA");
5971 return 0;
5972 }
5973
5974 if (qc->streams.sent + size > qc->streams.send_max_data) {
5975 size = qc->streams.send_max_data - qc->streams.sent;
5976 }
5977
5978 if (sent >= qs->send_max_data) {
5979 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
5980 "quic send flow hit MAX_STREAM_DATA");
5981 return 0;
5982 }
5983
5984 if (sent + size > qs->send_max_data) {
5985 size = qs->send_max_data - sent;
5986 }
5987
5988 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
5989 "quic send flow:%uz", size);
5990
5991 return size;
5992 }
5993
5994
5995 static void
5996 ngx_quic_stream_cleanup_handler(void *data)
5997 {
5998 ngx_connection_t *c = data;
5999
6000 ngx_connection_t *pc;
6001 ngx_quic_frame_t *frame;
6002 ngx_quic_stream_t *qs;
6003 ngx_quic_connection_t *qc;
6004
6005 qs = c->quic;
6006 pc = qs->parent;
6007 qc = ngx_quic_get_connection(pc);
6008
6009 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
6010 "quic stream id:0x%xL cleanup", qs->id);
6011
6012 ngx_rbtree_delete(&qc->streams.tree, &qs->node);
6013 ngx_quic_free_frames(pc, &qs->fs.frames);
6014
6015 if (qc->closing) {
6016 /* schedule handler call to continue ngx_quic_close_connection() */
6017 ngx_post_event(pc->read, &ngx_posted_events);
6018 return;
6019 }
6020
6021 if ((qs->id & NGX_QUIC_STREAM_SERVER_INITIATED) == 0
6022 || (qs->id & NGX_QUIC_STREAM_UNIDIRECTIONAL) == 0)
6023 {
6024 if (!c->read->pending_eof && !c->read->error) {
6025 frame = ngx_quic_alloc_frame(pc);
6026 if (frame == NULL) {
6027 return;
6028 }
6029
6030 frame->level = ssl_encryption_application;
6031 frame->type = NGX_QUIC_FT_STOP_SENDING;
6032 frame->u.stop_sending.id = qs->id;
6033 frame->u.stop_sending.error_code = 0x100; /* HTTP/3 no error */
6034
6035 ngx_quic_queue_frame(qc, frame);
6036 }
6037 }
6038
6039 if ((qs->id & NGX_QUIC_STREAM_SERVER_INITIATED) == 0) {
6040 frame = ngx_quic_alloc_frame(pc);
6041 if (frame == NULL) {
6042 return;
6043 }
6044
6045 frame->level = ssl_encryption_application;
6046 frame->type = NGX_QUIC_FT_MAX_STREAMS;
6047
6048 if (qs->id & NGX_QUIC_STREAM_UNIDIRECTIONAL) {
6049 frame->u.max_streams.limit = ++qc->streams.client_max_streams_uni;
6050 frame->u.max_streams.bidi = 0;
6051
6052 } else {
6053 frame->u.max_streams.limit = ++qc->streams.client_max_streams_bidi;
6054 frame->u.max_streams.bidi = 1;
6055 }
6056
6057 ngx_quic_queue_frame(qc, frame);
6058
6059 if (qs->id & NGX_QUIC_STREAM_UNIDIRECTIONAL) {
6060 /* do not send fin for client unidirectional streams */
6061 return;
6062 }
6063 }
6064
6065 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
6066 "quic stream id:0x%xL send fin", qs->id);
6067
6068 frame = ngx_quic_alloc_frame(pc);
6069 if (frame == NULL) {
6070 return;
6071 }
6072
6073 frame->level = ssl_encryption_application;
6074 frame->type = NGX_QUIC_FT_STREAM7; /* OFF=1 LEN=1 FIN=1 */
6075 frame->u.stream.off = 1;
6076 frame->u.stream.len = 1;
6077 frame->u.stream.fin = 1;
6078
6079 frame->u.stream.type = frame->type;
6080 frame->u.stream.stream_id = qs->id;
6081 frame->u.stream.offset = c->sent;
6082 frame->u.stream.length = 0;
6083
6084 ngx_quic_queue_frame(qc, frame);
6085
6086 (void) ngx_quic_output(pc);
6087 }
6088
6089
6090 static ngx_quic_frame_t *
6091 ngx_quic_alloc_frame(ngx_connection_t *c)
6092 {
6093 ngx_queue_t *q;
6094 ngx_quic_frame_t *frame;
6095 ngx_quic_connection_t *qc;
6096
6097 qc = ngx_quic_get_connection(c);
6098
6099 if (!ngx_queue_empty(&qc->free_frames)) {
6100
6101 q = ngx_queue_head(&qc->free_frames);
6102 frame = ngx_queue_data(q, ngx_quic_frame_t, queue);
6103
6104 ngx_queue_remove(&frame->queue);
6105
6106 #ifdef NGX_QUIC_DEBUG_ALLOC
6107 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
6108 "quic reuse frame n:%ui", qc->nframes);
6109 #endif
6110
6111 } else {
6112 frame = ngx_pcalloc(c->pool, sizeof(ngx_quic_frame_t));
6113 if (frame == NULL) {
6114 return NULL;
6115 }
6116
6117 #ifdef NGX_QUIC_DEBUG_ALLOC
6118 ++qc->nframes;
6119
6120 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
6121 "quic alloc frame n:%ui", qc->nframes);
6122 #endif
6123 }
6124
6125 ngx_memzero(frame, sizeof(ngx_quic_frame_t));
6126
6127 return frame;
6128 }
6129
6130
6131 static void
6132 ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f)
6133 {
6134 ngx_msec_t timer;
6135 ngx_quic_congestion_t *cg;
6136 ngx_quic_connection_t *qc;
6137
6138 if (f->plen == 0) {
6139 return;
6140 }
6141
6142 qc = ngx_quic_get_connection(c);
6143 cg = &qc->congestion;
6144
6145 cg->in_flight -= f->plen;
6146
6147 timer = f->last - cg->recovery_start;
6148
6149 if ((ngx_msec_int_t) timer <= 0) {
6150 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
6151 "quic congestion ack recovery win:%uz ss:%z if:%uz",
6152 cg->window, cg->ssthresh, cg->in_flight);
6153
6154 return;
6155 }
6156
6157 if (cg->window < cg->ssthresh) {
6158 cg->window += f->plen;
6159
6160 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
6161 "quic congestion slow start win:%uz ss:%z if:%uz",
6162 cg->window, cg->ssthresh, cg->in_flight);
6163
6164 } else {
6165 cg->window += qc->tp.max_udp_payload_size * f->plen / cg->window;
6166
6167 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
6168 "quic congestion avoidance win:%uz ss:%z if:%uz",
6169 cg->window, cg->ssthresh, cg->in_flight);
6170 }
6171
6172 /* prevent recovery_start from wrapping */
6173
6174 timer = cg->recovery_start - ngx_current_msec + qc->tp.max_idle_timeout * 2;
6175
6176 if ((ngx_msec_int_t) timer < 0) {
6177 cg->recovery_start = ngx_current_msec - qc->tp.max_idle_timeout * 2;
6178 }
6179 }
6180
6181
6182 static void
6183 ngx_quic_congestion_lost(ngx_connection_t *c, ngx_quic_frame_t *f)
6184 {
6185 ngx_msec_t timer;
6186 ngx_quic_congestion_t *cg;
6187 ngx_quic_connection_t *qc;
6188
6189 if (f->plen == 0) {
6190 return;
6191 }
6192
6193 qc = ngx_quic_get_connection(c);
6194 cg = &qc->congestion;
6195
6196 cg->in_flight -= f->plen;
6197 f->plen = 0;
6198
6199 timer = f->last - cg->recovery_start;
6200
6201 if ((ngx_msec_int_t) timer <= 0) {
6202 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
6203 "quic congestion lost recovery win:%uz ss:%z if:%uz",
6204 cg->window, cg->ssthresh, cg->in_flight);
6205
6206 return;
6207 }
6208
6209 cg->recovery_start = ngx_current_msec;
6210 cg->window /= 2;
6211
6212 if (cg->window < qc->tp.max_udp_payload_size * 2) {
6213 cg->window = qc->tp.max_udp_payload_size * 2;
6214 }
6215
6216 cg->ssthresh = cg->window;
6217
6218 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
6219 "quic congestion lost win:%uz ss:%z if:%uz",
6220 cg->window, cg->ssthresh, cg->in_flight);
6221 }
6222
6223
6224 static void
6225 ngx_quic_free_frame(ngx_connection_t *c, ngx_quic_frame_t *frame)
6226 {
6227 ngx_quic_connection_t *qc;
6228
6229 qc = ngx_quic_get_connection(c);
6230
6231 if (frame->data) {
6232 ngx_quic_free_bufs(c, frame->data);
6233 }
6234
6235 ngx_queue_insert_head(&qc->free_frames, &frame->queue);
6236
6237 #ifdef NGX_QUIC_DEBUG_ALLOC
6238 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
6239 "quic free frame n:%ui", qc->nframes);
6240 #endif
6241 }
6242
6243
6244 uint32_t
6245 ngx_quic_version(ngx_connection_t *c)
6246 {
6247 uint32_t version;
6248 ngx_quic_connection_t *qc;
6249
6250 qc = ngx_quic_get_connection(c);
6251
6252 version = qc->version;
6253
6254 return (version & 0xff000000) == 0xff000000 ? version & 0xff : version;
6255 }
6256
6257
6258 static ngx_chain_t *
6259 ngx_quic_alloc_buf(ngx_connection_t *c)
6260 {
6261 ngx_buf_t *b;
6262 ngx_chain_t *cl;
6263 ngx_quic_connection_t *qc;
6264
6265 qc = ngx_quic_get_connection(c);
6266
6267 if (qc->free_bufs) {
6268 cl = qc->free_bufs;
6269 qc->free_bufs = cl->next;
6270
6271 b = cl->buf;
6272 b->pos = b->start;
6273 b->last = b->start;
6274
6275 #ifdef NGX_QUIC_DEBUG_ALLOC
6276 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
6277 "quic reuse buffer n:%ui", qc->nbufs);
6278 #endif
6279
6280 return cl;
6281 }
6282
6283 cl = ngx_alloc_chain_link(c->pool);
6284 if (cl == NULL) {
6285 return NULL;
6286 }
6287
6288 b = ngx_create_temp_buf(c->pool, NGX_QUIC_BUFFER_SIZE);
6289 if (b == NULL) {
6290 return NULL;
6291 }
6292
6293 b->tag = (ngx_buf_tag_t) &ngx_quic_alloc_buf;
6294
6295 cl->buf = b;
6296
6297 #ifdef NGX_QUIC_DEBUG_ALLOC
6298 ++qc->nbufs;
6299
6300 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
6301 "quic alloc buffer n:%ui", qc->nbufs);
6302 #endif
6303
6304 return cl;
6305 }
6306
6307
6308 static void
6309 ngx_quic_free_bufs(ngx_connection_t *c, ngx_chain_t *in)
6310 {
6311 ngx_buf_t *b, *shadow;
6312 ngx_chain_t *cl;
6313 ngx_quic_connection_t *qc;
6314
6315 qc = ngx_quic_get_connection(c);
6316
6317 while (in) {
6318 #ifdef NGX_QUIC_DEBUG_ALLOC
6319 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
6320 "quic free buffer n:%ui", qc->nbufs);
6321 #endif
6322
6323 cl = in;
6324 in = in->next;
6325 b = cl->buf;
6326
6327 if (b->shadow) {
6328 if (!b->last_shadow) {
6329 b->recycled = 1;
6330 ngx_free_chain(c->pool, cl);
6331 continue;
6332 }
6333
6334 do {
6335 shadow = b->shadow;
6336 b->shadow = qc->free_shadow_bufs;
6337 qc->free_shadow_bufs = b;
6338 b = shadow;
6339 } while (b->recycled);
6340
6341 if (b->shadow) {
6342 b->last_shadow = 1;
6343 ngx_free_chain(c->pool, cl);
6344 continue;
6345 }
6346
6347 cl->buf = b;
6348 }
6349
6350 cl->next = qc->free_bufs;
6351 qc->free_bufs = cl;
6352 }
6353 }
6354
6355
6356 static ngx_chain_t *
6357 ngx_quic_copy_buf(ngx_connection_t *c, u_char *data, size_t len)
6358 {
6359 size_t n;
6360 ngx_buf_t *b;
6361 ngx_chain_t *cl, *out, **ll;
6362
6363 out = NULL;
6364 ll = &out;
6365
6366 while (len) {
6367 cl = ngx_quic_alloc_buf(c);
6368 if (cl == NULL) {
6369 return NGX_CHAIN_ERROR;
6370 }
6371
6372 b = cl->buf;
6373 n = ngx_min((size_t) (b->end - b->last), len);
6374
6375 b->last = ngx_cpymem(b->last, data, n);
6376
6377 data += n;
6378 len -= n;
6379
6380 *ll = cl;
6381 ll = &cl->next;
6382 }
6383
6384 *ll = NULL;
6385
6386 return out;
6387 }
6388
6389
6390 static ngx_chain_t *
6391 ngx_quic_copy_chain(ngx_connection_t *c, ngx_chain_t *in, size_t limit)
6392 {
6393 size_t n;
6394 ngx_buf_t *b;
6395 ngx_chain_t *cl, *out, **ll;
6396
6397 out = NULL;
6398 ll = &out;
6399
6400 while (in) {
6401 if (!ngx_buf_in_memory(in->buf) || ngx_buf_size(in->buf) == 0) {
6402 in = in->next;
6403 continue;
6404 }
6405
6406 cl = ngx_quic_alloc_buf(c);
6407 if (cl == NULL) {
6408 return NGX_CHAIN_ERROR;
6409 }
6410
6411 *ll = cl;
6412 ll = &cl->next;
6413
6414 b = cl->buf;
6415
6416 while (in && b->last != b->end) {
6417
6418 n = ngx_min(in->buf->last - in->buf->pos, b->end - b->last);
6419
6420 if (limit > 0 && n > limit) {
6421 n = limit;
6422 }
6423
6424 b->last = ngx_cpymem(b->last, in->buf->pos, n);
6425
6426 in->buf->pos += n;
6427 if (in->buf->pos == in->buf->last) {
6428 in = in->next;
6429 }
6430
6431 if (limit > 0) {
6432 if (limit == n) {
6433 goto done;
6434 }
6435
6436 limit -= n;
6437 }
6438 }
6439
6440 }
6441
6442 done:
6443
6444 *ll = NULL;
6445
6446 return out;
6447 }
6448
6449
6450 static ngx_chain_t *
6451 ngx_quic_split_bufs(ngx_connection_t *c, ngx_chain_t *in, size_t len)
6452 {
6453 size_t n;
6454 ngx_buf_t *b;
6455 ngx_chain_t *out;
6456 ngx_quic_connection_t *qc;
6457
6458 qc = ngx_quic_get_connection(c);
6459
6460 while (in) {
6461 n = ngx_buf_size(in->buf);
6462
6463 if (n == len) {
6464 out = in->next;
6465 in->next = NULL;
6466 return out;
6467 }
6468
6469 if (n > len) {
6470 break;
6471 }
6472
6473 len -= n;
6474 in = in->next;
6475 }
6476
6477 if (in == NULL) {
6478 return NULL;
6479 }
6480
6481 /* split in->buf by creating shadow bufs which reference it */
6482
6483 if (in->buf->shadow == NULL) {
6484 if (qc->free_shadow_bufs) {
6485 b = qc->free_shadow_bufs;
6486 qc->free_shadow_bufs = b->shadow;
6487
6488 } else {
6489 b = ngx_alloc_buf(c->pool);
6490 if (b == NULL) {
6491 return NGX_CHAIN_ERROR;
6492 }
6493 }
6494
6495 *b = *in->buf;
6496 b->shadow = in->buf;
6497 b->last_shadow = 1;
6498 in->buf = b;
6499 }
6500
6501 out = ngx_alloc_chain_link(c->pool);
6502 if (out == NULL) {
6503 return NGX_CHAIN_ERROR;
6504 }
6505
6506 if (qc->free_shadow_bufs) {
6507 b = qc->free_shadow_bufs;
6508 qc->free_shadow_bufs = b->shadow;
6509
6510 } else {
6511 b = ngx_alloc_buf(c->pool);
6512 if (b == NULL) {
6513 ngx_free_chain(c->pool, out);
6514 return NGX_CHAIN_ERROR;
6515 }
6516 }
6517
6518 out->buf = b;
6519 out->next = in->next;
6520 in->next = NULL;
6521
6522 *b = *in->buf;
6523 b->last_shadow = 0;
6524 b->pos = b->pos + len;
6525
6526 in->buf->shadow = b;
6527 in->buf->last = in->buf->pos + len;
6528
6529 return out;
6530 }