Mercurial > hg > nginx

comparison src/event/quic/ngx_event_quic.c @ 8749:660c4a2f95f3 quic

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
QUIC: separate files for frames related processing.
author Vladimir Homutov <vl@nginx.com>
date Tue, 13 Apr 2021 14:38:46 +0300
parents e0cb1e58ca13
children 41807e581de9
comparison
equal deleted inserted replaced
8748:e0cb1e58ca13 8749:660c4a2f95f3
32 32
33 #define NGX_QUIC_MIN_SR_PACKET 43 /* 5 random + 16 srt + 22 padding */ 33 #define NGX_QUIC_MIN_SR_PACKET 43 /* 5 random + 16 srt + 22 padding */
34 #define NGX_QUIC_MAX_SR_PACKET 1200 34 #define NGX_QUIC_MAX_SR_PACKET 1200
35 35
36 #define NGX_QUIC_MAX_ACK_GAP 2 36 #define NGX_QUIC_MAX_ACK_GAP 2
37
38
39 typedef ngx_int_t (*ngx_quic_frame_handler_pt)(ngx_connection_t *c,
40 ngx_quic_frame_t *frame, void *data);
41 37
42 38
43 #if BORINGSSL_API_VERSION >= 10 39 #if BORINGSSL_API_VERSION >= 10
44 static int ngx_quic_set_read_secret(ngx_ssl_conn_t *ssl_conn, 40 static int ngx_quic_set_read_secret(ngx_ssl_conn_t *ssl_conn,
45 enum ssl_encryption_level_t level, const SSL_CIPHER *cipher, 41 enum ssl_encryption_level_t level, const SSL_CIPHER *cipher,
120 static void ngx_quic_rtt_sample(ngx_connection_t *c, ngx_quic_ack_frame_t *ack, 116 static void ngx_quic_rtt_sample(ngx_connection_t *c, ngx_quic_ack_frame_t *ack,
121 enum ssl_encryption_level_t level, ngx_msec_t send_time); 117 enum ssl_encryption_level_t level, ngx_msec_t send_time);
122 static void ngx_quic_handle_stream_ack(ngx_connection_t *c, 118 static void ngx_quic_handle_stream_ack(ngx_connection_t *c,
123 ngx_quic_frame_t *f); 119 ngx_quic_frame_t *f);
124 120
125 static ngx_int_t ngx_quic_handle_ordered_frame(ngx_connection_t *c,
126 ngx_quic_frames_stream_t *fs, ngx_quic_frame_t *frame,
127 ngx_quic_frame_handler_pt handler, void *data);
128 static ngx_int_t ngx_quic_adjust_frame_offset(ngx_connection_t *c,
129 ngx_quic_frame_t *f, uint64_t offset_in);
130 static ngx_int_t ngx_quic_buffer_frame(ngx_connection_t *c,
131 ngx_quic_frames_stream_t *stream, ngx_quic_frame_t *f);
132
133 static ngx_int_t ngx_quic_handle_crypto_frame(ngx_connection_t *c, 121 static ngx_int_t ngx_quic_handle_crypto_frame(ngx_connection_t *c,
134 ngx_quic_header_t *pkt, ngx_quic_frame_t *frame); 122 ngx_quic_header_t *pkt, ngx_quic_frame_t *frame);
135 static ngx_int_t ngx_quic_crypto_input(ngx_connection_t *c, 123 ngx_int_t ngx_quic_crypto_input(ngx_connection_t *c,
136 ngx_quic_frame_t *frame, void *data); 124 ngx_quic_frame_t *frame, void *data);
137 static ngx_int_t ngx_quic_handle_stream_frame(ngx_connection_t *c, 125 static ngx_int_t ngx_quic_handle_stream_frame(ngx_connection_t *c,
138 ngx_quic_header_t *pkt, ngx_quic_frame_t *frame); 126 ngx_quic_header_t *pkt, ngx_quic_frame_t *frame);
139 static ngx_int_t ngx_quic_stream_input(ngx_connection_t *c, 127 static ngx_int_t ngx_quic_stream_input(ngx_connection_t *c,
140 ngx_quic_frame_t *frame, void *data); 128 ngx_quic_frame_t *frame, void *data);
158 static ngx_uint_t ngx_quic_get_padding_level(ngx_connection_t *c); 146 static ngx_uint_t ngx_quic_get_padding_level(ngx_connection_t *c);
159 static ngx_int_t ngx_quic_generate_ack(ngx_connection_t *c, 147 static ngx_int_t ngx_quic_generate_ack(ngx_connection_t *c,
160 ngx_quic_send_ctx_t *ctx); 148 ngx_quic_send_ctx_t *ctx);
161 static ssize_t ngx_quic_output_packet(ngx_connection_t *c, 149 static ssize_t ngx_quic_output_packet(ngx_connection_t *c,
162 ngx_quic_send_ctx_t *ctx, u_char *data, size_t max, size_t min); 150 ngx_quic_send_ctx_t *ctx, u_char *data, size_t max, size_t min);
163 static ngx_int_t ngx_quic_split_frame(ngx_connection_t *c, ngx_quic_frame_t *f,
164 size_t len);
165 static void ngx_quic_free_frames(ngx_connection_t *c, ngx_queue_t *frames);
166 static ssize_t ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len); 151 static ssize_t ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len);
167 152
168 static void ngx_quic_set_packet_number(ngx_quic_header_t *pkt, 153 static void ngx_quic_set_packet_number(ngx_quic_header_t *pkt,
169 ngx_quic_send_ctx_t *ctx); 154 ngx_quic_send_ctx_t *ctx);
170 static void ngx_quic_pto_handler(ngx_event_t *ev); 155 static void ngx_quic_pto_handler(ngx_event_t *ev);
190 static ngx_chain_t *ngx_quic_stream_send_chain(ngx_connection_t *c, 175 static ngx_chain_t *ngx_quic_stream_send_chain(ngx_connection_t *c,
191 ngx_chain_t *in, off_t limit); 176 ngx_chain_t *in, off_t limit);
192 static size_t ngx_quic_max_stream_flow(ngx_connection_t *c); 177 static size_t ngx_quic_max_stream_flow(ngx_connection_t *c);
193 static void ngx_quic_stream_cleanup_handler(void *data); 178 static void ngx_quic_stream_cleanup_handler(void *data);
194 static void ngx_quic_shutdown_quic(ngx_connection_t *c); 179 static void ngx_quic_shutdown_quic(ngx_connection_t *c);
195 static void ngx_quic_free_frame(ngx_connection_t *c, ngx_quic_frame_t *frame);
196 180
197 static void ngx_quic_congestion_ack(ngx_connection_t *c, 181 static void ngx_quic_congestion_ack(ngx_connection_t *c,
198 ngx_quic_frame_t *frame); 182 ngx_quic_frame_t *frame);
199 static void ngx_quic_congestion_lost(ngx_connection_t *c, 183 static void ngx_quic_congestion_lost(ngx_connection_t *c,
200 ngx_quic_frame_t *frame); 184 ngx_quic_frame_t *frame);
201
202 static ngx_chain_t *ngx_quic_alloc_buf(ngx_connection_t *c);
203 static void ngx_quic_free_bufs(ngx_connection_t *c, ngx_chain_t *in);
204 static ngx_chain_t *ngx_quic_copy_buf(ngx_connection_t *c, u_char *data,
205 size_t len);
206 static ngx_chain_t *ngx_quic_copy_chain(ngx_connection_t *c, ngx_chain_t *in,
207 size_t limit);
208 static ngx_chain_t *ngx_quic_split_bufs(ngx_connection_t *c, ngx_chain_t *in,
209 size_t len);
210 185
211 186
212 static ngx_core_module_t ngx_quic_module_ctx = { 187 static ngx_core_module_t ngx_quic_module_ctx = {
213 ngx_string("quic"), 188 ngx_string("quic"),
214 NULL, 189 NULL,
246 221
247 222
248 #if (NGX_DEBUG) 223 #if (NGX_DEBUG)
249 224
250 static void 225 static void
251 ngx_quic_log_frame(ngx_log_t *log, ngx_quic_frame_t *f, ngx_uint_t tx)
252 {
253 u_char *p, *last, *pos, *end;
254 ssize_t n;
255 uint64_t gap, range, largest, smallest;
256 ngx_uint_t i;
257 u_char buf[NGX_MAX_ERROR_STR];
258
259 p = buf;
260 last = buf + sizeof(buf);
261
262 switch (f->type) {
263
264 case NGX_QUIC_FT_CRYPTO:
265 p = ngx_slprintf(p, last, "CRYPTO len:%uL off:%uL",
266 f->u.crypto.length, f->u.crypto.offset);
267 break;
268
269 case NGX_QUIC_FT_PADDING:
270 p = ngx_slprintf(p, last, "PADDING");
271 break;
272
273 case NGX_QUIC_FT_ACK:
274 case NGX_QUIC_FT_ACK_ECN:
275
276 p = ngx_slprintf(p, last, "ACK n:%ui delay:%uL ",
277 f->u.ack.range_count, f->u.ack.delay);
278
279 if (f->data) {
280 pos = f->data->buf->pos;
281 end = f->data->buf->last;
282
283 } else {
284 pos = NULL;
285 end = NULL;
286 }
287
288 largest = f->u.ack.largest;
289 smallest = f->u.ack.largest - f->u.ack.first_range;
290
291 if (largest == smallest) {
292 p = ngx_slprintf(p, last, "%uL", largest);
293
294 } else {
295 p = ngx_slprintf(p, last, "%uL-%uL", largest, smallest);
296 }
297
298 for (i = 0; i < f->u.ack.range_count; i++) {
299 n = ngx_quic_parse_ack_range(log, pos, end, &gap, &range);
300 if (n == NGX_ERROR) {
301 break;
302 }
303
304 pos += n;
305
306 largest = smallest - gap - 2;
307 smallest = largest - range;
308
309 if (largest == smallest) {
310 p = ngx_slprintf(p, last, " %uL", largest);
311
312 } else {
313 p = ngx_slprintf(p, last, " %uL-%uL", largest, smallest);
314 }
315 }
316
317 if (f->type == NGX_QUIC_FT_ACK_ECN) {
318 p = ngx_slprintf(p, last, " ECN counters ect0:%uL ect1:%uL ce:%uL",
319 f->u.ack.ect0, f->u.ack.ect1, f->u.ack.ce);
320 }
321 break;
322
323 case NGX_QUIC_FT_PING:
324 p = ngx_slprintf(p, last, "PING");
325 break;
326
327 case NGX_QUIC_FT_NEW_CONNECTION_ID:
328 p = ngx_slprintf(p, last,
329 "NEW_CONNECTION_ID seq:%uL retire:%uL len:%ud",
330 f->u.ncid.seqnum, f->u.ncid.retire, f->u.ncid.len);
331 break;
332
333 case NGX_QUIC_FT_RETIRE_CONNECTION_ID:
334 p = ngx_slprintf(p, last, "RETIRE_CONNECTION_ID seqnum:%uL",
335 f->u.retire_cid.sequence_number);
336 break;
337
338 case NGX_QUIC_FT_CONNECTION_CLOSE:
339 case NGX_QUIC_FT_CONNECTION_CLOSE_APP:
340 p = ngx_slprintf(p, last, "CONNECTION_CLOSE%s err:%ui",
341 f->type == NGX_QUIC_FT_CONNECTION_CLOSE ? "" : "_APP",
342 f->u.close.error_code);
343
344 if (f->u.close.reason.len) {
345 p = ngx_slprintf(p, last, " %V", &f->u.close.reason);
346 }
347
348 if (f->type == NGX_QUIC_FT_CONNECTION_CLOSE) {
349 p = ngx_slprintf(p, last, " ft:%ui", f->u.close.frame_type);
350 }
351
352 break;
353
354 case NGX_QUIC_FT_STREAM0:
355 case NGX_QUIC_FT_STREAM1:
356 case NGX_QUIC_FT_STREAM2:
357 case NGX_QUIC_FT_STREAM3:
358 case NGX_QUIC_FT_STREAM4:
359 case NGX_QUIC_FT_STREAM5:
360 case NGX_QUIC_FT_STREAM6:
361 case NGX_QUIC_FT_STREAM7:
362
363 p = ngx_slprintf(p, last, "STREAM id:0x%xL", f->u.stream.stream_id);
364
365 if (f->u.stream.off) {
366 p = ngx_slprintf(p, last, " off:%uL", f->u.stream.offset);
367 }
368
369 if (f->u.stream.len) {
370 p = ngx_slprintf(p, last, " len:%uL", f->u.stream.length);
371 }
372
373 if (f->u.stream.fin) {
374 p = ngx_slprintf(p, last, " fin:1");
375 }
376
377 #ifdef NGX_QUIC_DEBUG_FRAMES
378 {
379 ngx_chain_t *cl;
380
381 p = ngx_slprintf(p, last, " data:");
382
383 for (cl = f->data; cl; cl = cl->next) {
384 p = ngx_slprintf(p, last, "%*xs",
385 cl->buf->last - cl->buf->pos, cl->buf->pos);
386 }
387 }
388 #endif
389
390 break;
391
392 case NGX_QUIC_FT_MAX_DATA:
393 p = ngx_slprintf(p, last, "MAX_DATA max_data:%uL on recv",
394 f->u.max_data.max_data);
395 break;
396
397 case NGX_QUIC_FT_RESET_STREAM:
398 p = ngx_slprintf(p, last, "RESET_STREAM"
399 " id:0x%xL error_code:0x%xL final_size:0x%xL",
400 f->u.reset_stream.id, f->u.reset_stream.error_code,
401 f->u.reset_stream.final_size);
402 break;
403
404 case NGX_QUIC_FT_STOP_SENDING:
405 p = ngx_slprintf(p, last, "STOP_SENDING id:0x%xL err:0x%xL",
406 f->u.stop_sending.id, f->u.stop_sending.error_code);
407 break;
408
409 case NGX_QUIC_FT_STREAMS_BLOCKED:
410 case NGX_QUIC_FT_STREAMS_BLOCKED2:
411 p = ngx_slprintf(p, last, "STREAMS_BLOCKED limit:%uL bidi:%ui",
412 f->u.streams_blocked.limit, f->u.streams_blocked.bidi);
413 break;
414
415 case NGX_QUIC_FT_MAX_STREAMS:
416 case NGX_QUIC_FT_MAX_STREAMS2:
417 p = ngx_slprintf(p, last, "MAX_STREAMS limit:%uL bidi:%ui",
418 f->u.max_streams.limit, f->u.max_streams.bidi);
419 break;
420
421 case NGX_QUIC_FT_MAX_STREAM_DATA:
422 p = ngx_slprintf(p, last, "MAX_STREAM_DATA id:0x%xL limit:%uL",
423 f->u.max_stream_data.id, f->u.max_stream_data.limit);
424 break;
425
426
427 case NGX_QUIC_FT_DATA_BLOCKED:
428 p = ngx_slprintf(p, last, "DATA_BLOCKED limit:%uL",
429 f->u.data_blocked.limit);
430 break;
431
432 case NGX_QUIC_FT_STREAM_DATA_BLOCKED:
433 p = ngx_slprintf(p, last, "STREAM_DATA_BLOCKED id:0x%xL limit:%uL",
434 f->u.stream_data_blocked.id,
435 f->u.stream_data_blocked.limit);
436 break;
437
438 case NGX_QUIC_FT_PATH_CHALLENGE:
439 p = ngx_slprintf(p, last, "PATH_CHALLENGE data:0x%*xs",
440 sizeof(f->u.path_challenge.data),
441 f->u.path_challenge.data);
442 break;
443
444 case NGX_QUIC_FT_PATH_RESPONSE:
445 p = ngx_slprintf(p, last, "PATH_RESPONSE data:0x%*xs",
446 sizeof(f->u.path_challenge.data),
447 f->u.path_challenge.data);
448 break;
449
450 case NGX_QUIC_FT_NEW_TOKEN:
451 p = ngx_slprintf(p, last, "NEW_TOKEN");
452 break;
453
454 case NGX_QUIC_FT_HANDSHAKE_DONE:
455 p = ngx_slprintf(p, last, "HANDSHAKE DONE");
456 break;
457
458 default:
459 p = ngx_slprintf(p, last, "unknown type 0x%xi", f->type);
460 break;
461 }
462
463 ngx_log_debug4(NGX_LOG_DEBUG_EVENT, log, 0, "quic frame %s %s %*s",
464 tx ? "tx" : "rx", ngx_quic_level_name(f->level),
465 p - buf, buf);
466 }
467
468
469 static void
470 ngx_quic_connstate_dbg(ngx_connection_t *c) 226 ngx_quic_connstate_dbg(ngx_connection_t *c)
471 { 227 {
472 u_char *p, *last; 228 u_char *p, *last;
473 ngx_quic_connection_t *qc; 229 ngx_quic_connection_t *qc;
474 u_char buf[NGX_MAX_ERROR_STR]; 230 u_char buf[NGX_MAX_ERROR_STR];
529 "quic %*s", p - buf, buf); 285 "quic %*s", p - buf, buf);
530 } 286 }
531 287
532 #else 288 #else
533 289
534 #define ngx_quic_log_frame(log, f, tx)
535 #define ngx_quic_connstate_dbg(c) 290 #define ngx_quic_connstate_dbg(c)
536 291
537 #endif 292 #endif
538 293
539 294
3428 f->u.stream.length, sn->acked, sent - sn->acked); 3183 f->u.stream.length, sn->acked, sent - sn->acked);
3429 } 3184 }
3430 3185
3431 3186
3432 static ngx_int_t 3187 static ngx_int_t
3433 ngx_quic_handle_ordered_frame(ngx_connection_t *c, ngx_quic_frames_stream_t *fs,
3434 ngx_quic_frame_t *frame, ngx_quic_frame_handler_pt handler, void *data)
3435 {
3436 size_t full_len;
3437 ngx_int_t rc;
3438 ngx_queue_t *q;
3439 ngx_quic_ordered_frame_t *f;
3440
3441 f = &frame->u.ord;
3442
3443 if (f->offset > fs->received) {
3444 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
3445 "quic out-of-order frame: expecting:%uL got:%uL",
3446 fs->received, f->offset);
3447
3448 return ngx_quic_buffer_frame(c, fs, frame);
3449 }
3450
3451 if (f->offset < fs->received) {
3452
3453 if (ngx_quic_adjust_frame_offset(c, frame, fs->received)
3454 == NGX_DONE)
3455 {
3456 /* old/duplicate data range */
3457 return handler == ngx_quic_crypto_input ? NGX_DECLINED : NGX_OK;
3458 }
3459
3460 /* intersecting data range, frame modified */
3461 }
3462
3463 /* f->offset == fs->received */
3464
3465 rc = handler(c, frame, data);
3466 if (rc == NGX_ERROR) {
3467 return NGX_ERROR;
3468
3469 } else if (rc == NGX_DONE) {
3470 /* handler destroyed stream, queue no longer exists */
3471 return NGX_OK;
3472 }
3473
3474 /* rc == NGX_OK */
3475
3476 fs->received += f->length;
3477
3478 /* now check the queue if we can continue with buffered frames */
3479
3480 do {
3481 q = ngx_queue_head(&fs->frames);
3482 if (q == ngx_queue_sentinel(&fs->frames)) {
3483 break;
3484 }
3485
3486 frame = ngx_queue_data(q, ngx_quic_frame_t, queue);
3487 f = &frame->u.ord;
3488
3489 if (f->offset > fs->received) {
3490 /* gap found, nothing more to do */
3491 break;
3492 }
3493
3494 full_len = f->length;
3495
3496 if (f->offset < fs->received) {
3497
3498 if (ngx_quic_adjust_frame_offset(c, frame, fs->received)
3499 == NGX_DONE)
3500 {
3501 /* old/duplicate data range */
3502 ngx_queue_remove(q);
3503 fs->total -= f->length;
3504
3505 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
3506 "quic skipped buffered frame, total:%ui",
3507 fs->total);
3508 ngx_quic_free_frame(c, frame);
3509 continue;
3510 }
3511
3512 /* frame was adjusted, proceed to input */
3513 }
3514
3515 /* f->offset == fs->received */
3516
3517 rc = handler(c, frame, data);
3518
3519 if (rc == NGX_ERROR) {
3520 return NGX_ERROR;
3521
3522 } else if (rc == NGX_DONE) {
3523 /* handler destroyed stream, queue no longer exists */
3524 return NGX_OK;
3525 }
3526
3527 fs->received += f->length;
3528 fs->total -= full_len;
3529
3530 ngx_queue_remove(q);
3531
3532 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
3533 "quic consumed buffered frame, total:%ui", fs->total);
3534
3535 ngx_quic_free_frame(c, frame);
3536
3537 } while (1);
3538
3539 return NGX_OK;
3540 }
3541
3542
3543 static ngx_int_t
3544 ngx_quic_adjust_frame_offset(ngx_connection_t *c, ngx_quic_frame_t *frame,
3545 uint64_t offset_in)
3546 {
3547 size_t tail, n;
3548 ngx_buf_t *b;
3549 ngx_chain_t *cl;
3550 ngx_quic_ordered_frame_t *f;
3551
3552 f = &frame->u.ord;
3553
3554 tail = offset_in - f->offset;
3555
3556 if (tail >= f->length) {
3557 /* range preceeding already received data or duplicate, ignore */
3558
3559 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
3560 "quic old or duplicate data in ordered frame, ignored");
3561 return NGX_DONE;
3562 }
3563
3564 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
3565 "quic adjusted ordered frame data start to expected offset");
3566
3567 /* intersecting range: adjust data size */
3568
3569 f->offset += tail;
3570 f->length -= tail;
3571
3572 for (cl = frame->data; cl; cl = cl->next) {
3573 b = cl->buf;
3574 n = ngx_buf_size(b);
3575
3576 if (n >= tail) {
3577 b->pos += tail;
3578 break;
3579 }
3580
3581 cl->buf->pos = cl->buf->last;
3582 tail -= n;
3583 }
3584
3585 return NGX_OK;
3586 }
3587
3588
3589 static ngx_int_t
3590 ngx_quic_buffer_frame(ngx_connection_t *c, ngx_quic_frames_stream_t *fs,
3591 ngx_quic_frame_t *frame)
3592 {
3593 ngx_queue_t *q;
3594 ngx_quic_frame_t *dst, *item;
3595 ngx_quic_ordered_frame_t *f, *df;
3596
3597 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
3598 "quic ngx_quic_buffer_frame");
3599
3600 f = &frame->u.ord;
3601
3602 /* frame start offset is in the future, buffer it */
3603
3604 dst = ngx_quic_alloc_frame(c);
3605 if (dst == NULL) {
3606 return NGX_ERROR;
3607 }
3608
3609 ngx_memcpy(dst, frame, sizeof(ngx_quic_frame_t));
3610
3611 dst->data = ngx_quic_copy_chain(c, frame->data, 0);
3612 if (dst->data == NGX_CHAIN_ERROR) {
3613 return NGX_ERROR;
3614 }
3615
3616 df = &dst->u.ord;
3617
3618 fs->total += f->length;
3619
3620 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
3621 "quic ordered frame with unexpected offset:"
3622 " buffered total:%ui", fs->total);
3623
3624 if (ngx_queue_empty(&fs->frames)) {
3625 ngx_queue_insert_after(&fs->frames, &dst->queue);
3626 return NGX_OK;
3627 }
3628
3629 for (q = ngx_queue_last(&fs->frames);
3630 q != ngx_queue_sentinel(&fs->frames);
3631 q = ngx_queue_prev(q))
3632 {
3633 item = ngx_queue_data(q, ngx_quic_frame_t, queue);
3634 f = &item->u.ord;
3635
3636 if (f->offset < df->offset) {
3637 ngx_queue_insert_after(q, &dst->queue);
3638 return NGX_OK;
3639 }
3640 }
3641
3642 ngx_queue_insert_after(&fs->frames, &dst->queue);
3643
3644 return NGX_OK;
3645 }
3646
3647
3648 static ngx_int_t
3649 ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, 3188 ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt,
3650 ngx_quic_frame_t *frame) 3189 ngx_quic_frame_t *frame)
3651 { 3190 {
3652 uint64_t last; 3191 uint64_t last;
3653 ngx_int_t rc; 3192 ngx_int_t rc;
3691 3230
3692 return NGX_OK; 3231 return NGX_OK;
3693 } 3232 }
3694 3233
3695 3234
3696 static ngx_int_t 3235 ngx_int_t
3697 ngx_quic_crypto_input(ngx_connection_t *c, ngx_quic_frame_t *frame, void *data) 3236 ngx_quic_crypto_input(ngx_connection_t *c, ngx_quic_frame_t *frame, void *data)
3698 { 3237 {
3699 int n, sslerr; 3238 int n, sslerr;
3700 ngx_buf_t *b; 3239 ngx_buf_t *b;
3701 ngx_chain_t *cl; 3240 ngx_chain_t *cl;
4233 "quic max_streams_uni:%uL", f->limit); 3772 "quic max_streams_uni:%uL", f->limit);
4234 } 3773 }
4235 } 3774 }
4236 3775
4237 return NGX_OK; 3776 return NGX_OK;
4238 }
4239
4240
4241 void
4242 ngx_quic_queue_frame(ngx_quic_connection_t *qc, ngx_quic_frame_t *frame)
4243 {
4244 ngx_quic_send_ctx_t *ctx;
4245
4246 ctx = ngx_quic_get_send_ctx(qc, frame->level);
4247
4248 ngx_queue_insert_tail(&ctx->frames, &frame->queue);
4249
4250 frame->len = ngx_quic_create_frame(NULL, frame);
4251 /* always succeeds */
4252
4253 if (qc->closing) {
4254 return;
4255 }
4256
4257 ngx_post_event(&qc->push, &ngx_posted_events);
4258 } 3777 }
4259 3778
4260 3779
4261 static ngx_int_t 3780 static ngx_int_t
4262 ngx_quic_output(ngx_connection_t *c) 3781 ngx_quic_output(ngx_connection_t *c)
4599 4118
4600 return res.len; 4119 return res.len;
4601 } 4120 }
4602 4121
4603 4122
4604 static ngx_int_t
4605 ngx_quic_split_frame(ngx_connection_t *c, ngx_quic_frame_t *f, size_t len)
4606 {
4607 size_t shrink;
4608 ngx_quic_frame_t *nf;
4609 ngx_quic_ordered_frame_t *of, *onf;
4610
4611 switch (f->type) {
4612 case NGX_QUIC_FT_CRYPTO:
4613 case NGX_QUIC_FT_STREAM0:
4614 case NGX_QUIC_FT_STREAM1:
4615 case NGX_QUIC_FT_STREAM2:
4616 case NGX_QUIC_FT_STREAM3:
4617 case NGX_QUIC_FT_STREAM4:
4618 case NGX_QUIC_FT_STREAM5:
4619 case NGX_QUIC_FT_STREAM6:
4620 case NGX_QUIC_FT_STREAM7:
4621 break;
4622
4623 default:
4624 return NGX_DECLINED;
4625 }
4626
4627 if ((size_t) f->len <= len) {
4628 return NGX_OK;
4629 }
4630
4631 shrink = f->len - len;
4632
4633 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
4634 "quic split frame now:%uz need:%uz shrink:%uz",
4635 f->len, len, shrink);
4636
4637 of = &f->u.ord;
4638
4639 if (of->length <= shrink) {
4640 return NGX_DECLINED;
4641 }
4642
4643 of->length -= shrink;
4644 f->len = ngx_quic_create_frame(NULL, f);
4645
4646 if ((size_t) f->len > len) {
4647 ngx_log_error(NGX_LOG_ERR, c->log, 0, "could not split QUIC frame");
4648 return NGX_ERROR;
4649 }
4650
4651 nf = ngx_quic_alloc_frame(c);
4652 if (nf == NULL) {
4653 return NGX_ERROR;
4654 }
4655
4656 *nf = *f;
4657 onf = &nf->u.ord;
4658 onf->offset += of->length;
4659 onf->length = shrink;
4660 nf->len = ngx_quic_create_frame(NULL, nf);
4661
4662 nf->data = ngx_quic_split_bufs(c, f->data, of->length);
4663 if (nf->data == NGX_CHAIN_ERROR) {
4664 return NGX_ERROR;
4665 }
4666
4667 ngx_queue_insert_after(&f->queue, &nf->queue);
4668
4669 return NGX_OK;
4670 }
4671
4672
4673 static void
4674 ngx_quic_free_frames(ngx_connection_t *c, ngx_queue_t *frames)
4675 {
4676 ngx_queue_t *q;
4677 ngx_quic_frame_t *f;
4678
4679 do {
4680 q = ngx_queue_head(frames);
4681
4682 if (q == ngx_queue_sentinel(frames)) {
4683 break;
4684 }
4685
4686 ngx_queue_remove(q);
4687
4688 f = ngx_queue_data(q, ngx_quic_frame_t, queue);
4689
4690 ngx_quic_free_frame(c, f);
4691 } while (1);
4692 }
4693
4694
4695 static ssize_t 4123 static ssize_t
4696 ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len) 4124 ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len)
4697 { 4125 {
4698 ngx_buf_t b; 4126 ngx_buf_t b;
4699 ngx_chain_t cl, *res; 4127 ngx_chain_t cl, *res;
5834 5262
5835 ngx_quic_finalize_connection(c, qc->shutdown_code, qc->shutdown_reason); 5263 ngx_quic_finalize_connection(c, qc->shutdown_code, qc->shutdown_reason);
5836 } 5264 }
5837 5265
5838 5266
5839 ngx_quic_frame_t *
5840 ngx_quic_alloc_frame(ngx_connection_t *c)
5841 {
5842 ngx_queue_t *q;
5843 ngx_quic_frame_t *frame;
5844 ngx_quic_connection_t *qc;
5845
5846 qc = ngx_quic_get_connection(c);
5847
5848 if (!ngx_queue_empty(&qc->free_frames)) {
5849
5850 q = ngx_queue_head(&qc->free_frames);
5851 frame = ngx_queue_data(q, ngx_quic_frame_t, queue);
5852
5853 ngx_queue_remove(&frame->queue);
5854
5855 #ifdef NGX_QUIC_DEBUG_ALLOC
5856 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
5857 "quic reuse frame n:%ui", qc->nframes);
5858 #endif
5859
5860 } else {
5861 frame = ngx_pcalloc(c->pool, sizeof(ngx_quic_frame_t));
5862 if (frame == NULL) {
5863 return NULL;
5864 }
5865
5866 #ifdef NGX_QUIC_DEBUG_ALLOC
5867 ++qc->nframes;
5868
5869 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
5870 "quic alloc frame n:%ui", qc->nframes);
5871 #endif
5872 }
5873
5874 ngx_memzero(frame, sizeof(ngx_quic_frame_t));
5875
5876 return frame;
5877 }
5878
5879 5267
5880 static void 5268 static void
5881 ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f) 5269 ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f)
5882 { 5270 {
5883 ngx_msec_t timer; 5271 ngx_msec_t timer;
5968 "quic congestion lost win:%uz ss:%z if:%uz", 5356 "quic congestion lost win:%uz ss:%z if:%uz",
5969 cg->window, cg->ssthresh, cg->in_flight); 5357 cg->window, cg->ssthresh, cg->in_flight);
5970 } 5358 }
5971 5359
5972 5360
5973 static void
5974 ngx_quic_free_frame(ngx_connection_t *c, ngx_quic_frame_t *frame)
5975 {
5976 ngx_quic_connection_t *qc;
5977
5978 qc = ngx_quic_get_connection(c);
5979
5980 if (frame->data) {
5981 ngx_quic_free_bufs(c, frame->data);
5982 }
5983
5984 ngx_queue_insert_head(&qc->free_frames, &frame->queue);
5985
5986 #ifdef NGX_QUIC_DEBUG_ALLOC
5987 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
5988 "quic free frame n:%ui", qc->nframes);
5989 #endif
5990 }
5991
5992
5993 uint32_t 5361 uint32_t
5994 ngx_quic_version(ngx_connection_t *c) 5362 ngx_quic_version(ngx_connection_t *c)
5995 { 5363 {
5996 uint32_t version; 5364 uint32_t version;
5997 ngx_quic_connection_t *qc; 5365 ngx_quic_connection_t *qc;
6000 5368
6001 version = qc->version; 5369 version = qc->version;
6002 5370
6003 return (version & 0xff000000) == 0xff000000 ? version & 0xff : version; 5371 return (version & 0xff000000) == 0xff000000 ? version & 0xff : version;
6004 } 5372 }
6005
6006
6007 static ngx_chain_t *
6008 ngx_quic_alloc_buf(ngx_connection_t *c)
6009 {
6010 ngx_buf_t *b;
6011 ngx_chain_t *cl;
6012 ngx_quic_connection_t *qc;
6013
6014 qc = ngx_quic_get_connection(c);
6015
6016 if (qc->free_bufs) {
6017 cl = qc->free_bufs;
6018 qc->free_bufs = cl->next;
6019
6020 b = cl->buf;
6021 b->pos = b->start;
6022 b->last = b->start;
6023
6024 #ifdef NGX_QUIC_DEBUG_ALLOC
6025 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
6026 "quic reuse buffer n:%ui", qc->nbufs);
6027 #endif
6028
6029 return cl;
6030 }
6031
6032 cl = ngx_alloc_chain_link(c->pool);
6033 if (cl == NULL) {
6034 return NULL;
6035 }
6036
6037 b = ngx_create_temp_buf(c->pool, NGX_QUIC_BUFFER_SIZE);
6038 if (b == NULL) {
6039 return NULL;
6040 }
6041
6042 b->tag = (ngx_buf_tag_t) &ngx_quic_alloc_buf;
6043
6044 cl->buf = b;
6045
6046 #ifdef NGX_QUIC_DEBUG_ALLOC
6047 ++qc->nbufs;
6048
6049 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
6050 "quic alloc buffer n:%ui", qc->nbufs);
6051 #endif
6052
6053 return cl;
6054 }
6055
6056
6057 static void
6058 ngx_quic_free_bufs(ngx_connection_t *c, ngx_chain_t *in)
6059 {
6060 ngx_buf_t *b, *shadow;
6061 ngx_chain_t *cl;
6062 ngx_quic_connection_t *qc;
6063
6064 qc = ngx_quic_get_connection(c);
6065
6066 while (in) {
6067 #ifdef NGX_QUIC_DEBUG_ALLOC
6068 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
6069 "quic free buffer n:%ui", qc->nbufs);
6070 #endif
6071
6072 cl = in;
6073 in = in->next;
6074 b = cl->buf;
6075
6076 if (b->shadow) {
6077 if (!b->last_shadow) {
6078 b->recycled = 1;
6079 ngx_free_chain(c->pool, cl);
6080 continue;
6081 }
6082
6083 do {
6084 shadow = b->shadow;
6085 b->shadow = qc->free_shadow_bufs;
6086 qc->free_shadow_bufs = b;
6087 b = shadow;
6088 } while (b->recycled);
6089
6090 if (b->shadow) {
6091 b->last_shadow = 1;
6092 ngx_free_chain(c->pool, cl);
6093 continue;
6094 }
6095
6096 cl->buf = b;
6097 }
6098
6099 cl->next = qc->free_bufs;
6100 qc->free_bufs = cl;
6101 }
6102 }
6103
6104
6105 static ngx_chain_t *
6106 ngx_quic_copy_buf(ngx_connection_t *c, u_char *data, size_t len)
6107 {
6108 size_t n;
6109 ngx_buf_t *b;
6110 ngx_chain_t *cl, *out, **ll;
6111
6112 out = NULL;
6113 ll = &out;
6114
6115 while (len) {
6116 cl = ngx_quic_alloc_buf(c);
6117 if (cl == NULL) {
6118 return NGX_CHAIN_ERROR;
6119 }
6120
6121 b = cl->buf;
6122 n = ngx_min((size_t) (b->end - b->last), len);
6123
6124 b->last = ngx_cpymem(b->last, data, n);
6125
6126 data += n;
6127 len -= n;
6128
6129 *ll = cl;
6130 ll = &cl->next;
6131 }
6132
6133 *ll = NULL;
6134
6135 return out;
6136 }
6137
6138
6139 static ngx_chain_t *
6140 ngx_quic_copy_chain(ngx_connection_t *c, ngx_chain_t *in, size_t limit)
6141 {
6142 size_t n;
6143 ngx_buf_t *b;
6144 ngx_chain_t *cl, *out, **ll;
6145
6146 out = NULL;
6147 ll = &out;
6148
6149 while (in) {
6150 if (!ngx_buf_in_memory(in->buf) || ngx_buf_size(in->buf) == 0) {
6151 in = in->next;
6152 continue;
6153 }
6154
6155 cl = ngx_quic_alloc_buf(c);
6156 if (cl == NULL) {
6157 return NGX_CHAIN_ERROR;
6158 }
6159
6160 *ll = cl;
6161 ll = &cl->next;
6162
6163 b = cl->buf;
6164
6165 while (in && b->last != b->end) {
6166
6167 n = ngx_min(in->buf->last - in->buf->pos, b->end - b->last);
6168
6169 if (limit > 0 && n > limit) {
6170 n = limit;
6171 }
6172
6173 b->last = ngx_cpymem(b->last, in->buf->pos, n);
6174
6175 in->buf->pos += n;
6176 if (in->buf->pos == in->buf->last) {
6177 in = in->next;
6178 }
6179
6180 if (limit > 0) {
6181 if (limit == n) {
6182 goto done;
6183 }
6184
6185 limit -= n;
6186 }
6187 }
6188
6189 }
6190
6191 done:
6192
6193 *ll = NULL;
6194
6195 return out;
6196 }
6197
6198
6199 static ngx_chain_t *
6200 ngx_quic_split_bufs(ngx_connection_t *c, ngx_chain_t *in, size_t len)
6201 {
6202 size_t n;
6203 ngx_buf_t *b;
6204 ngx_chain_t *out;
6205 ngx_quic_connection_t *qc;
6206
6207 qc = ngx_quic_get_connection(c);
6208
6209 while (in) {
6210 n = ngx_buf_size(in->buf);
6211
6212 if (n == len) {
6213 out = in->next;
6214 in->next = NULL;
6215 return out;
6216 }
6217
6218 if (n > len) {
6219 break;
6220 }
6221
6222 len -= n;
6223 in = in->next;
6224 }
6225
6226 if (in == NULL) {
6227 return NULL;
6228 }
6229
6230 /* split in->buf by creating shadow bufs which reference it */
6231
6232 if (in->buf->shadow == NULL) {
6233 if (qc->free_shadow_bufs) {
6234 b = qc->free_shadow_bufs;
6235 qc->free_shadow_bufs = b->shadow;
6236
6237 } else {
6238 b = ngx_alloc_buf(c->pool);
6239 if (b == NULL) {
6240 return NGX_CHAIN_ERROR;
6241 }
6242 }
6243
6244 *b = *in->buf;
6245 b->shadow = in->buf;
6246 b->last_shadow = 1;
6247 in->buf = b;
6248 }
6249
6250 out = ngx_alloc_chain_link(c->pool);
6251 if (out == NULL) {
6252 return NGX_CHAIN_ERROR;
6253 }
6254
6255 if (qc->free_shadow_bufs) {
6256 b = qc->free_shadow_bufs;
6257 qc->free_shadow_bufs = b->shadow;
6258
6259 } else {
6260 b = ngx_alloc_buf(c->pool);
6261 if (b == NULL) {
6262 ngx_free_chain(c->pool, out);
6263 return NGX_CHAIN_ERROR;
6264 }
6265 }
6266
6267 out->buf = b;
6268 out->next = in->next;
6269 in->next = NULL;
6270
6271 *b = *in->buf;
6272 b->last_shadow = 0;
6273 b->pos = b->pos + len;
6274
6275 in->buf->shadow = b;
6276 in->buf->last = in->buf->pos + len;
6277
6278 return out;
6279 }