Mercurial > hg > nginx
comparison src/event/quic/ngx_event_quic.c @ 8749:660c4a2f95f3 quic
QUIC: separate files for frames related processing.
author | Vladimir Homutov <vl@nginx.com> |
---|---|
date | Tue, 13 Apr 2021 14:38:46 +0300 |
parents | e0cb1e58ca13 |
children | 41807e581de9 |
comparison
equal
deleted
inserted
replaced
8748:e0cb1e58ca13 | 8749:660c4a2f95f3 |
---|---|
32 | 32 |
33 #define NGX_QUIC_MIN_SR_PACKET 43 /* 5 random + 16 srt + 22 padding */ | 33 #define NGX_QUIC_MIN_SR_PACKET 43 /* 5 random + 16 srt + 22 padding */ |
34 #define NGX_QUIC_MAX_SR_PACKET 1200 | 34 #define NGX_QUIC_MAX_SR_PACKET 1200 |
35 | 35 |
36 #define NGX_QUIC_MAX_ACK_GAP 2 | 36 #define NGX_QUIC_MAX_ACK_GAP 2 |
37 | |
38 | |
39 typedef ngx_int_t (*ngx_quic_frame_handler_pt)(ngx_connection_t *c, | |
40 ngx_quic_frame_t *frame, void *data); | |
41 | 37 |
42 | 38 |
43 #if BORINGSSL_API_VERSION >= 10 | 39 #if BORINGSSL_API_VERSION >= 10 |
44 static int ngx_quic_set_read_secret(ngx_ssl_conn_t *ssl_conn, | 40 static int ngx_quic_set_read_secret(ngx_ssl_conn_t *ssl_conn, |
45 enum ssl_encryption_level_t level, const SSL_CIPHER *cipher, | 41 enum ssl_encryption_level_t level, const SSL_CIPHER *cipher, |
120 static void ngx_quic_rtt_sample(ngx_connection_t *c, ngx_quic_ack_frame_t *ack, | 116 static void ngx_quic_rtt_sample(ngx_connection_t *c, ngx_quic_ack_frame_t *ack, |
121 enum ssl_encryption_level_t level, ngx_msec_t send_time); | 117 enum ssl_encryption_level_t level, ngx_msec_t send_time); |
122 static void ngx_quic_handle_stream_ack(ngx_connection_t *c, | 118 static void ngx_quic_handle_stream_ack(ngx_connection_t *c, |
123 ngx_quic_frame_t *f); | 119 ngx_quic_frame_t *f); |
124 | 120 |
125 static ngx_int_t ngx_quic_handle_ordered_frame(ngx_connection_t *c, | |
126 ngx_quic_frames_stream_t *fs, ngx_quic_frame_t *frame, | |
127 ngx_quic_frame_handler_pt handler, void *data); | |
128 static ngx_int_t ngx_quic_adjust_frame_offset(ngx_connection_t *c, | |
129 ngx_quic_frame_t *f, uint64_t offset_in); | |
130 static ngx_int_t ngx_quic_buffer_frame(ngx_connection_t *c, | |
131 ngx_quic_frames_stream_t *stream, ngx_quic_frame_t *f); | |
132 | |
133 static ngx_int_t ngx_quic_handle_crypto_frame(ngx_connection_t *c, | 121 static ngx_int_t ngx_quic_handle_crypto_frame(ngx_connection_t *c, |
134 ngx_quic_header_t *pkt, ngx_quic_frame_t *frame); | 122 ngx_quic_header_t *pkt, ngx_quic_frame_t *frame); |
135 static ngx_int_t ngx_quic_crypto_input(ngx_connection_t *c, | 123 ngx_int_t ngx_quic_crypto_input(ngx_connection_t *c, |
136 ngx_quic_frame_t *frame, void *data); | 124 ngx_quic_frame_t *frame, void *data); |
137 static ngx_int_t ngx_quic_handle_stream_frame(ngx_connection_t *c, | 125 static ngx_int_t ngx_quic_handle_stream_frame(ngx_connection_t *c, |
138 ngx_quic_header_t *pkt, ngx_quic_frame_t *frame); | 126 ngx_quic_header_t *pkt, ngx_quic_frame_t *frame); |
139 static ngx_int_t ngx_quic_stream_input(ngx_connection_t *c, | 127 static ngx_int_t ngx_quic_stream_input(ngx_connection_t *c, |
140 ngx_quic_frame_t *frame, void *data); | 128 ngx_quic_frame_t *frame, void *data); |
158 static ngx_uint_t ngx_quic_get_padding_level(ngx_connection_t *c); | 146 static ngx_uint_t ngx_quic_get_padding_level(ngx_connection_t *c); |
159 static ngx_int_t ngx_quic_generate_ack(ngx_connection_t *c, | 147 static ngx_int_t ngx_quic_generate_ack(ngx_connection_t *c, |
160 ngx_quic_send_ctx_t *ctx); | 148 ngx_quic_send_ctx_t *ctx); |
161 static ssize_t ngx_quic_output_packet(ngx_connection_t *c, | 149 static ssize_t ngx_quic_output_packet(ngx_connection_t *c, |
162 ngx_quic_send_ctx_t *ctx, u_char *data, size_t max, size_t min); | 150 ngx_quic_send_ctx_t *ctx, u_char *data, size_t max, size_t min); |
163 static ngx_int_t ngx_quic_split_frame(ngx_connection_t *c, ngx_quic_frame_t *f, | |
164 size_t len); | |
165 static void ngx_quic_free_frames(ngx_connection_t *c, ngx_queue_t *frames); | |
166 static ssize_t ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len); | 151 static ssize_t ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len); |
167 | 152 |
168 static void ngx_quic_set_packet_number(ngx_quic_header_t *pkt, | 153 static void ngx_quic_set_packet_number(ngx_quic_header_t *pkt, |
169 ngx_quic_send_ctx_t *ctx); | 154 ngx_quic_send_ctx_t *ctx); |
170 static void ngx_quic_pto_handler(ngx_event_t *ev); | 155 static void ngx_quic_pto_handler(ngx_event_t *ev); |
190 static ngx_chain_t *ngx_quic_stream_send_chain(ngx_connection_t *c, | 175 static ngx_chain_t *ngx_quic_stream_send_chain(ngx_connection_t *c, |
191 ngx_chain_t *in, off_t limit); | 176 ngx_chain_t *in, off_t limit); |
192 static size_t ngx_quic_max_stream_flow(ngx_connection_t *c); | 177 static size_t ngx_quic_max_stream_flow(ngx_connection_t *c); |
193 static void ngx_quic_stream_cleanup_handler(void *data); | 178 static void ngx_quic_stream_cleanup_handler(void *data); |
194 static void ngx_quic_shutdown_quic(ngx_connection_t *c); | 179 static void ngx_quic_shutdown_quic(ngx_connection_t *c); |
195 static void ngx_quic_free_frame(ngx_connection_t *c, ngx_quic_frame_t *frame); | |
196 | 180 |
197 static void ngx_quic_congestion_ack(ngx_connection_t *c, | 181 static void ngx_quic_congestion_ack(ngx_connection_t *c, |
198 ngx_quic_frame_t *frame); | 182 ngx_quic_frame_t *frame); |
199 static void ngx_quic_congestion_lost(ngx_connection_t *c, | 183 static void ngx_quic_congestion_lost(ngx_connection_t *c, |
200 ngx_quic_frame_t *frame); | 184 ngx_quic_frame_t *frame); |
201 | |
202 static ngx_chain_t *ngx_quic_alloc_buf(ngx_connection_t *c); | |
203 static void ngx_quic_free_bufs(ngx_connection_t *c, ngx_chain_t *in); | |
204 static ngx_chain_t *ngx_quic_copy_buf(ngx_connection_t *c, u_char *data, | |
205 size_t len); | |
206 static ngx_chain_t *ngx_quic_copy_chain(ngx_connection_t *c, ngx_chain_t *in, | |
207 size_t limit); | |
208 static ngx_chain_t *ngx_quic_split_bufs(ngx_connection_t *c, ngx_chain_t *in, | |
209 size_t len); | |
210 | 185 |
211 | 186 |
212 static ngx_core_module_t ngx_quic_module_ctx = { | 187 static ngx_core_module_t ngx_quic_module_ctx = { |
213 ngx_string("quic"), | 188 ngx_string("quic"), |
214 NULL, | 189 NULL, |
246 | 221 |
247 | 222 |
248 #if (NGX_DEBUG) | 223 #if (NGX_DEBUG) |
249 | 224 |
250 static void | 225 static void |
251 ngx_quic_log_frame(ngx_log_t *log, ngx_quic_frame_t *f, ngx_uint_t tx) | |
252 { | |
253 u_char *p, *last, *pos, *end; | |
254 ssize_t n; | |
255 uint64_t gap, range, largest, smallest; | |
256 ngx_uint_t i; | |
257 u_char buf[NGX_MAX_ERROR_STR]; | |
258 | |
259 p = buf; | |
260 last = buf + sizeof(buf); | |
261 | |
262 switch (f->type) { | |
263 | |
264 case NGX_QUIC_FT_CRYPTO: | |
265 p = ngx_slprintf(p, last, "CRYPTO len:%uL off:%uL", | |
266 f->u.crypto.length, f->u.crypto.offset); | |
267 break; | |
268 | |
269 case NGX_QUIC_FT_PADDING: | |
270 p = ngx_slprintf(p, last, "PADDING"); | |
271 break; | |
272 | |
273 case NGX_QUIC_FT_ACK: | |
274 case NGX_QUIC_FT_ACK_ECN: | |
275 | |
276 p = ngx_slprintf(p, last, "ACK n:%ui delay:%uL ", | |
277 f->u.ack.range_count, f->u.ack.delay); | |
278 | |
279 if (f->data) { | |
280 pos = f->data->buf->pos; | |
281 end = f->data->buf->last; | |
282 | |
283 } else { | |
284 pos = NULL; | |
285 end = NULL; | |
286 } | |
287 | |
288 largest = f->u.ack.largest; | |
289 smallest = f->u.ack.largest - f->u.ack.first_range; | |
290 | |
291 if (largest == smallest) { | |
292 p = ngx_slprintf(p, last, "%uL", largest); | |
293 | |
294 } else { | |
295 p = ngx_slprintf(p, last, "%uL-%uL", largest, smallest); | |
296 } | |
297 | |
298 for (i = 0; i < f->u.ack.range_count; i++) { | |
299 n = ngx_quic_parse_ack_range(log, pos, end, &gap, &range); | |
300 if (n == NGX_ERROR) { | |
301 break; | |
302 } | |
303 | |
304 pos += n; | |
305 | |
306 largest = smallest - gap - 2; | |
307 smallest = largest - range; | |
308 | |
309 if (largest == smallest) { | |
310 p = ngx_slprintf(p, last, " %uL", largest); | |
311 | |
312 } else { | |
313 p = ngx_slprintf(p, last, " %uL-%uL", largest, smallest); | |
314 } | |
315 } | |
316 | |
317 if (f->type == NGX_QUIC_FT_ACK_ECN) { | |
318 p = ngx_slprintf(p, last, " ECN counters ect0:%uL ect1:%uL ce:%uL", | |
319 f->u.ack.ect0, f->u.ack.ect1, f->u.ack.ce); | |
320 } | |
321 break; | |
322 | |
323 case NGX_QUIC_FT_PING: | |
324 p = ngx_slprintf(p, last, "PING"); | |
325 break; | |
326 | |
327 case NGX_QUIC_FT_NEW_CONNECTION_ID: | |
328 p = ngx_slprintf(p, last, | |
329 "NEW_CONNECTION_ID seq:%uL retire:%uL len:%ud", | |
330 f->u.ncid.seqnum, f->u.ncid.retire, f->u.ncid.len); | |
331 break; | |
332 | |
333 case NGX_QUIC_FT_RETIRE_CONNECTION_ID: | |
334 p = ngx_slprintf(p, last, "RETIRE_CONNECTION_ID seqnum:%uL", | |
335 f->u.retire_cid.sequence_number); | |
336 break; | |
337 | |
338 case NGX_QUIC_FT_CONNECTION_CLOSE: | |
339 case NGX_QUIC_FT_CONNECTION_CLOSE_APP: | |
340 p = ngx_slprintf(p, last, "CONNECTION_CLOSE%s err:%ui", | |
341 f->type == NGX_QUIC_FT_CONNECTION_CLOSE ? "" : "_APP", | |
342 f->u.close.error_code); | |
343 | |
344 if (f->u.close.reason.len) { | |
345 p = ngx_slprintf(p, last, " %V", &f->u.close.reason); | |
346 } | |
347 | |
348 if (f->type == NGX_QUIC_FT_CONNECTION_CLOSE) { | |
349 p = ngx_slprintf(p, last, " ft:%ui", f->u.close.frame_type); | |
350 } | |
351 | |
352 break; | |
353 | |
354 case NGX_QUIC_FT_STREAM0: | |
355 case NGX_QUIC_FT_STREAM1: | |
356 case NGX_QUIC_FT_STREAM2: | |
357 case NGX_QUIC_FT_STREAM3: | |
358 case NGX_QUIC_FT_STREAM4: | |
359 case NGX_QUIC_FT_STREAM5: | |
360 case NGX_QUIC_FT_STREAM6: | |
361 case NGX_QUIC_FT_STREAM7: | |
362 | |
363 p = ngx_slprintf(p, last, "STREAM id:0x%xL", f->u.stream.stream_id); | |
364 | |
365 if (f->u.stream.off) { | |
366 p = ngx_slprintf(p, last, " off:%uL", f->u.stream.offset); | |
367 } | |
368 | |
369 if (f->u.stream.len) { | |
370 p = ngx_slprintf(p, last, " len:%uL", f->u.stream.length); | |
371 } | |
372 | |
373 if (f->u.stream.fin) { | |
374 p = ngx_slprintf(p, last, " fin:1"); | |
375 } | |
376 | |
377 #ifdef NGX_QUIC_DEBUG_FRAMES | |
378 { | |
379 ngx_chain_t *cl; | |
380 | |
381 p = ngx_slprintf(p, last, " data:"); | |
382 | |
383 for (cl = f->data; cl; cl = cl->next) { | |
384 p = ngx_slprintf(p, last, "%*xs", | |
385 cl->buf->last - cl->buf->pos, cl->buf->pos); | |
386 } | |
387 } | |
388 #endif | |
389 | |
390 break; | |
391 | |
392 case NGX_QUIC_FT_MAX_DATA: | |
393 p = ngx_slprintf(p, last, "MAX_DATA max_data:%uL on recv", | |
394 f->u.max_data.max_data); | |
395 break; | |
396 | |
397 case NGX_QUIC_FT_RESET_STREAM: | |
398 p = ngx_slprintf(p, last, "RESET_STREAM" | |
399 " id:0x%xL error_code:0x%xL final_size:0x%xL", | |
400 f->u.reset_stream.id, f->u.reset_stream.error_code, | |
401 f->u.reset_stream.final_size); | |
402 break; | |
403 | |
404 case NGX_QUIC_FT_STOP_SENDING: | |
405 p = ngx_slprintf(p, last, "STOP_SENDING id:0x%xL err:0x%xL", | |
406 f->u.stop_sending.id, f->u.stop_sending.error_code); | |
407 break; | |
408 | |
409 case NGX_QUIC_FT_STREAMS_BLOCKED: | |
410 case NGX_QUIC_FT_STREAMS_BLOCKED2: | |
411 p = ngx_slprintf(p, last, "STREAMS_BLOCKED limit:%uL bidi:%ui", | |
412 f->u.streams_blocked.limit, f->u.streams_blocked.bidi); | |
413 break; | |
414 | |
415 case NGX_QUIC_FT_MAX_STREAMS: | |
416 case NGX_QUIC_FT_MAX_STREAMS2: | |
417 p = ngx_slprintf(p, last, "MAX_STREAMS limit:%uL bidi:%ui", | |
418 f->u.max_streams.limit, f->u.max_streams.bidi); | |
419 break; | |
420 | |
421 case NGX_QUIC_FT_MAX_STREAM_DATA: | |
422 p = ngx_slprintf(p, last, "MAX_STREAM_DATA id:0x%xL limit:%uL", | |
423 f->u.max_stream_data.id, f->u.max_stream_data.limit); | |
424 break; | |
425 | |
426 | |
427 case NGX_QUIC_FT_DATA_BLOCKED: | |
428 p = ngx_slprintf(p, last, "DATA_BLOCKED limit:%uL", | |
429 f->u.data_blocked.limit); | |
430 break; | |
431 | |
432 case NGX_QUIC_FT_STREAM_DATA_BLOCKED: | |
433 p = ngx_slprintf(p, last, "STREAM_DATA_BLOCKED id:0x%xL limit:%uL", | |
434 f->u.stream_data_blocked.id, | |
435 f->u.stream_data_blocked.limit); | |
436 break; | |
437 | |
438 case NGX_QUIC_FT_PATH_CHALLENGE: | |
439 p = ngx_slprintf(p, last, "PATH_CHALLENGE data:0x%*xs", | |
440 sizeof(f->u.path_challenge.data), | |
441 f->u.path_challenge.data); | |
442 break; | |
443 | |
444 case NGX_QUIC_FT_PATH_RESPONSE: | |
445 p = ngx_slprintf(p, last, "PATH_RESPONSE data:0x%*xs", | |
446 sizeof(f->u.path_challenge.data), | |
447 f->u.path_challenge.data); | |
448 break; | |
449 | |
450 case NGX_QUIC_FT_NEW_TOKEN: | |
451 p = ngx_slprintf(p, last, "NEW_TOKEN"); | |
452 break; | |
453 | |
454 case NGX_QUIC_FT_HANDSHAKE_DONE: | |
455 p = ngx_slprintf(p, last, "HANDSHAKE DONE"); | |
456 break; | |
457 | |
458 default: | |
459 p = ngx_slprintf(p, last, "unknown type 0x%xi", f->type); | |
460 break; | |
461 } | |
462 | |
463 ngx_log_debug4(NGX_LOG_DEBUG_EVENT, log, 0, "quic frame %s %s %*s", | |
464 tx ? "tx" : "rx", ngx_quic_level_name(f->level), | |
465 p - buf, buf); | |
466 } | |
467 | |
468 | |
469 static void | |
470 ngx_quic_connstate_dbg(ngx_connection_t *c) | 226 ngx_quic_connstate_dbg(ngx_connection_t *c) |
471 { | 227 { |
472 u_char *p, *last; | 228 u_char *p, *last; |
473 ngx_quic_connection_t *qc; | 229 ngx_quic_connection_t *qc; |
474 u_char buf[NGX_MAX_ERROR_STR]; | 230 u_char buf[NGX_MAX_ERROR_STR]; |
529 "quic %*s", p - buf, buf); | 285 "quic %*s", p - buf, buf); |
530 } | 286 } |
531 | 287 |
532 #else | 288 #else |
533 | 289 |
534 #define ngx_quic_log_frame(log, f, tx) | |
535 #define ngx_quic_connstate_dbg(c) | 290 #define ngx_quic_connstate_dbg(c) |
536 | 291 |
537 #endif | 292 #endif |
538 | 293 |
539 | 294 |
3428 f->u.stream.length, sn->acked, sent - sn->acked); | 3183 f->u.stream.length, sn->acked, sent - sn->acked); |
3429 } | 3184 } |
3430 | 3185 |
3431 | 3186 |
3432 static ngx_int_t | 3187 static ngx_int_t |
3433 ngx_quic_handle_ordered_frame(ngx_connection_t *c, ngx_quic_frames_stream_t *fs, | |
3434 ngx_quic_frame_t *frame, ngx_quic_frame_handler_pt handler, void *data) | |
3435 { | |
3436 size_t full_len; | |
3437 ngx_int_t rc; | |
3438 ngx_queue_t *q; | |
3439 ngx_quic_ordered_frame_t *f; | |
3440 | |
3441 f = &frame->u.ord; | |
3442 | |
3443 if (f->offset > fs->received) { | |
3444 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, | |
3445 "quic out-of-order frame: expecting:%uL got:%uL", | |
3446 fs->received, f->offset); | |
3447 | |
3448 return ngx_quic_buffer_frame(c, fs, frame); | |
3449 } | |
3450 | |
3451 if (f->offset < fs->received) { | |
3452 | |
3453 if (ngx_quic_adjust_frame_offset(c, frame, fs->received) | |
3454 == NGX_DONE) | |
3455 { | |
3456 /* old/duplicate data range */ | |
3457 return handler == ngx_quic_crypto_input ? NGX_DECLINED : NGX_OK; | |
3458 } | |
3459 | |
3460 /* intersecting data range, frame modified */ | |
3461 } | |
3462 | |
3463 /* f->offset == fs->received */ | |
3464 | |
3465 rc = handler(c, frame, data); | |
3466 if (rc == NGX_ERROR) { | |
3467 return NGX_ERROR; | |
3468 | |
3469 } else if (rc == NGX_DONE) { | |
3470 /* handler destroyed stream, queue no longer exists */ | |
3471 return NGX_OK; | |
3472 } | |
3473 | |
3474 /* rc == NGX_OK */ | |
3475 | |
3476 fs->received += f->length; | |
3477 | |
3478 /* now check the queue if we can continue with buffered frames */ | |
3479 | |
3480 do { | |
3481 q = ngx_queue_head(&fs->frames); | |
3482 if (q == ngx_queue_sentinel(&fs->frames)) { | |
3483 break; | |
3484 } | |
3485 | |
3486 frame = ngx_queue_data(q, ngx_quic_frame_t, queue); | |
3487 f = &frame->u.ord; | |
3488 | |
3489 if (f->offset > fs->received) { | |
3490 /* gap found, nothing more to do */ | |
3491 break; | |
3492 } | |
3493 | |
3494 full_len = f->length; | |
3495 | |
3496 if (f->offset < fs->received) { | |
3497 | |
3498 if (ngx_quic_adjust_frame_offset(c, frame, fs->received) | |
3499 == NGX_DONE) | |
3500 { | |
3501 /* old/duplicate data range */ | |
3502 ngx_queue_remove(q); | |
3503 fs->total -= f->length; | |
3504 | |
3505 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, | |
3506 "quic skipped buffered frame, total:%ui", | |
3507 fs->total); | |
3508 ngx_quic_free_frame(c, frame); | |
3509 continue; | |
3510 } | |
3511 | |
3512 /* frame was adjusted, proceed to input */ | |
3513 } | |
3514 | |
3515 /* f->offset == fs->received */ | |
3516 | |
3517 rc = handler(c, frame, data); | |
3518 | |
3519 if (rc == NGX_ERROR) { | |
3520 return NGX_ERROR; | |
3521 | |
3522 } else if (rc == NGX_DONE) { | |
3523 /* handler destroyed stream, queue no longer exists */ | |
3524 return NGX_OK; | |
3525 } | |
3526 | |
3527 fs->received += f->length; | |
3528 fs->total -= full_len; | |
3529 | |
3530 ngx_queue_remove(q); | |
3531 | |
3532 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, | |
3533 "quic consumed buffered frame, total:%ui", fs->total); | |
3534 | |
3535 ngx_quic_free_frame(c, frame); | |
3536 | |
3537 } while (1); | |
3538 | |
3539 return NGX_OK; | |
3540 } | |
3541 | |
3542 | |
3543 static ngx_int_t | |
3544 ngx_quic_adjust_frame_offset(ngx_connection_t *c, ngx_quic_frame_t *frame, | |
3545 uint64_t offset_in) | |
3546 { | |
3547 size_t tail, n; | |
3548 ngx_buf_t *b; | |
3549 ngx_chain_t *cl; | |
3550 ngx_quic_ordered_frame_t *f; | |
3551 | |
3552 f = &frame->u.ord; | |
3553 | |
3554 tail = offset_in - f->offset; | |
3555 | |
3556 if (tail >= f->length) { | |
3557 /* range preceeding already received data or duplicate, ignore */ | |
3558 | |
3559 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, | |
3560 "quic old or duplicate data in ordered frame, ignored"); | |
3561 return NGX_DONE; | |
3562 } | |
3563 | |
3564 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, | |
3565 "quic adjusted ordered frame data start to expected offset"); | |
3566 | |
3567 /* intersecting range: adjust data size */ | |
3568 | |
3569 f->offset += tail; | |
3570 f->length -= tail; | |
3571 | |
3572 for (cl = frame->data; cl; cl = cl->next) { | |
3573 b = cl->buf; | |
3574 n = ngx_buf_size(b); | |
3575 | |
3576 if (n >= tail) { | |
3577 b->pos += tail; | |
3578 break; | |
3579 } | |
3580 | |
3581 cl->buf->pos = cl->buf->last; | |
3582 tail -= n; | |
3583 } | |
3584 | |
3585 return NGX_OK; | |
3586 } | |
3587 | |
3588 | |
3589 static ngx_int_t | |
3590 ngx_quic_buffer_frame(ngx_connection_t *c, ngx_quic_frames_stream_t *fs, | |
3591 ngx_quic_frame_t *frame) | |
3592 { | |
3593 ngx_queue_t *q; | |
3594 ngx_quic_frame_t *dst, *item; | |
3595 ngx_quic_ordered_frame_t *f, *df; | |
3596 | |
3597 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, | |
3598 "quic ngx_quic_buffer_frame"); | |
3599 | |
3600 f = &frame->u.ord; | |
3601 | |
3602 /* frame start offset is in the future, buffer it */ | |
3603 | |
3604 dst = ngx_quic_alloc_frame(c); | |
3605 if (dst == NULL) { | |
3606 return NGX_ERROR; | |
3607 } | |
3608 | |
3609 ngx_memcpy(dst, frame, sizeof(ngx_quic_frame_t)); | |
3610 | |
3611 dst->data = ngx_quic_copy_chain(c, frame->data, 0); | |
3612 if (dst->data == NGX_CHAIN_ERROR) { | |
3613 return NGX_ERROR; | |
3614 } | |
3615 | |
3616 df = &dst->u.ord; | |
3617 | |
3618 fs->total += f->length; | |
3619 | |
3620 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, | |
3621 "quic ordered frame with unexpected offset:" | |
3622 " buffered total:%ui", fs->total); | |
3623 | |
3624 if (ngx_queue_empty(&fs->frames)) { | |
3625 ngx_queue_insert_after(&fs->frames, &dst->queue); | |
3626 return NGX_OK; | |
3627 } | |
3628 | |
3629 for (q = ngx_queue_last(&fs->frames); | |
3630 q != ngx_queue_sentinel(&fs->frames); | |
3631 q = ngx_queue_prev(q)) | |
3632 { | |
3633 item = ngx_queue_data(q, ngx_quic_frame_t, queue); | |
3634 f = &item->u.ord; | |
3635 | |
3636 if (f->offset < df->offset) { | |
3637 ngx_queue_insert_after(q, &dst->queue); | |
3638 return NGX_OK; | |
3639 } | |
3640 } | |
3641 | |
3642 ngx_queue_insert_after(&fs->frames, &dst->queue); | |
3643 | |
3644 return NGX_OK; | |
3645 } | |
3646 | |
3647 | |
3648 static ngx_int_t | |
3649 ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, | 3188 ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, |
3650 ngx_quic_frame_t *frame) | 3189 ngx_quic_frame_t *frame) |
3651 { | 3190 { |
3652 uint64_t last; | 3191 uint64_t last; |
3653 ngx_int_t rc; | 3192 ngx_int_t rc; |
3691 | 3230 |
3692 return NGX_OK; | 3231 return NGX_OK; |
3693 } | 3232 } |
3694 | 3233 |
3695 | 3234 |
3696 static ngx_int_t | 3235 ngx_int_t |
3697 ngx_quic_crypto_input(ngx_connection_t *c, ngx_quic_frame_t *frame, void *data) | 3236 ngx_quic_crypto_input(ngx_connection_t *c, ngx_quic_frame_t *frame, void *data) |
3698 { | 3237 { |
3699 int n, sslerr; | 3238 int n, sslerr; |
3700 ngx_buf_t *b; | 3239 ngx_buf_t *b; |
3701 ngx_chain_t *cl; | 3240 ngx_chain_t *cl; |
4233 "quic max_streams_uni:%uL", f->limit); | 3772 "quic max_streams_uni:%uL", f->limit); |
4234 } | 3773 } |
4235 } | 3774 } |
4236 | 3775 |
4237 return NGX_OK; | 3776 return NGX_OK; |
4238 } | |
4239 | |
4240 | |
4241 void | |
4242 ngx_quic_queue_frame(ngx_quic_connection_t *qc, ngx_quic_frame_t *frame) | |
4243 { | |
4244 ngx_quic_send_ctx_t *ctx; | |
4245 | |
4246 ctx = ngx_quic_get_send_ctx(qc, frame->level); | |
4247 | |
4248 ngx_queue_insert_tail(&ctx->frames, &frame->queue); | |
4249 | |
4250 frame->len = ngx_quic_create_frame(NULL, frame); | |
4251 /* always succeeds */ | |
4252 | |
4253 if (qc->closing) { | |
4254 return; | |
4255 } | |
4256 | |
4257 ngx_post_event(&qc->push, &ngx_posted_events); | |
4258 } | 3777 } |
4259 | 3778 |
4260 | 3779 |
4261 static ngx_int_t | 3780 static ngx_int_t |
4262 ngx_quic_output(ngx_connection_t *c) | 3781 ngx_quic_output(ngx_connection_t *c) |
4599 | 4118 |
4600 return res.len; | 4119 return res.len; |
4601 } | 4120 } |
4602 | 4121 |
4603 | 4122 |
4604 static ngx_int_t | |
4605 ngx_quic_split_frame(ngx_connection_t *c, ngx_quic_frame_t *f, size_t len) | |
4606 { | |
4607 size_t shrink; | |
4608 ngx_quic_frame_t *nf; | |
4609 ngx_quic_ordered_frame_t *of, *onf; | |
4610 | |
4611 switch (f->type) { | |
4612 case NGX_QUIC_FT_CRYPTO: | |
4613 case NGX_QUIC_FT_STREAM0: | |
4614 case NGX_QUIC_FT_STREAM1: | |
4615 case NGX_QUIC_FT_STREAM2: | |
4616 case NGX_QUIC_FT_STREAM3: | |
4617 case NGX_QUIC_FT_STREAM4: | |
4618 case NGX_QUIC_FT_STREAM5: | |
4619 case NGX_QUIC_FT_STREAM6: | |
4620 case NGX_QUIC_FT_STREAM7: | |
4621 break; | |
4622 | |
4623 default: | |
4624 return NGX_DECLINED; | |
4625 } | |
4626 | |
4627 if ((size_t) f->len <= len) { | |
4628 return NGX_OK; | |
4629 } | |
4630 | |
4631 shrink = f->len - len; | |
4632 | |
4633 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, | |
4634 "quic split frame now:%uz need:%uz shrink:%uz", | |
4635 f->len, len, shrink); | |
4636 | |
4637 of = &f->u.ord; | |
4638 | |
4639 if (of->length <= shrink) { | |
4640 return NGX_DECLINED; | |
4641 } | |
4642 | |
4643 of->length -= shrink; | |
4644 f->len = ngx_quic_create_frame(NULL, f); | |
4645 | |
4646 if ((size_t) f->len > len) { | |
4647 ngx_log_error(NGX_LOG_ERR, c->log, 0, "could not split QUIC frame"); | |
4648 return NGX_ERROR; | |
4649 } | |
4650 | |
4651 nf = ngx_quic_alloc_frame(c); | |
4652 if (nf == NULL) { | |
4653 return NGX_ERROR; | |
4654 } | |
4655 | |
4656 *nf = *f; | |
4657 onf = &nf->u.ord; | |
4658 onf->offset += of->length; | |
4659 onf->length = shrink; | |
4660 nf->len = ngx_quic_create_frame(NULL, nf); | |
4661 | |
4662 nf->data = ngx_quic_split_bufs(c, f->data, of->length); | |
4663 if (nf->data == NGX_CHAIN_ERROR) { | |
4664 return NGX_ERROR; | |
4665 } | |
4666 | |
4667 ngx_queue_insert_after(&f->queue, &nf->queue); | |
4668 | |
4669 return NGX_OK; | |
4670 } | |
4671 | |
4672 | |
4673 static void | |
4674 ngx_quic_free_frames(ngx_connection_t *c, ngx_queue_t *frames) | |
4675 { | |
4676 ngx_queue_t *q; | |
4677 ngx_quic_frame_t *f; | |
4678 | |
4679 do { | |
4680 q = ngx_queue_head(frames); | |
4681 | |
4682 if (q == ngx_queue_sentinel(frames)) { | |
4683 break; | |
4684 } | |
4685 | |
4686 ngx_queue_remove(q); | |
4687 | |
4688 f = ngx_queue_data(q, ngx_quic_frame_t, queue); | |
4689 | |
4690 ngx_quic_free_frame(c, f); | |
4691 } while (1); | |
4692 } | |
4693 | |
4694 | |
4695 static ssize_t | 4123 static ssize_t |
4696 ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len) | 4124 ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len) |
4697 { | 4125 { |
4698 ngx_buf_t b; | 4126 ngx_buf_t b; |
4699 ngx_chain_t cl, *res; | 4127 ngx_chain_t cl, *res; |
5834 | 5262 |
5835 ngx_quic_finalize_connection(c, qc->shutdown_code, qc->shutdown_reason); | 5263 ngx_quic_finalize_connection(c, qc->shutdown_code, qc->shutdown_reason); |
5836 } | 5264 } |
5837 | 5265 |
5838 | 5266 |
5839 ngx_quic_frame_t * | |
5840 ngx_quic_alloc_frame(ngx_connection_t *c) | |
5841 { | |
5842 ngx_queue_t *q; | |
5843 ngx_quic_frame_t *frame; | |
5844 ngx_quic_connection_t *qc; | |
5845 | |
5846 qc = ngx_quic_get_connection(c); | |
5847 | |
5848 if (!ngx_queue_empty(&qc->free_frames)) { | |
5849 | |
5850 q = ngx_queue_head(&qc->free_frames); | |
5851 frame = ngx_queue_data(q, ngx_quic_frame_t, queue); | |
5852 | |
5853 ngx_queue_remove(&frame->queue); | |
5854 | |
5855 #ifdef NGX_QUIC_DEBUG_ALLOC | |
5856 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, | |
5857 "quic reuse frame n:%ui", qc->nframes); | |
5858 #endif | |
5859 | |
5860 } else { | |
5861 frame = ngx_pcalloc(c->pool, sizeof(ngx_quic_frame_t)); | |
5862 if (frame == NULL) { | |
5863 return NULL; | |
5864 } | |
5865 | |
5866 #ifdef NGX_QUIC_DEBUG_ALLOC | |
5867 ++qc->nframes; | |
5868 | |
5869 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, | |
5870 "quic alloc frame n:%ui", qc->nframes); | |
5871 #endif | |
5872 } | |
5873 | |
5874 ngx_memzero(frame, sizeof(ngx_quic_frame_t)); | |
5875 | |
5876 return frame; | |
5877 } | |
5878 | |
5879 | 5267 |
5880 static void | 5268 static void |
5881 ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f) | 5269 ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f) |
5882 { | 5270 { |
5883 ngx_msec_t timer; | 5271 ngx_msec_t timer; |
5968 "quic congestion lost win:%uz ss:%z if:%uz", | 5356 "quic congestion lost win:%uz ss:%z if:%uz", |
5969 cg->window, cg->ssthresh, cg->in_flight); | 5357 cg->window, cg->ssthresh, cg->in_flight); |
5970 } | 5358 } |
5971 | 5359 |
5972 | 5360 |
5973 static void | |
5974 ngx_quic_free_frame(ngx_connection_t *c, ngx_quic_frame_t *frame) | |
5975 { | |
5976 ngx_quic_connection_t *qc; | |
5977 | |
5978 qc = ngx_quic_get_connection(c); | |
5979 | |
5980 if (frame->data) { | |
5981 ngx_quic_free_bufs(c, frame->data); | |
5982 } | |
5983 | |
5984 ngx_queue_insert_head(&qc->free_frames, &frame->queue); | |
5985 | |
5986 #ifdef NGX_QUIC_DEBUG_ALLOC | |
5987 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, | |
5988 "quic free frame n:%ui", qc->nframes); | |
5989 #endif | |
5990 } | |
5991 | |
5992 | |
5993 uint32_t | 5361 uint32_t |
5994 ngx_quic_version(ngx_connection_t *c) | 5362 ngx_quic_version(ngx_connection_t *c) |
5995 { | 5363 { |
5996 uint32_t version; | 5364 uint32_t version; |
5997 ngx_quic_connection_t *qc; | 5365 ngx_quic_connection_t *qc; |
6000 | 5368 |
6001 version = qc->version; | 5369 version = qc->version; |
6002 | 5370 |
6003 return (version & 0xff000000) == 0xff000000 ? version & 0xff : version; | 5371 return (version & 0xff000000) == 0xff000000 ? version & 0xff : version; |
6004 } | 5372 } |
6005 | |
6006 | |
6007 static ngx_chain_t * | |
6008 ngx_quic_alloc_buf(ngx_connection_t *c) | |
6009 { | |
6010 ngx_buf_t *b; | |
6011 ngx_chain_t *cl; | |
6012 ngx_quic_connection_t *qc; | |
6013 | |
6014 qc = ngx_quic_get_connection(c); | |
6015 | |
6016 if (qc->free_bufs) { | |
6017 cl = qc->free_bufs; | |
6018 qc->free_bufs = cl->next; | |
6019 | |
6020 b = cl->buf; | |
6021 b->pos = b->start; | |
6022 b->last = b->start; | |
6023 | |
6024 #ifdef NGX_QUIC_DEBUG_ALLOC | |
6025 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, | |
6026 "quic reuse buffer n:%ui", qc->nbufs); | |
6027 #endif | |
6028 | |
6029 return cl; | |
6030 } | |
6031 | |
6032 cl = ngx_alloc_chain_link(c->pool); | |
6033 if (cl == NULL) { | |
6034 return NULL; | |
6035 } | |
6036 | |
6037 b = ngx_create_temp_buf(c->pool, NGX_QUIC_BUFFER_SIZE); | |
6038 if (b == NULL) { | |
6039 return NULL; | |
6040 } | |
6041 | |
6042 b->tag = (ngx_buf_tag_t) &ngx_quic_alloc_buf; | |
6043 | |
6044 cl->buf = b; | |
6045 | |
6046 #ifdef NGX_QUIC_DEBUG_ALLOC | |
6047 ++qc->nbufs; | |
6048 | |
6049 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, | |
6050 "quic alloc buffer n:%ui", qc->nbufs); | |
6051 #endif | |
6052 | |
6053 return cl; | |
6054 } | |
6055 | |
6056 | |
6057 static void | |
6058 ngx_quic_free_bufs(ngx_connection_t *c, ngx_chain_t *in) | |
6059 { | |
6060 ngx_buf_t *b, *shadow; | |
6061 ngx_chain_t *cl; | |
6062 ngx_quic_connection_t *qc; | |
6063 | |
6064 qc = ngx_quic_get_connection(c); | |
6065 | |
6066 while (in) { | |
6067 #ifdef NGX_QUIC_DEBUG_ALLOC | |
6068 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, | |
6069 "quic free buffer n:%ui", qc->nbufs); | |
6070 #endif | |
6071 | |
6072 cl = in; | |
6073 in = in->next; | |
6074 b = cl->buf; | |
6075 | |
6076 if (b->shadow) { | |
6077 if (!b->last_shadow) { | |
6078 b->recycled = 1; | |
6079 ngx_free_chain(c->pool, cl); | |
6080 continue; | |
6081 } | |
6082 | |
6083 do { | |
6084 shadow = b->shadow; | |
6085 b->shadow = qc->free_shadow_bufs; | |
6086 qc->free_shadow_bufs = b; | |
6087 b = shadow; | |
6088 } while (b->recycled); | |
6089 | |
6090 if (b->shadow) { | |
6091 b->last_shadow = 1; | |
6092 ngx_free_chain(c->pool, cl); | |
6093 continue; | |
6094 } | |
6095 | |
6096 cl->buf = b; | |
6097 } | |
6098 | |
6099 cl->next = qc->free_bufs; | |
6100 qc->free_bufs = cl; | |
6101 } | |
6102 } | |
6103 | |
6104 | |
6105 static ngx_chain_t * | |
6106 ngx_quic_copy_buf(ngx_connection_t *c, u_char *data, size_t len) | |
6107 { | |
6108 size_t n; | |
6109 ngx_buf_t *b; | |
6110 ngx_chain_t *cl, *out, **ll; | |
6111 | |
6112 out = NULL; | |
6113 ll = &out; | |
6114 | |
6115 while (len) { | |
6116 cl = ngx_quic_alloc_buf(c); | |
6117 if (cl == NULL) { | |
6118 return NGX_CHAIN_ERROR; | |
6119 } | |
6120 | |
6121 b = cl->buf; | |
6122 n = ngx_min((size_t) (b->end - b->last), len); | |
6123 | |
6124 b->last = ngx_cpymem(b->last, data, n); | |
6125 | |
6126 data += n; | |
6127 len -= n; | |
6128 | |
6129 *ll = cl; | |
6130 ll = &cl->next; | |
6131 } | |
6132 | |
6133 *ll = NULL; | |
6134 | |
6135 return out; | |
6136 } | |
6137 | |
6138 | |
6139 static ngx_chain_t * | |
6140 ngx_quic_copy_chain(ngx_connection_t *c, ngx_chain_t *in, size_t limit) | |
6141 { | |
6142 size_t n; | |
6143 ngx_buf_t *b; | |
6144 ngx_chain_t *cl, *out, **ll; | |
6145 | |
6146 out = NULL; | |
6147 ll = &out; | |
6148 | |
6149 while (in) { | |
6150 if (!ngx_buf_in_memory(in->buf) || ngx_buf_size(in->buf) == 0) { | |
6151 in = in->next; | |
6152 continue; | |
6153 } | |
6154 | |
6155 cl = ngx_quic_alloc_buf(c); | |
6156 if (cl == NULL) { | |
6157 return NGX_CHAIN_ERROR; | |
6158 } | |
6159 | |
6160 *ll = cl; | |
6161 ll = &cl->next; | |
6162 | |
6163 b = cl->buf; | |
6164 | |
6165 while (in && b->last != b->end) { | |
6166 | |
6167 n = ngx_min(in->buf->last - in->buf->pos, b->end - b->last); | |
6168 | |
6169 if (limit > 0 && n > limit) { | |
6170 n = limit; | |
6171 } | |
6172 | |
6173 b->last = ngx_cpymem(b->last, in->buf->pos, n); | |
6174 | |
6175 in->buf->pos += n; | |
6176 if (in->buf->pos == in->buf->last) { | |
6177 in = in->next; | |
6178 } | |
6179 | |
6180 if (limit > 0) { | |
6181 if (limit == n) { | |
6182 goto done; | |
6183 } | |
6184 | |
6185 limit -= n; | |
6186 } | |
6187 } | |
6188 | |
6189 } | |
6190 | |
6191 done: | |
6192 | |
6193 *ll = NULL; | |
6194 | |
6195 return out; | |
6196 } | |
6197 | |
6198 | |
6199 static ngx_chain_t * | |
6200 ngx_quic_split_bufs(ngx_connection_t *c, ngx_chain_t *in, size_t len) | |
6201 { | |
6202 size_t n; | |
6203 ngx_buf_t *b; | |
6204 ngx_chain_t *out; | |
6205 ngx_quic_connection_t *qc; | |
6206 | |
6207 qc = ngx_quic_get_connection(c); | |
6208 | |
6209 while (in) { | |
6210 n = ngx_buf_size(in->buf); | |
6211 | |
6212 if (n == len) { | |
6213 out = in->next; | |
6214 in->next = NULL; | |
6215 return out; | |
6216 } | |
6217 | |
6218 if (n > len) { | |
6219 break; | |
6220 } | |
6221 | |
6222 len -= n; | |
6223 in = in->next; | |
6224 } | |
6225 | |
6226 if (in == NULL) { | |
6227 return NULL; | |
6228 } | |
6229 | |
6230 /* split in->buf by creating shadow bufs which reference it */ | |
6231 | |
6232 if (in->buf->shadow == NULL) { | |
6233 if (qc->free_shadow_bufs) { | |
6234 b = qc->free_shadow_bufs; | |
6235 qc->free_shadow_bufs = b->shadow; | |
6236 | |
6237 } else { | |
6238 b = ngx_alloc_buf(c->pool); | |
6239 if (b == NULL) { | |
6240 return NGX_CHAIN_ERROR; | |
6241 } | |
6242 } | |
6243 | |
6244 *b = *in->buf; | |
6245 b->shadow = in->buf; | |
6246 b->last_shadow = 1; | |
6247 in->buf = b; | |
6248 } | |
6249 | |
6250 out = ngx_alloc_chain_link(c->pool); | |
6251 if (out == NULL) { | |
6252 return NGX_CHAIN_ERROR; | |
6253 } | |
6254 | |
6255 if (qc->free_shadow_bufs) { | |
6256 b = qc->free_shadow_bufs; | |
6257 qc->free_shadow_bufs = b->shadow; | |
6258 | |
6259 } else { | |
6260 b = ngx_alloc_buf(c->pool); | |
6261 if (b == NULL) { | |
6262 ngx_free_chain(c->pool, out); | |
6263 return NGX_CHAIN_ERROR; | |
6264 } | |
6265 } | |
6266 | |
6267 out->buf = b; | |
6268 out->next = in->next; | |
6269 in->next = NULL; | |
6270 | |
6271 *b = *in->buf; | |
6272 b->last_shadow = 0; | |
6273 b->pos = b->pos + len; | |
6274 | |
6275 in->buf->shadow = b; | |
6276 in->buf->last = in->buf->pos + len; | |
6277 | |
6278 return out; | |
6279 } |