Mercurial > hg > nginx-quic
comparison src/event/ngx_event_quic.c @ 8027:bce9e9643444 quic
QUIC: coalesce neighbouring stream send buffers.
Previously a single STREAM frame was created for each buffer in stream output
chain which is wasteful with respect to memory. The following changes were
made in the stream send code:
- ngx_quic_stream_send_chain() no longer calls ngx_quic_stream_send() and got
a separate implementation that coalesces neighbouring buffers into a single
frame
- the new ngx_quic_stream_send_chain() respects the limit argument, which fixes
sendfile_max_chunk and limit_rate
- ngx_quic_stream_send() is reimplemented to call ngx_quic_stream_send_chain()
- stream frame size limit is moved out to a separate function
ngx_quic_max_stream_frame()
- flow control is moved out to a separate function ngx_quic_max_stream_flow()
- ngx_quic_stream_send_chain() is relocated next to ngx_quic_stream_send()
author | Roman Arutyunyan <arut@nginx.com> |
---|---|
date | Tue, 18 Aug 2020 12:28:33 +0300 |
parents | 4604e6043657 |
children | 6e1c88f82280 |
comparison
equal
deleted
inserted
replaced
8026:4604e6043657 | 8027:bce9e9643444 |
---|---|
278 uint64_t id, size_t rcvbuf_size); | 278 uint64_t id, size_t rcvbuf_size); |
279 static ssize_t ngx_quic_stream_recv(ngx_connection_t *c, u_char *buf, | 279 static ssize_t ngx_quic_stream_recv(ngx_connection_t *c, u_char *buf, |
280 size_t size); | 280 size_t size); |
281 static ssize_t ngx_quic_stream_send(ngx_connection_t *c, u_char *buf, | 281 static ssize_t ngx_quic_stream_send(ngx_connection_t *c, u_char *buf, |
282 size_t size); | 282 size_t size); |
283 static void ngx_quic_stream_cleanup_handler(void *data); | |
284 static ngx_chain_t *ngx_quic_stream_send_chain(ngx_connection_t *c, | 283 static ngx_chain_t *ngx_quic_stream_send_chain(ngx_connection_t *c, |
285 ngx_chain_t *in, off_t limit); | 284 ngx_chain_t *in, off_t limit); |
285 static size_t ngx_quic_max_stream_frame(ngx_quic_connection_t *qc); | |
286 static size_t ngx_quic_max_stream_flow(ngx_connection_t *c); | |
287 static void ngx_quic_stream_cleanup_handler(void *data); | |
286 static ngx_quic_frame_t *ngx_quic_alloc_frame(ngx_connection_t *c, size_t size); | 288 static ngx_quic_frame_t *ngx_quic_alloc_frame(ngx_connection_t *c, size_t size); |
287 static void ngx_quic_free_frame(ngx_connection_t *c, ngx_quic_frame_t *frame); | 289 static void ngx_quic_free_frame(ngx_connection_t *c, ngx_quic_frame_t *frame); |
288 | 290 |
289 static void ngx_quic_congestion_ack(ngx_connection_t *c, | 291 static void ngx_quic_congestion_ack(ngx_connection_t *c, |
290 ngx_quic_frame_t *frame); | 292 ngx_quic_frame_t *frame); |
4226 | 4228 |
4227 | 4229 |
4228 static ssize_t | 4230 static ssize_t |
4229 ngx_quic_stream_send(ngx_connection_t *c, u_char *buf, size_t size) | 4231 ngx_quic_stream_send(ngx_connection_t *c, u_char *buf, size_t size) |
4230 { | 4232 { |
4231 u_char *p, *end; | 4233 ngx_buf_t b; |
4232 size_t fsize, limit, n, len; | 4234 ngx_chain_t cl; |
4233 uint64_t sent, unacked; | 4235 |
4236 ngx_memzero(&b, sizeof(ngx_buf_t)); | |
4237 | |
4238 b.memory = 1; | |
4239 b.pos = buf; | |
4240 b.last = buf + size; | |
4241 | |
4242 cl.buf = &b; | |
4243 cl.next = NULL; | |
4244 | |
4245 if (ngx_quic_stream_send_chain(c, &cl, 0) == NGX_CHAIN_ERROR) { | |
4246 return NGX_ERROR; | |
4247 } | |
4248 | |
4249 if (b.pos == buf) { | |
4250 return NGX_AGAIN; | |
4251 } | |
4252 | |
4253 return b.pos - buf; | |
4254 } | |
4255 | |
4256 | |
4257 static ngx_chain_t * | |
4258 ngx_quic_stream_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) | |
4259 { | |
4260 u_char *p; | |
4261 size_t n, max, max_frame, max_flow, max_limit, len; | |
4262 #if (NGX_DEBUG) | |
4263 size_t sent; | |
4264 #endif | |
4265 ngx_buf_t *b; | |
4266 #if (NGX_DEBUG) | |
4267 ngx_uint_t nframes; | |
4268 #endif | |
4234 ngx_event_t *wev; | 4269 ngx_event_t *wev; |
4270 ngx_chain_t *cl; | |
4235 ngx_connection_t *pc; | 4271 ngx_connection_t *pc; |
4236 ngx_quic_frame_t *frame; | 4272 ngx_quic_frame_t *frame; |
4237 ngx_quic_stream_t *qs; | 4273 ngx_quic_stream_t *qs; |
4238 ngx_quic_connection_t *qc; | 4274 ngx_quic_connection_t *qc; |
4239 | 4275 |
4241 pc = qs->parent; | 4277 pc = qs->parent; |
4242 qc = pc->quic; | 4278 qc = pc->quic; |
4243 wev = c->write; | 4279 wev = c->write; |
4244 | 4280 |
4245 if (wev->error) { | 4281 if (wev->error) { |
4246 return NGX_ERROR; | 4282 return NGX_CHAIN_ERROR; |
4247 } | 4283 } |
4248 | 4284 |
4249 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, | 4285 max_frame = ngx_quic_max_stream_frame(qc); |
4250 "quic stream id 0x%xL send: %uz", qs->id, size); | 4286 max_flow = ngx_quic_max_stream_flow(c); |
4251 | 4287 max_limit = limit; |
4252 /* | 4288 |
4253 * we need to fit at least 1 frame into a packet, thus account head/tail; | 4289 #if (NGX_DEBUG) |
4254 * 25 = 1 + 8x3 is max header for STREAM frame, with 1 byte for frame type | 4290 sent = 0; |
4255 */ | 4291 nframes = 0; |
4256 limit = qc->ctp.max_udp_payload_size - NGX_QUIC_MAX_SHORT_HEADER - 25 | 4292 #endif |
4257 - EVP_GCM_TLS_TAG_LEN; | 4293 |
4258 | 4294 for ( ;; ) { |
4259 len = size; | 4295 max = ngx_min(max_frame, max_flow); |
4260 sent = c->sent; | 4296 |
4261 unacked = sent - qs->acked; | 4297 if (limit) { |
4262 | 4298 max = ngx_min(max, max_limit); |
4263 if (qc->streams.send_max_data == 0) { | 4299 } |
4264 qc->streams.send_max_data = qc->ctp.initial_max_data; | 4300 |
4265 } | 4301 for (cl = in, n = 0; in; in = in->next) { |
4266 | 4302 |
4267 if (unacked >= NGX_QUIC_STREAM_BUFSIZE) { | 4303 if (!ngx_buf_in_memory(in->buf)) { |
4268 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, | 4304 continue; |
4269 "quic send hit buffer size"); | 4305 } |
4270 len = 0; | 4306 |
4271 | 4307 n += ngx_buf_size(in->buf); |
4272 } else if (unacked + len > NGX_QUIC_STREAM_BUFSIZE) { | 4308 |
4273 len = NGX_QUIC_STREAM_BUFSIZE - unacked; | 4309 if (n > max) { |
4274 } | 4310 n = max; |
4275 | 4311 break; |
4276 if (qc->streams.sent >= qc->streams.send_max_data) { | 4312 } |
4277 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, | 4313 } |
4278 "quic send hit MAX_DATA"); | 4314 |
4279 len = 0; | 4315 if (n == 0) { |
4280 | 4316 wev->ready = (max_flow ? 1 : 0); |
4281 } else if (qc->streams.sent + len > qc->streams.send_max_data) { | 4317 break; |
4282 len = qc->streams.send_max_data - qc->streams.sent; | 4318 } |
4283 } | 4319 |
4284 | 4320 frame = ngx_quic_alloc_frame(pc, n); |
4285 if (sent >= qs->send_max_data) { | |
4286 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, | |
4287 "quic send hit MAX_STREAM_DATA"); | |
4288 len = 0; | |
4289 | |
4290 } else if (sent + len > qs->send_max_data) { | |
4291 len = qs->send_max_data - sent; | |
4292 } | |
4293 | |
4294 p = (u_char *) buf; | |
4295 end = (u_char *) buf + len; | |
4296 n = 0; | |
4297 | |
4298 while (p < end) { | |
4299 | |
4300 fsize = ngx_min(limit, (size_t) (end - p)); | |
4301 | |
4302 frame = ngx_quic_alloc_frame(pc, fsize); | |
4303 if (frame == NULL) { | 4321 if (frame == NULL) { |
4304 return 0; | 4322 return NGX_CHAIN_ERROR; |
4305 } | 4323 } |
4306 | |
4307 ngx_memcpy(frame->data, p, fsize); | |
4308 | 4324 |
4309 frame->level = ssl_encryption_application; | 4325 frame->level = ssl_encryption_application; |
4310 frame->type = NGX_QUIC_FT_STREAM6; /* OFF=1 LEN=1 FIN=0 */ | 4326 frame->type = NGX_QUIC_FT_STREAM6; /* OFF=1 LEN=1 FIN=0 */ |
4311 frame->u.stream.off = 1; | 4327 frame->u.stream.off = 1; |
4312 frame->u.stream.len = 1; | 4328 frame->u.stream.len = 1; |
4313 frame->u.stream.fin = 0; | 4329 frame->u.stream.fin = 0; |
4314 | 4330 |
4315 frame->u.stream.type = frame->type; | 4331 frame->u.stream.type = frame->type; |
4316 frame->u.stream.stream_id = qs->id; | 4332 frame->u.stream.stream_id = qs->id; |
4317 frame->u.stream.offset = c->sent; | 4333 frame->u.stream.offset = c->sent; |
4318 frame->u.stream.length = fsize; | 4334 frame->u.stream.length = n; |
4319 frame->u.stream.data = frame->data; | 4335 frame->u.stream.data = frame->data; |
4320 | 4336 |
4321 c->sent += fsize; | 4337 ngx_sprintf(frame->info, "STREAM id:0x%xL len:%uz level:%d", |
4322 qc->streams.sent += fsize; | 4338 qs->id, n, frame->level); |
4323 p += fsize; | 4339 |
4324 n += fsize; | 4340 c->sent += n; |
4325 | 4341 qc->streams.sent += n; |
4326 ngx_sprintf(frame->info, "stream 0x%xL len=%ui level=%d", | 4342 max_flow -= n; |
4327 qs->id, fsize, frame->level); | 4343 |
4344 if (limit) { | |
4345 max_limit -= n; | |
4346 } | |
4347 | |
4348 #if (NGX_DEBUG) | |
4349 sent += n; | |
4350 nframes++; | |
4351 #endif | |
4352 | |
4353 for (p = frame->data; n > 0; cl = cl->next) { | |
4354 b = cl->buf; | |
4355 | |
4356 if (!ngx_buf_in_memory(b)) { | |
4357 continue; | |
4358 } | |
4359 | |
4360 len = ngx_min(n, (size_t) (b->last - b->pos)); | |
4361 p = ngx_cpymem(p, b->pos, len); | |
4362 | |
4363 b->pos += len; | |
4364 n -= len; | |
4365 } | |
4328 | 4366 |
4329 ngx_quic_queue_frame(qc, frame); | 4367 ngx_quic_queue_frame(qc, frame); |
4330 } | 4368 } |
4331 | 4369 |
4332 ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, | 4370 ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, |
4333 "quic send %uz of %uz, sent:%O, unacked:%uL", | 4371 "quic send_chain sent:%uz, frames:%ui", sent, nframes); |
4334 n, size, c->sent, (uint64_t) c->sent - qs->acked); | 4372 |
4335 | 4373 return in; |
4336 if (n != size) { | 4374 } |
4337 c->write->ready = 0; | 4375 |
4338 } | 4376 |
4339 | 4377 static size_t |
4340 if (n == 0) { | 4378 ngx_quic_max_stream_frame(ngx_quic_connection_t *qc) |
4341 return NGX_AGAIN; | 4379 { |
4342 } | 4380 /* |
4343 | 4381 * we need to fit at least 1 frame into a packet, thus account head/tail; |
4344 return n; | 4382 * 25 = 1 + 8x3 is max header for STREAM frame, with 1 byte for frame type |
4383 */ | |
4384 | |
4385 return qc->ctp.max_udp_payload_size - NGX_QUIC_MAX_SHORT_HEADER - 25 | |
4386 - EVP_GCM_TLS_TAG_LEN; | |
4387 } | |
4388 | |
4389 | |
4390 static size_t | |
4391 ngx_quic_max_stream_flow(ngx_connection_t *c) | |
4392 { | |
4393 size_t size; | |
4394 uint64_t sent, unacked; | |
4395 ngx_quic_stream_t *qs; | |
4396 ngx_quic_connection_t *qc; | |
4397 | |
4398 qs = c->qs; | |
4399 qc = qs->parent->quic; | |
4400 | |
4401 size = NGX_QUIC_STREAM_BUFSIZE; | |
4402 sent = c->sent; | |
4403 unacked = sent - qs->acked; | |
4404 | |
4405 if (qc->streams.send_max_data == 0) { | |
4406 qc->streams.send_max_data = qc->ctp.initial_max_data; | |
4407 } | |
4408 | |
4409 if (unacked >= NGX_QUIC_STREAM_BUFSIZE) { | |
4410 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, | |
4411 "quic send flow hit buffer size"); | |
4412 return 0; | |
4413 } | |
4414 | |
4415 if (unacked + size > NGX_QUIC_STREAM_BUFSIZE) { | |
4416 size = NGX_QUIC_STREAM_BUFSIZE - unacked; | |
4417 } | |
4418 | |
4419 if (qc->streams.sent >= qc->streams.send_max_data) { | |
4420 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, | |
4421 "quic send flow hit MAX_DATA"); | |
4422 return 0; | |
4423 } | |
4424 | |
4425 if (qc->streams.sent + size > qc->streams.send_max_data) { | |
4426 size = qc->streams.send_max_data - qc->streams.sent; | |
4427 } | |
4428 | |
4429 if (sent >= qs->send_max_data) { | |
4430 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, | |
4431 "quic send flow hit MAX_STREAM_DATA"); | |
4432 return 0; | |
4433 } | |
4434 | |
4435 if (sent + size > qs->send_max_data) { | |
4436 size = qs->send_max_data - sent; | |
4437 } | |
4438 | |
4439 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, | |
4440 "quic send flow: %uz", size); | |
4441 | |
4442 return size; | |
4345 } | 4443 } |
4346 | 4444 |
4347 | 4445 |
4348 static void | 4446 static void |
4349 ngx_quic_stream_cleanup_handler(void *data) | 4447 ngx_quic_stream_cleanup_handler(void *data) |
4429 | 4527 |
4430 (void) ngx_quic_output(pc); | 4528 (void) ngx_quic_output(pc); |
4431 } | 4529 } |
4432 | 4530 |
4433 | 4531 |
4434 static ngx_chain_t * | |
4435 ngx_quic_stream_send_chain(ngx_connection_t *c, ngx_chain_t *in, | |
4436 off_t limit) | |
4437 { | |
4438 size_t len; | |
4439 ssize_t n; | |
4440 ngx_buf_t *b; | |
4441 | |
4442 for ( /* void */; in; in = in->next) { | |
4443 b = in->buf; | |
4444 | |
4445 if (!ngx_buf_in_memory(b)) { | |
4446 continue; | |
4447 } | |
4448 | |
4449 if (ngx_buf_size(b) == 0) { | |
4450 continue; | |
4451 } | |
4452 | |
4453 len = b->last - b->pos; | |
4454 | |
4455 n = ngx_quic_stream_send(c, b->pos, len); | |
4456 | |
4457 if (n == NGX_ERROR) { | |
4458 return NGX_CHAIN_ERROR; | |
4459 } | |
4460 | |
4461 if (n == NGX_AGAIN) { | |
4462 return in; | |
4463 } | |
4464 | |
4465 b->pos += n; | |
4466 | |
4467 if (n != (ssize_t) len) { | |
4468 return in; | |
4469 } | |
4470 } | |
4471 | |
4472 return NULL; | |
4473 } | |
4474 | |
4475 | |
4476 static ngx_quic_frame_t * | 4532 static ngx_quic_frame_t * |
4477 ngx_quic_alloc_frame(ngx_connection_t *c, size_t size) | 4533 ngx_quic_alloc_frame(ngx_connection_t *c, size_t size) |
4478 { | 4534 { |
4479 u_char *p; | 4535 u_char *p; |
4480 ngx_queue_t *q; | 4536 ngx_queue_t *q; |