Mercurial > hg > nginx

comparison src/event/quic/ngx_event_quic_output.c @ 8811:bb5152ed045b quic

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
QUIC: added support for segmentation offloading. To improve output performance, UDP segmentation offloading is used if available. If there is a significant amount of data in an output queue and path is verified, QUIC packets are not sent one-by-one, but instead are collected in a buffer, which is then passed to kernel in a single sendmsg call, using UDP GSO. Such method greatly decreases number of system calls and thus system load.
author Vladimir Homutov <vl@nginx.com>
date Thu, 15 Jul 2021 14:22:00 +0300
parents 5b0c229ba5fe
children 8ab0d609af09
comparison
equal deleted inserted replaced
8810:2dfd313f22f2 8811:bb5152ed045b
14 #define NGX_QUIC_MAX_LONG_HEADER 56 14 #define NGX_QUIC_MAX_LONG_HEADER 56
15 /* 1 flags + 4 version + 2 x (1 + 20) s/dcid + 4 pn + 4 len + token len */ 15 /* 1 flags + 4 version + 2 x (1 + 20) s/dcid + 4 pn + 4 len + token len */
16 16
17 #define NGX_QUIC_MAX_UDP_PAYLOAD_OUT 1252 17 #define NGX_QUIC_MAX_UDP_PAYLOAD_OUT 1252
18 #define NGX_QUIC_MAX_UDP_PAYLOAD_OUT6 1232 18 #define NGX_QUIC_MAX_UDP_PAYLOAD_OUT6 1232
19
20 #define NGX_QUIC_MAX_UDP_SEGMENT_BUF 65487 /* 65K - IPv6 header */
21 #define NGX_QUIC_MAX_SEGMENTS 64 /* UDP_MAX_SEGMENTS */
19 22
20 #define NGX_QUIC_RETRY_TOKEN_LIFETIME 3 /* seconds */ 23 #define NGX_QUIC_RETRY_TOKEN_LIFETIME 3 /* seconds */
21 #define NGX_QUIC_NEW_TOKEN_LIFETIME 600 /* seconds */ 24 #define NGX_QUIC_NEW_TOKEN_LIFETIME 600 /* seconds */
22 #define NGX_QUIC_RETRY_BUFFER_SIZE 256 25 #define NGX_QUIC_RETRY_BUFFER_SIZE 256
23 /* 1 flags + 4 version + 3 x (1 + 20) s/o/dcid + itag + token(64) */ 26 /* 1 flags + 4 version + 3 x (1 + 20) s/o/dcid + itag + token(64) */
37 #define NGX_QUIC_CC_MIN_INTERVAL 1000 /* 1s */ 40 #define NGX_QUIC_CC_MIN_INTERVAL 1000 /* 1s */
38 41
39 42
40 static ngx_int_t ngx_quic_socket_output(ngx_connection_t *c, 43 static ngx_int_t ngx_quic_socket_output(ngx_connection_t *c,
41 ngx_quic_socket_t *qsock); 44 ngx_quic_socket_t *qsock);
45 static ngx_int_t ngx_quic_create_datagrams(ngx_connection_t *c,
46 ngx_quic_socket_t *qsock);
47 #if ((NGX_HAVE_UDP_SEGMENT) && (NGX_HAVE_MSGHDR_MSG_CONTROL))
48 static ngx_uint_t ngx_quic_allow_segmentation(ngx_connection_t *c,
49 ngx_quic_socket_t *qsock);
50 static ngx_int_t ngx_quic_create_segments(ngx_connection_t *c,
51 ngx_quic_socket_t *qsock);
52 static ssize_t ngx_quic_send_segments(ngx_connection_t *c, u_char *buf,
53 size_t len, struct sockaddr *sockaddr, socklen_t socklen, size_t segment);
54 #endif
42 static ssize_t ngx_quic_output_packet(ngx_connection_t *c, 55 static ssize_t ngx_quic_output_packet(ngx_connection_t *c,
43 ngx_quic_send_ctx_t *ctx, u_char *data, size_t max, size_t min, 56 ngx_quic_send_ctx_t *ctx, u_char *data, size_t max, size_t min,
44 ngx_quic_socket_t *qsock); 57 ngx_quic_socket_t *qsock);
45 static ngx_uint_t ngx_quic_get_padding_level(ngx_connection_t *c); 58 static ngx_uint_t ngx_quic_get_padding_level(ngx_connection_t *c);
46 static ssize_t ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len, 59 static ssize_t ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len,
82 95
83 96
84 static ngx_int_t 97 static ngx_int_t
85 ngx_quic_socket_output(ngx_connection_t *c, ngx_quic_socket_t *qsock) 98 ngx_quic_socket_output(ngx_connection_t *c, ngx_quic_socket_t *qsock)
86 { 99 {
100 size_t in_flight;
101 ngx_int_t rc;
102 ngx_quic_congestion_t *cg;
103 ngx_quic_connection_t *qc;
104
105 c->log->action = "sending frames";
106
107 qc = ngx_quic_get_connection(c);
108 cg = &qc->congestion;
109
110 in_flight = cg->in_flight;
111
112 #if ((NGX_HAVE_UDP_SEGMENT) && (NGX_HAVE_MSGHDR_MSG_CONTROL))
113 if (ngx_quic_allow_segmentation(c, qsock)) {
114 rc = ngx_quic_create_segments(c, qsock);
115 } else
116 #endif
117 {
118 rc = ngx_quic_create_datagrams(c, qsock);
119 }
120
121 if (rc != NGX_OK) {
122 return NGX_ERROR;
123 }
124
125 if (in_flight != cg->in_flight && !qc->send_timer_set && !qc->closing) {
126 qc->send_timer_set = 1;
127 ngx_add_timer(c->read, qc->tp.max_idle_timeout);
128 }
129
130 return NGX_OK;
131 }
132
133
134 static ngx_int_t
135 ngx_quic_create_datagrams(ngx_connection_t *c, ngx_quic_socket_t *qsock)
136 {
87 off_t max; 137 off_t max;
88 size_t len, min, in_flight; 138 size_t len, min;
89 ssize_t n; 139 ssize_t n;
90 u_char *p; 140 u_char *p;
91 ngx_uint_t i, pad; 141 ngx_uint_t i, pad;
92 ngx_quic_path_t *path; 142 ngx_quic_path_t *path;
93 ngx_quic_send_ctx_t *ctx; 143 ngx_quic_send_ctx_t *ctx;
94 ngx_quic_congestion_t *cg;
95 ngx_quic_connection_t *qc; 144 ngx_quic_connection_t *qc;
96 static u_char dst[NGX_QUIC_MAX_UDP_PAYLOAD_SIZE]; 145 static u_char dst[NGX_QUIC_MAX_UDP_PAYLOAD_SIZE];
97 146
98 c->log->action = "sending frames";
99
100 qc = ngx_quic_get_connection(c); 147 qc = ngx_quic_get_connection(c);
101 cg = &qc->congestion;
102
103 in_flight = cg->in_flight;
104 148
105 path = qsock->path; 149 path = qsock->path;
106 150
107 for ( ;; ) { 151 for ( ;; ) {
108 p = dst; 152 p = dst;
151 } 195 }
152 196
153 path->sent += len; 197 path->sent += len;
154 } 198 }
155 199
156 if (in_flight != cg->in_flight && !qc->send_timer_set && !qc->closing) {
157 qc->send_timer_set = 1;
158 ngx_add_timer(c->read, qc->tp.max_idle_timeout);
159 }
160
161
162 return NGX_OK; 200 return NGX_OK;
163 } 201 }
202
203
204 #if ((NGX_HAVE_UDP_SEGMENT) && (NGX_HAVE_MSGHDR_MSG_CONTROL))
205
206 static ngx_uint_t
207 ngx_quic_allow_segmentation(ngx_connection_t *c, ngx_quic_socket_t *qsock)
208 {
209 size_t bytes, len;
210 ngx_queue_t *q;
211 ngx_quic_frame_t *f;
212 ngx_quic_send_ctx_t *ctx;
213 ngx_quic_connection_t *qc;
214
215 if (qsock->path->state != NGX_QUIC_PATH_VALIDATED) {
216 /* don't even try to be faster on non-validated paths */
217 return 0;
218 }
219
220 qc = ngx_quic_get_connection(c);
221
222 ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_initial);
223 if (!ngx_queue_empty(&ctx->frames)) {
224 return 0;
225 }
226
227 ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_handshake);
228 if (!ngx_queue_empty(&ctx->frames)) {
229 return 0;
230 }
231
232 ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
233
234 bytes = 0;
235
236 len = ngx_min(qc->ctp.max_udp_payload_size,
237 NGX_QUIC_MAX_UDP_SEGMENT_BUF);
238
239 for (q = ngx_queue_head(&ctx->frames);
240 q != ngx_queue_sentinel(&ctx->frames);
241 q = ngx_queue_next(q))
242 {
243 f = ngx_queue_data(q, ngx_quic_frame_t, queue);
244
245 bytes += f->len;
246
247 if (bytes > len * 3) {
248 /* require at least ~3 full packets to batch */
249 return 1;
250 }
251 }
252
253 return 0;
254 }
255
256
257 static ngx_int_t
258 ngx_quic_create_segments(ngx_connection_t *c, ngx_quic_socket_t *qsock)
259 {
260 size_t len, segsize;
261 ssize_t n;
262 u_char *p, *end;
263 ngx_uint_t nseg;
264 ngx_quic_send_ctx_t *ctx;
265 ngx_quic_path_t *path;
266 ngx_quic_connection_t *qc;
267 static u_char dst[NGX_QUIC_MAX_UDP_SEGMENT_BUF];
268
269 qc = ngx_quic_get_connection(c);
270 path = qsock->path;
271
272 ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
273
274 if (ngx_quic_generate_ack(c, ctx) != NGX_OK) {
275 return NGX_ERROR;
276 }
277
278 segsize = ngx_min(qc->ctp.max_udp_payload_size,
279 NGX_QUIC_MAX_UDP_SEGMENT_BUF);
280 p = dst;
281 end = dst + sizeof(dst);
282
283 nseg = 0;
284
285 for ( ;; ) {
286
287 len = ngx_min(segsize, (size_t) (end - p));
288
289 if (len) {
290
291 n = ngx_quic_output_packet(c, ctx, p, len, len, qsock);
292 if (n == NGX_ERROR) {
293 return NGX_ERROR;
294 }
295
296 p += n;
297 nseg++;
298
299 } else {
300 n = 0;
301 }
302
303 if (p == dst) {
304 break;
305 }
306
307 if (n == 0 || nseg == NGX_QUIC_MAX_SEGMENTS) {
308 n = ngx_quic_send_segments(c, dst, p - dst, path->sockaddr,
309 path->socklen, segsize);
310 if (n == NGX_ERROR) {
311 return NGX_ERROR;
312 }
313
314 path->sent += n;
315
316 p = dst;
317 nseg = 0;
318 }
319 }
320
321 return NGX_OK;
322 }
323
324
325 static ssize_t
326 ngx_quic_send_segments(ngx_connection_t *c, u_char *buf, size_t len,
327 struct sockaddr *sockaddr, socklen_t socklen, size_t segment)
328 {
329 size_t clen;
330 ssize_t n;
331 uint16_t *valp;
332 struct iovec iov;
333 struct msghdr msg;
334 struct cmsghdr *cmsg;
335
336 #if defined(NGX_HAVE_ADDRINFO_CMSG)
337 char msg_control[CMSG_SPACE(sizeof(uint16_t))
338 + CMSG_SPACE(sizeof(ngx_addrinfo_t))];
339 #else
340 char msg_control[CMSG_SPACE(sizeof(uint16_t))];
341 #endif
342
343 ngx_memzero(&msg, sizeof(struct msghdr));
344 ngx_memzero(msg_control, sizeof(msg_control));
345
346 iov.iov_len = len;
347 iov.iov_base = buf;
348
349 msg.msg_iov = &iov;
350 msg.msg_iovlen = 1;
351
352 msg.msg_name = sockaddr;
353 msg.msg_namelen = socklen;
354
355 msg.msg_control = msg_control;
356 msg.msg_controllen = sizeof(msg_control);
357
358 cmsg = CMSG_FIRSTHDR(&msg);
359
360 cmsg->cmsg_level = SOL_UDP;
361 cmsg->cmsg_type = UDP_SEGMENT;
362 cmsg->cmsg_len = CMSG_LEN(sizeof(uint16_t));
363
364 clen = CMSG_SPACE(sizeof(uint16_t));
365
366 valp = (void *) CMSG_DATA(cmsg);
367 *valp = segment;
368
369 #if defined(NGX_HAVE_ADDRINFO_CMSG)
370 if (c->listening && c->listening->wildcard && c->local_sockaddr) {
371 cmsg = CMSG_NXTHDR(&msg, cmsg);
372 clen += ngx_set_srcaddr_cmsg(cmsg, c->local_sockaddr);
373 }
374 #endif
375
376 msg.msg_controllen = clen;
377
378 n = ngx_sendmsg(c, &msg, 0);
379 if (n == -1) {
380 return NGX_ERROR;
381 }
382
383 c->sent += n;
384
385 return n;
386 }
387
388 #endif
389
164 390
165 391
166 static ngx_uint_t 392 static ngx_uint_t
167 ngx_quic_get_padding_level(ngx_connection_t *c) 393 ngx_quic_get_padding_level(ngx_connection_t *c)
168 { 394 {