Mercurial > hg > nginx-quic
comparison src/event/quic/ngx_event_quic_output.c @ 8512:bb5152ed045b quic
QUIC: added support for segmentation offloading.
To improve output performance, UDP segmentation offloading is used
if available. If there is a significant amount of data in an output
queue and path is verified, QUIC packets are not sent one-by-one,
but instead are collected in a buffer, which is then passed to kernel
in a single sendmsg call, using UDP GSO. Such method greatly decreases
number of system calls and thus system load.
author | Vladimir Homutov <vl@nginx.com> |
---|---|
date | Thu, 15 Jul 2021 14:22:00 +0300 |
parents | 5b0c229ba5fe |
children | 8ab0d609af09 |
comparison
equal
deleted
inserted
replaced
8511:2dfd313f22f2 | 8512:bb5152ed045b |
---|---|
14 #define NGX_QUIC_MAX_LONG_HEADER 56 | 14 #define NGX_QUIC_MAX_LONG_HEADER 56 |
15 /* 1 flags + 4 version + 2 x (1 + 20) s/dcid + 4 pn + 4 len + token len */ | 15 /* 1 flags + 4 version + 2 x (1 + 20) s/dcid + 4 pn + 4 len + token len */ |
16 | 16 |
17 #define NGX_QUIC_MAX_UDP_PAYLOAD_OUT 1252 | 17 #define NGX_QUIC_MAX_UDP_PAYLOAD_OUT 1252 |
18 #define NGX_QUIC_MAX_UDP_PAYLOAD_OUT6 1232 | 18 #define NGX_QUIC_MAX_UDP_PAYLOAD_OUT6 1232 |
19 | |
20 #define NGX_QUIC_MAX_UDP_SEGMENT_BUF 65487 /* 65K - IPv6 header */ | |
21 #define NGX_QUIC_MAX_SEGMENTS 64 /* UDP_MAX_SEGMENTS */ | |
19 | 22 |
20 #define NGX_QUIC_RETRY_TOKEN_LIFETIME 3 /* seconds */ | 23 #define NGX_QUIC_RETRY_TOKEN_LIFETIME 3 /* seconds */ |
21 #define NGX_QUIC_NEW_TOKEN_LIFETIME 600 /* seconds */ | 24 #define NGX_QUIC_NEW_TOKEN_LIFETIME 600 /* seconds */ |
22 #define NGX_QUIC_RETRY_BUFFER_SIZE 256 | 25 #define NGX_QUIC_RETRY_BUFFER_SIZE 256 |
23 /* 1 flags + 4 version + 3 x (1 + 20) s/o/dcid + itag + token(64) */ | 26 /* 1 flags + 4 version + 3 x (1 + 20) s/o/dcid + itag + token(64) */ |
37 #define NGX_QUIC_CC_MIN_INTERVAL 1000 /* 1s */ | 40 #define NGX_QUIC_CC_MIN_INTERVAL 1000 /* 1s */ |
38 | 41 |
39 | 42 |
40 static ngx_int_t ngx_quic_socket_output(ngx_connection_t *c, | 43 static ngx_int_t ngx_quic_socket_output(ngx_connection_t *c, |
41 ngx_quic_socket_t *qsock); | 44 ngx_quic_socket_t *qsock); |
45 static ngx_int_t ngx_quic_create_datagrams(ngx_connection_t *c, | |
46 ngx_quic_socket_t *qsock); | |
47 #if ((NGX_HAVE_UDP_SEGMENT) && (NGX_HAVE_MSGHDR_MSG_CONTROL)) | |
48 static ngx_uint_t ngx_quic_allow_segmentation(ngx_connection_t *c, | |
49 ngx_quic_socket_t *qsock); | |
50 static ngx_int_t ngx_quic_create_segments(ngx_connection_t *c, | |
51 ngx_quic_socket_t *qsock); | |
52 static ssize_t ngx_quic_send_segments(ngx_connection_t *c, u_char *buf, | |
53 size_t len, struct sockaddr *sockaddr, socklen_t socklen, size_t segment); | |
54 #endif | |
42 static ssize_t ngx_quic_output_packet(ngx_connection_t *c, | 55 static ssize_t ngx_quic_output_packet(ngx_connection_t *c, |
43 ngx_quic_send_ctx_t *ctx, u_char *data, size_t max, size_t min, | 56 ngx_quic_send_ctx_t *ctx, u_char *data, size_t max, size_t min, |
44 ngx_quic_socket_t *qsock); | 57 ngx_quic_socket_t *qsock); |
45 static ngx_uint_t ngx_quic_get_padding_level(ngx_connection_t *c); | 58 static ngx_uint_t ngx_quic_get_padding_level(ngx_connection_t *c); |
46 static ssize_t ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len, | 59 static ssize_t ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len, |
82 | 95 |
83 | 96 |
84 static ngx_int_t | 97 static ngx_int_t |
85 ngx_quic_socket_output(ngx_connection_t *c, ngx_quic_socket_t *qsock) | 98 ngx_quic_socket_output(ngx_connection_t *c, ngx_quic_socket_t *qsock) |
86 { | 99 { |
100 size_t in_flight; | |
101 ngx_int_t rc; | |
102 ngx_quic_congestion_t *cg; | |
103 ngx_quic_connection_t *qc; | |
104 | |
105 c->log->action = "sending frames"; | |
106 | |
107 qc = ngx_quic_get_connection(c); | |
108 cg = &qc->congestion; | |
109 | |
110 in_flight = cg->in_flight; | |
111 | |
112 #if ((NGX_HAVE_UDP_SEGMENT) && (NGX_HAVE_MSGHDR_MSG_CONTROL)) | |
113 if (ngx_quic_allow_segmentation(c, qsock)) { | |
114 rc = ngx_quic_create_segments(c, qsock); | |
115 } else | |
116 #endif | |
117 { | |
118 rc = ngx_quic_create_datagrams(c, qsock); | |
119 } | |
120 | |
121 if (rc != NGX_OK) { | |
122 return NGX_ERROR; | |
123 } | |
124 | |
125 if (in_flight != cg->in_flight && !qc->send_timer_set && !qc->closing) { | |
126 qc->send_timer_set = 1; | |
127 ngx_add_timer(c->read, qc->tp.max_idle_timeout); | |
128 } | |
129 | |
130 return NGX_OK; | |
131 } | |
132 | |
133 | |
134 static ngx_int_t | |
135 ngx_quic_create_datagrams(ngx_connection_t *c, ngx_quic_socket_t *qsock) | |
136 { | |
87 off_t max; | 137 off_t max; |
88 size_t len, min, in_flight; | 138 size_t len, min; |
89 ssize_t n; | 139 ssize_t n; |
90 u_char *p; | 140 u_char *p; |
91 ngx_uint_t i, pad; | 141 ngx_uint_t i, pad; |
92 ngx_quic_path_t *path; | 142 ngx_quic_path_t *path; |
93 ngx_quic_send_ctx_t *ctx; | 143 ngx_quic_send_ctx_t *ctx; |
94 ngx_quic_congestion_t *cg; | |
95 ngx_quic_connection_t *qc; | 144 ngx_quic_connection_t *qc; |
96 static u_char dst[NGX_QUIC_MAX_UDP_PAYLOAD_SIZE]; | 145 static u_char dst[NGX_QUIC_MAX_UDP_PAYLOAD_SIZE]; |
97 | 146 |
98 c->log->action = "sending frames"; | |
99 | |
100 qc = ngx_quic_get_connection(c); | 147 qc = ngx_quic_get_connection(c); |
101 cg = &qc->congestion; | |
102 | |
103 in_flight = cg->in_flight; | |
104 | 148 |
105 path = qsock->path; | 149 path = qsock->path; |
106 | 150 |
107 for ( ;; ) { | 151 for ( ;; ) { |
108 p = dst; | 152 p = dst; |
151 } | 195 } |
152 | 196 |
153 path->sent += len; | 197 path->sent += len; |
154 } | 198 } |
155 | 199 |
156 if (in_flight != cg->in_flight && !qc->send_timer_set && !qc->closing) { | |
157 qc->send_timer_set = 1; | |
158 ngx_add_timer(c->read, qc->tp.max_idle_timeout); | |
159 } | |
160 | |
161 | |
162 return NGX_OK; | 200 return NGX_OK; |
163 } | 201 } |
202 | |
203 | |
204 #if ((NGX_HAVE_UDP_SEGMENT) && (NGX_HAVE_MSGHDR_MSG_CONTROL)) | |
205 | |
206 static ngx_uint_t | |
207 ngx_quic_allow_segmentation(ngx_connection_t *c, ngx_quic_socket_t *qsock) | |
208 { | |
209 size_t bytes, len; | |
210 ngx_queue_t *q; | |
211 ngx_quic_frame_t *f; | |
212 ngx_quic_send_ctx_t *ctx; | |
213 ngx_quic_connection_t *qc; | |
214 | |
215 if (qsock->path->state != NGX_QUIC_PATH_VALIDATED) { | |
216 /* don't even try to be faster on non-validated paths */ | |
217 return 0; | |
218 } | |
219 | |
220 qc = ngx_quic_get_connection(c); | |
221 | |
222 ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_initial); | |
223 if (!ngx_queue_empty(&ctx->frames)) { | |
224 return 0; | |
225 } | |
226 | |
227 ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_handshake); | |
228 if (!ngx_queue_empty(&ctx->frames)) { | |
229 return 0; | |
230 } | |
231 | |
232 ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); | |
233 | |
234 bytes = 0; | |
235 | |
236 len = ngx_min(qc->ctp.max_udp_payload_size, | |
237 NGX_QUIC_MAX_UDP_SEGMENT_BUF); | |
238 | |
239 for (q = ngx_queue_head(&ctx->frames); | |
240 q != ngx_queue_sentinel(&ctx->frames); | |
241 q = ngx_queue_next(q)) | |
242 { | |
243 f = ngx_queue_data(q, ngx_quic_frame_t, queue); | |
244 | |
245 bytes += f->len; | |
246 | |
247 if (bytes > len * 3) { | |
248 /* require at least ~3 full packets to batch */ | |
249 return 1; | |
250 } | |
251 } | |
252 | |
253 return 0; | |
254 } | |
255 | |
256 | |
257 static ngx_int_t | |
258 ngx_quic_create_segments(ngx_connection_t *c, ngx_quic_socket_t *qsock) | |
259 { | |
260 size_t len, segsize; | |
261 ssize_t n; | |
262 u_char *p, *end; | |
263 ngx_uint_t nseg; | |
264 ngx_quic_send_ctx_t *ctx; | |
265 ngx_quic_path_t *path; | |
266 ngx_quic_connection_t *qc; | |
267 static u_char dst[NGX_QUIC_MAX_UDP_SEGMENT_BUF]; | |
268 | |
269 qc = ngx_quic_get_connection(c); | |
270 path = qsock->path; | |
271 | |
272 ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); | |
273 | |
274 if (ngx_quic_generate_ack(c, ctx) != NGX_OK) { | |
275 return NGX_ERROR; | |
276 } | |
277 | |
278 segsize = ngx_min(qc->ctp.max_udp_payload_size, | |
279 NGX_QUIC_MAX_UDP_SEGMENT_BUF); | |
280 p = dst; | |
281 end = dst + sizeof(dst); | |
282 | |
283 nseg = 0; | |
284 | |
285 for ( ;; ) { | |
286 | |
287 len = ngx_min(segsize, (size_t) (end - p)); | |
288 | |
289 if (len) { | |
290 | |
291 n = ngx_quic_output_packet(c, ctx, p, len, len, qsock); | |
292 if (n == NGX_ERROR) { | |
293 return NGX_ERROR; | |
294 } | |
295 | |
296 p += n; | |
297 nseg++; | |
298 | |
299 } else { | |
300 n = 0; | |
301 } | |
302 | |
303 if (p == dst) { | |
304 break; | |
305 } | |
306 | |
307 if (n == 0 || nseg == NGX_QUIC_MAX_SEGMENTS) { | |
308 n = ngx_quic_send_segments(c, dst, p - dst, path->sockaddr, | |
309 path->socklen, segsize); | |
310 if (n == NGX_ERROR) { | |
311 return NGX_ERROR; | |
312 } | |
313 | |
314 path->sent += n; | |
315 | |
316 p = dst; | |
317 nseg = 0; | |
318 } | |
319 } | |
320 | |
321 return NGX_OK; | |
322 } | |
323 | |
324 | |
325 static ssize_t | |
326 ngx_quic_send_segments(ngx_connection_t *c, u_char *buf, size_t len, | |
327 struct sockaddr *sockaddr, socklen_t socklen, size_t segment) | |
328 { | |
329 size_t clen; | |
330 ssize_t n; | |
331 uint16_t *valp; | |
332 struct iovec iov; | |
333 struct msghdr msg; | |
334 struct cmsghdr *cmsg; | |
335 | |
336 #if defined(NGX_HAVE_ADDRINFO_CMSG) | |
337 char msg_control[CMSG_SPACE(sizeof(uint16_t)) | |
338 + CMSG_SPACE(sizeof(ngx_addrinfo_t))]; | |
339 #else | |
340 char msg_control[CMSG_SPACE(sizeof(uint16_t))]; | |
341 #endif | |
342 | |
343 ngx_memzero(&msg, sizeof(struct msghdr)); | |
344 ngx_memzero(msg_control, sizeof(msg_control)); | |
345 | |
346 iov.iov_len = len; | |
347 iov.iov_base = buf; | |
348 | |
349 msg.msg_iov = &iov; | |
350 msg.msg_iovlen = 1; | |
351 | |
352 msg.msg_name = sockaddr; | |
353 msg.msg_namelen = socklen; | |
354 | |
355 msg.msg_control = msg_control; | |
356 msg.msg_controllen = sizeof(msg_control); | |
357 | |
358 cmsg = CMSG_FIRSTHDR(&msg); | |
359 | |
360 cmsg->cmsg_level = SOL_UDP; | |
361 cmsg->cmsg_type = UDP_SEGMENT; | |
362 cmsg->cmsg_len = CMSG_LEN(sizeof(uint16_t)); | |
363 | |
364 clen = CMSG_SPACE(sizeof(uint16_t)); | |
365 | |
366 valp = (void *) CMSG_DATA(cmsg); | |
367 *valp = segment; | |
368 | |
369 #if defined(NGX_HAVE_ADDRINFO_CMSG) | |
370 if (c->listening && c->listening->wildcard && c->local_sockaddr) { | |
371 cmsg = CMSG_NXTHDR(&msg, cmsg); | |
372 clen += ngx_set_srcaddr_cmsg(cmsg, c->local_sockaddr); | |
373 } | |
374 #endif | |
375 | |
376 msg.msg_controllen = clen; | |
377 | |
378 n = ngx_sendmsg(c, &msg, 0); | |
379 if (n == -1) { | |
380 return NGX_ERROR; | |
381 } | |
382 | |
383 c->sent += n; | |
384 | |
385 return n; | |
386 } | |
387 | |
388 #endif | |
389 | |
164 | 390 |
165 | 391 |
166 static ngx_uint_t | 392 static ngx_uint_t |
167 ngx_quic_get_padding_level(ngx_connection_t *c) | 393 ngx_quic_get_padding_level(ngx_connection_t *c) |
168 { | 394 { |