Mercurial > hg > nginx
annotate src/os/unix/ngx_linux_aio_read.c @ 8782:b3f6ad181df4 quic
QUIC: refactored CRYPTO and STREAM buffer ordering.
Generic function ngx_quic_order_bufs() is introduced. This function creates
and maintains a chain of buffers with holes. Holes are marked with b->sync
flag. Several buffers and holes in this chain may share the same underlying
memory buffer.
When processing STREAM frames with this function, frame data is copied only
once to the right place in the stream input chain. Previously data could
be copied twice. First when buffering an out-of-order frame data, and then
when filling stream buffer from ordered frame queue. Now there's only one
data chain for both tasks.
author | Roman Arutyunyan <arut@nginx.com> |
---|---|
date | Tue, 25 May 2021 13:55:12 +0300 |
parents | f01ab2dbcfdc |
children |
rev | line source |
---|---|
3052 | 1 |
2 /* | |
3 * Copyright (C) Igor Sysoev | |
4412 | 4 * Copyright (C) Nginx, Inc. |
3052 | 5 */ |
6 | |
7 | |
8 #include <ngx_config.h> | |
9 #include <ngx_core.h> | |
10 #include <ngx_event.h> | |
11 | |
12 | |
13 extern int ngx_eventfd; | |
14 extern aio_context_t ngx_aio_ctx; | |
15 | |
16 | |
17 static void ngx_file_aio_event_handler(ngx_event_t *ev); | |
18 | |
19 | |
4129
6903dac6ad19
Fixing Linux AIO syscalls return value handling:
Igor Sysoev <igor@sysoev.ru>
parents:
4076
diff
changeset
|
20 static int |
3052 | 21 io_submit(aio_context_t ctx, long n, struct iocb **paiocb) |
22 { | |
23 return syscall(SYS_io_submit, ctx, n, paiocb); | |
24 } | |
25 | |
26 | |
5980
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
27 ngx_int_t |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
28 ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool) |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
29 { |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
30 ngx_event_aio_t *aio; |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
31 |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
32 aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t)); |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
33 if (aio == NULL) { |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
34 return NGX_ERROR; |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
35 } |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
36 |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
37 aio->file = file; |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
38 aio->fd = file->fd; |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
39 aio->event.data = aio; |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
40 aio->event.ready = 1; |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
41 aio->event.log = file->log; |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
42 |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
43 file->aio = aio; |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
44 |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
45 return NGX_OK; |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
46 } |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
47 |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
48 |
3052 | 49 ssize_t |
50 ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset, | |
51 ngx_pool_t *pool) | |
52 { | |
4129
6903dac6ad19
Fixing Linux AIO syscalls return value handling:
Igor Sysoev <igor@sysoev.ru>
parents:
4076
diff
changeset
|
53 ngx_err_t err; |
3294
04cfc09b8b8d
export aio presence knowledge to prevent using "aio sendfile",
Igor Sysoev <igor@sysoev.ru>
parents:
3052
diff
changeset
|
54 struct iocb *piocb[1]; |
04cfc09b8b8d
export aio presence knowledge to prevent using "aio sendfile",
Igor Sysoev <igor@sysoev.ru>
parents:
3052
diff
changeset
|
55 ngx_event_t *ev; |
04cfc09b8b8d
export aio presence knowledge to prevent using "aio sendfile",
Igor Sysoev <igor@sysoev.ru>
parents:
3052
diff
changeset
|
56 ngx_event_aio_t *aio; |
3052 | 57 |
3294
04cfc09b8b8d
export aio presence knowledge to prevent using "aio sendfile",
Igor Sysoev <igor@sysoev.ru>
parents:
3052
diff
changeset
|
58 if (!ngx_file_aio) { |
3052 | 59 return ngx_read_file(file, buf, size, offset); |
60 } | |
61 | |
5980
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
62 if (file->aio == NULL && ngx_file_aio_init(file, pool) != NGX_OK) { |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
63 return NGX_ERROR; |
3052 | 64 } |
65 | |
5980
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
66 aio = file->aio; |
3052 | 67 ev = &aio->event; |
68 | |
69 if (!ev->ready) { | |
70 ngx_log_error(NGX_LOG_ALERT, file->log, 0, | |
71 "second aio post for \"%V\"", &file->name); | |
72 return NGX_AGAIN; | |
73 } | |
74 | |
75 ngx_log_debug4(NGX_LOG_DEBUG_CORE, file->log, 0, | |
6480 | 76 "aio complete:%d @%O:%uz %V", |
3052 | 77 ev->complete, offset, size, &file->name); |
78 | |
79 if (ev->complete) { | |
80 ev->active = 0; | |
81 ev->complete = 0; | |
82 | |
83 if (aio->res >= 0) { | |
84 ngx_set_errno(0); | |
85 return aio->res; | |
86 } | |
87 | |
88 ngx_set_errno(-aio->res); | |
4076
37da005a5808
Bugfix: open_file_cache lost is_directio flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
3777
diff
changeset
|
89 |
37da005a5808
Bugfix: open_file_cache lost is_directio flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
3777
diff
changeset
|
90 ngx_log_error(NGX_LOG_CRIT, file->log, ngx_errno, |
37da005a5808
Bugfix: open_file_cache lost is_directio flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
3777
diff
changeset
|
91 "aio read \"%s\" failed", file->name.data); |
37da005a5808
Bugfix: open_file_cache lost is_directio flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
3777
diff
changeset
|
92 |
3052 | 93 return NGX_ERROR; |
94 } | |
95 | |
96 ngx_memzero(&aio->aiocb, sizeof(struct iocb)); | |
97 | |
98 aio->aiocb.aio_data = (uint64_t) (uintptr_t) ev; | |
99 aio->aiocb.aio_lio_opcode = IOCB_CMD_PREAD; | |
100 aio->aiocb.aio_fildes = file->fd; | |
101 aio->aiocb.aio_buf = (uint64_t) (uintptr_t) buf; | |
102 aio->aiocb.aio_nbytes = size; | |
103 aio->aiocb.aio_offset = offset; | |
104 aio->aiocb.aio_flags = IOCB_FLAG_RESFD; | |
105 aio->aiocb.aio_resfd = ngx_eventfd; | |
106 | |
107 ev->handler = ngx_file_aio_event_handler; | |
108 | |
109 piocb[0] = &aio->aiocb; | |
110 | |
4129
6903dac6ad19
Fixing Linux AIO syscalls return value handling:
Igor Sysoev <igor@sysoev.ru>
parents:
4076
diff
changeset
|
111 if (io_submit(ngx_aio_ctx, 1, piocb) == 1) { |
3777
cd04f652478c
update event flags after successful io_submit()
Igor Sysoev <igor@sysoev.ru>
parents:
3294
diff
changeset
|
112 ev->active = 1; |
cd04f652478c
update event flags after successful io_submit()
Igor Sysoev <igor@sysoev.ru>
parents:
3294
diff
changeset
|
113 ev->ready = 0; |
cd04f652478c
update event flags after successful io_submit()
Igor Sysoev <igor@sysoev.ru>
parents:
3294
diff
changeset
|
114 ev->complete = 0; |
cd04f652478c
update event flags after successful io_submit()
Igor Sysoev <igor@sysoev.ru>
parents:
3294
diff
changeset
|
115 |
3052 | 116 return NGX_AGAIN; |
117 } | |
118 | |
4129
6903dac6ad19
Fixing Linux AIO syscalls return value handling:
Igor Sysoev <igor@sysoev.ru>
parents:
4076
diff
changeset
|
119 err = ngx_errno; |
3052 | 120 |
4129
6903dac6ad19
Fixing Linux AIO syscalls return value handling:
Igor Sysoev <igor@sysoev.ru>
parents:
4076
diff
changeset
|
121 if (err == NGX_EAGAIN) { |
3052 | 122 return ngx_read_file(file, buf, size, offset); |
123 } | |
124 | |
4129
6903dac6ad19
Fixing Linux AIO syscalls return value handling:
Igor Sysoev <igor@sysoev.ru>
parents:
4076
diff
changeset
|
125 ngx_log_error(NGX_LOG_CRIT, file->log, err, |
3052 | 126 "io_submit(\"%V\") failed", &file->name); |
127 | |
4129
6903dac6ad19
Fixing Linux AIO syscalls return value handling:
Igor Sysoev <igor@sysoev.ru>
parents:
4076
diff
changeset
|
128 if (err == NGX_ENOSYS) { |
3294
04cfc09b8b8d
export aio presence knowledge to prevent using "aio sendfile",
Igor Sysoev <igor@sysoev.ru>
parents:
3052
diff
changeset
|
129 ngx_file_aio = 0; |
3052 | 130 return ngx_read_file(file, buf, size, offset); |
131 } | |
132 | |
133 return NGX_ERROR; | |
134 } | |
135 | |
136 | |
137 static void | |
138 ngx_file_aio_event_handler(ngx_event_t *ev) | |
139 { | |
140 ngx_event_aio_t *aio; | |
141 | |
142 aio = ev->data; | |
143 | |
144 ngx_log_debug2(NGX_LOG_DEBUG_CORE, ev->log, 0, | |
145 "aio event handler fd:%d %V", aio->fd, &aio->file->name); | |
146 | |
147 aio->handler(ev); | |
148 } |