Mercurial > hg > nginx
annotate src/os/unix/ngx_linux_aio_read.c @ 7948:a2613fc1bce5
Fixed sendfile() limit handling on Linux.
On Linux starting with 2.6.16, sendfile() silently limits all operations
to MAX_RW_COUNT, defined as (INT_MAX & PAGE_MASK). This incorrectly
triggered the interrupt check, and resulted in 0-sized writev() on the
next loop iteration.
Fix is to make sure the limit is always checked, so we will return from
the loop if the limit is already reached even if number of bytes sent is
not exactly equal to the number of bytes we've tried to send.
author | Maxim Dounin <mdounin@mdounin.ru> |
---|---|
date | Fri, 29 Oct 2021 20:21:51 +0300 |
parents | f01ab2dbcfdc |
children |
rev | line source |
---|---|
3052 | 1 |
2 /* | |
3 * Copyright (C) Igor Sysoev | |
4412 | 4 * Copyright (C) Nginx, Inc. |
3052 | 5 */ |
6 | |
7 | |
8 #include <ngx_config.h> | |
9 #include <ngx_core.h> | |
10 #include <ngx_event.h> | |
11 | |
12 | |
13 extern int ngx_eventfd; | |
14 extern aio_context_t ngx_aio_ctx; | |
15 | |
16 | |
17 static void ngx_file_aio_event_handler(ngx_event_t *ev); | |
18 | |
19 | |
4129
6903dac6ad19
Fixing Linux AIO syscalls return value handling:
Igor Sysoev <igor@sysoev.ru>
parents:
4076
diff
changeset
|
20 static int |
3052 | 21 io_submit(aio_context_t ctx, long n, struct iocb **paiocb) |
22 { | |
23 return syscall(SYS_io_submit, ctx, n, paiocb); | |
24 } | |
25 | |
26 | |
5980
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
27 ngx_int_t |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
28 ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool) |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
29 { |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
30 ngx_event_aio_t *aio; |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
31 |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
32 aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t)); |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
33 if (aio == NULL) { |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
34 return NGX_ERROR; |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
35 } |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
36 |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
37 aio->file = file; |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
38 aio->fd = file->fd; |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
39 aio->event.data = aio; |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
40 aio->event.ready = 1; |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
41 aio->event.log = file->log; |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
42 |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
43 file->aio = aio; |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
44 |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
45 return NGX_OK; |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
46 } |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
47 |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
48 |
3052 | 49 ssize_t |
50 ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset, | |
51 ngx_pool_t *pool) | |
52 { | |
4129
6903dac6ad19
Fixing Linux AIO syscalls return value handling:
Igor Sysoev <igor@sysoev.ru>
parents:
4076
diff
changeset
|
53 ngx_err_t err; |
3294
04cfc09b8b8d
export aio presence knowledge to prevent using "aio sendfile",
Igor Sysoev <igor@sysoev.ru>
parents:
3052
diff
changeset
|
54 struct iocb *piocb[1]; |
04cfc09b8b8d
export aio presence knowledge to prevent using "aio sendfile",
Igor Sysoev <igor@sysoev.ru>
parents:
3052
diff
changeset
|
55 ngx_event_t *ev; |
04cfc09b8b8d
export aio presence knowledge to prevent using "aio sendfile",
Igor Sysoev <igor@sysoev.ru>
parents:
3052
diff
changeset
|
56 ngx_event_aio_t *aio; |
3052 | 57 |
3294
04cfc09b8b8d
export aio presence knowledge to prevent using "aio sendfile",
Igor Sysoev <igor@sysoev.ru>
parents:
3052
diff
changeset
|
58 if (!ngx_file_aio) { |
3052 | 59 return ngx_read_file(file, buf, size, offset); |
60 } | |
61 | |
5980
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
62 if (file->aio == NULL && ngx_file_aio_init(file, pool) != NGX_OK) { |
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
63 return NGX_ERROR; |
3052 | 64 } |
65 | |
5980
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4412
diff
changeset
|
66 aio = file->aio; |
3052 | 67 ev = &aio->event; |
68 | |
69 if (!ev->ready) { | |
70 ngx_log_error(NGX_LOG_ALERT, file->log, 0, | |
71 "second aio post for \"%V\"", &file->name); | |
72 return NGX_AGAIN; | |
73 } | |
74 | |
75 ngx_log_debug4(NGX_LOG_DEBUG_CORE, file->log, 0, | |
6480 | 76 "aio complete:%d @%O:%uz %V", |
3052 | 77 ev->complete, offset, size, &file->name); |
78 | |
79 if (ev->complete) { | |
80 ev->active = 0; | |
81 ev->complete = 0; | |
82 | |
83 if (aio->res >= 0) { | |
84 ngx_set_errno(0); | |
85 return aio->res; | |
86 } | |
87 | |
88 ngx_set_errno(-aio->res); | |
4076
37da005a5808
Bugfix: open_file_cache lost is_directio flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
3777
diff
changeset
|
89 |
37da005a5808
Bugfix: open_file_cache lost is_directio flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
3777
diff
changeset
|
90 ngx_log_error(NGX_LOG_CRIT, file->log, ngx_errno, |
37da005a5808
Bugfix: open_file_cache lost is_directio flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
3777
diff
changeset
|
91 "aio read \"%s\" failed", file->name.data); |
37da005a5808
Bugfix: open_file_cache lost is_directio flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
3777
diff
changeset
|
92 |
3052 | 93 return NGX_ERROR; |
94 } | |
95 | |
96 ngx_memzero(&aio->aiocb, sizeof(struct iocb)); | |
97 | |
98 aio->aiocb.aio_data = (uint64_t) (uintptr_t) ev; | |
99 aio->aiocb.aio_lio_opcode = IOCB_CMD_PREAD; | |
100 aio->aiocb.aio_fildes = file->fd; | |
101 aio->aiocb.aio_buf = (uint64_t) (uintptr_t) buf; | |
102 aio->aiocb.aio_nbytes = size; | |
103 aio->aiocb.aio_offset = offset; | |
104 aio->aiocb.aio_flags = IOCB_FLAG_RESFD; | |
105 aio->aiocb.aio_resfd = ngx_eventfd; | |
106 | |
107 ev->handler = ngx_file_aio_event_handler; | |
108 | |
109 piocb[0] = &aio->aiocb; | |
110 | |
4129
6903dac6ad19
Fixing Linux AIO syscalls return value handling:
Igor Sysoev <igor@sysoev.ru>
parents:
4076
diff
changeset
|
111 if (io_submit(ngx_aio_ctx, 1, piocb) == 1) { |
3777
cd04f652478c
update event flags after successful io_submit()
Igor Sysoev <igor@sysoev.ru>
parents:
3294
diff
changeset
|
112 ev->active = 1; |
cd04f652478c
update event flags after successful io_submit()
Igor Sysoev <igor@sysoev.ru>
parents:
3294
diff
changeset
|
113 ev->ready = 0; |
cd04f652478c
update event flags after successful io_submit()
Igor Sysoev <igor@sysoev.ru>
parents:
3294
diff
changeset
|
114 ev->complete = 0; |
cd04f652478c
update event flags after successful io_submit()
Igor Sysoev <igor@sysoev.ru>
parents:
3294
diff
changeset
|
115 |
3052 | 116 return NGX_AGAIN; |
117 } | |
118 | |
4129
6903dac6ad19
Fixing Linux AIO syscalls return value handling:
Igor Sysoev <igor@sysoev.ru>
parents:
4076
diff
changeset
|
119 err = ngx_errno; |
3052 | 120 |
4129
6903dac6ad19
Fixing Linux AIO syscalls return value handling:
Igor Sysoev <igor@sysoev.ru>
parents:
4076
diff
changeset
|
121 if (err == NGX_EAGAIN) { |
3052 | 122 return ngx_read_file(file, buf, size, offset); |
123 } | |
124 | |
4129
6903dac6ad19
Fixing Linux AIO syscalls return value handling:
Igor Sysoev <igor@sysoev.ru>
parents:
4076
diff
changeset
|
125 ngx_log_error(NGX_LOG_CRIT, file->log, err, |
3052 | 126 "io_submit(\"%V\") failed", &file->name); |
127 | |
4129
6903dac6ad19
Fixing Linux AIO syscalls return value handling:
Igor Sysoev <igor@sysoev.ru>
parents:
4076
diff
changeset
|
128 if (err == NGX_ENOSYS) { |
3294
04cfc09b8b8d
export aio presence knowledge to prevent using "aio sendfile",
Igor Sysoev <igor@sysoev.ru>
parents:
3052
diff
changeset
|
129 ngx_file_aio = 0; |
3052 | 130 return ngx_read_file(file, buf, size, offset); |
131 } | |
132 | |
133 return NGX_ERROR; | |
134 } | |
135 | |
136 | |
137 static void | |
138 ngx_file_aio_event_handler(ngx_event_t *ev) | |
139 { | |
140 ngx_event_aio_t *aio; | |
141 | |
142 aio = ev->data; | |
143 | |
144 ngx_log_debug2(NGX_LOG_DEBUG_CORE, ev->log, 0, | |
145 "aio event handler fd:%d %V", aio->fd, &aio->file->name); | |
146 | |
147 aio->handler(ev); | |
148 } |