Mercurial > hg > nginx
annotate src/os/unix/ngx_file_aio_read.c @ 5980:ccad84a174e0
Refactored sendfile() AIO preload.
This reduces layering violation and simplifies the logic of AIO preread, since
it's now triggered by the send chain function itself without falling back to
the copy filter. The context of AIO operation is now stored per file buffer,
which makes it possible to properly handle cases when multiple buffers come
from different locations, each with its own configuration.
| author | Valentin Bartenev <vbart@nginx.com> |
|---|---|
| date | Wed, 11 Feb 2015 17:52:15 +0300 |
| parents | 30eff7580d0c |
| children | f01ab2dbcfdc |
| rev | line source |
|---|---|
| 3052 | 1 |
| 2 /* | |
| 3 * Copyright (C) Igor Sysoev | |
| 4412 | 4 * Copyright (C) Nginx, Inc. |
| 3052 | 5 */ |
| 6 | |
| 7 | |
| 8 #include <ngx_config.h> | |
| 9 #include <ngx_core.h> | |
| 10 #include <ngx_event.h> | |
| 11 | |
| 12 | |
| 13 /* | |
| 14 * FreeBSD file AIO features and quirks: | |
| 15 * | |
| 16 * if an asked data are already in VM cache, then aio_error() returns 0, | |
| 17 * and the data are already copied in buffer; | |
| 18 * | |
| 3065 | 19 * aio_read() preread in VM cache as minimum 16K (probably BKVASIZE); |
| 20 * the first AIO preload may be up to 128K; | |
| 3052 | 21 * |
| 22 * aio_read/aio_error() may return EINPROGRESS for just written data; | |
| 23 * | |
| 24 * kqueue EVFILT_AIO filter is level triggered only: an event repeats | |
| 25 * until aio_return() will be called; | |
| 26 * | |
|
4133
59b99f217c6d
Replaced "can not" with "cannot" and "could not" in a bunch of places.
Ruslan Ermilov <ru@nginx.com>
parents:
4076
diff
changeset
|
27 * aio_cancel() cannot cancel file AIO: it returns AIO_NOTCANCELED always. |
| 3052 | 28 */ |
| 29 | |
| 30 | |
| 31 extern int ngx_kqueue; | |
| 32 | |
| 33 | |
| 34 static ssize_t ngx_file_aio_result(ngx_file_t *file, ngx_event_aio_t *aio, | |
| 35 ngx_event_t *ev); | |
| 36 static void ngx_file_aio_event_handler(ngx_event_t *ev); | |
| 37 | |
| 38 | |
|
5980
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
39 ngx_int_t |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
40 ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool) |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
41 { |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
42 ngx_event_aio_t *aio; |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
43 |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
44 aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t)); |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
45 if (aio == NULL) { |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
46 return NGX_ERROR; |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
47 } |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
48 |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
49 aio->file = file; |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
50 aio->fd = file->fd; |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
51 aio->event.data = aio; |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
52 aio->event.ready = 1; |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
53 aio->event.log = file->log; |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
54 |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
55 file->aio = aio; |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
56 |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
57 return NGX_OK; |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
58 } |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
59 |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
60 |
| 3052 | 61 ssize_t |
| 62 ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset, | |
| 63 ngx_pool_t *pool) | |
| 64 { | |
|
3294
04cfc09b8b8d
export aio presence knowledge to prevent using "aio sendfile",
Igor Sysoev <igor@sysoev.ru>
parents:
3065
diff
changeset
|
65 int n; |
|
04cfc09b8b8d
export aio presence knowledge to prevent using "aio sendfile",
Igor Sysoev <igor@sysoev.ru>
parents:
3065
diff
changeset
|
66 ngx_event_t *ev; |
|
04cfc09b8b8d
export aio presence knowledge to prevent using "aio sendfile",
Igor Sysoev <igor@sysoev.ru>
parents:
3065
diff
changeset
|
67 ngx_event_aio_t *aio; |
| 3052 | 68 |
|
3294
04cfc09b8b8d
export aio presence knowledge to prevent using "aio sendfile",
Igor Sysoev <igor@sysoev.ru>
parents:
3065
diff
changeset
|
69 if (!ngx_file_aio) { |
| 3052 | 70 return ngx_read_file(file, buf, size, offset); |
| 71 } | |
| 72 | |
|
5980
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
73 if (file->aio == NULL && ngx_file_aio_init(file, pool) != NGX_OK) { |
|
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
74 return NGX_ERROR; |
| 3052 | 75 } |
| 76 | |
|
5980
ccad84a174e0
Refactored sendfile() AIO preload.
Valentin Bartenev <vbart@nginx.com>
parents:
4415
diff
changeset
|
77 aio = file->aio; |
| 3052 | 78 ev = &aio->event; |
| 79 | |
| 80 if (!ev->ready) { | |
| 81 ngx_log_error(NGX_LOG_ALERT, file->log, 0, | |
| 82 "second aio post for \"%V\"", &file->name); | |
| 83 return NGX_AGAIN; | |
| 84 } | |
| 85 | |
| 86 ngx_log_debug4(NGX_LOG_DEBUG_CORE, file->log, 0, | |
| 87 "aio complete:%d @%O:%z %V", | |
| 88 ev->complete, offset, size, &file->name); | |
| 89 | |
| 90 if (ev->complete) { | |
| 91 ev->complete = 0; | |
| 92 ngx_set_errno(aio->err); | |
| 93 | |
| 94 if (aio->err == 0) { | |
| 95 return aio->nbytes; | |
| 96 } | |
| 97 | |
|
4076
37da005a5808
Bugfix: open_file_cache lost is_directio flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
3294
diff
changeset
|
98 ngx_log_error(NGX_LOG_CRIT, file->log, ngx_errno, |
|
37da005a5808
Bugfix: open_file_cache lost is_directio flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
3294
diff
changeset
|
99 "aio read \"%s\" failed", file->name.data); |
|
37da005a5808
Bugfix: open_file_cache lost is_directio flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
3294
diff
changeset
|
100 |
| 3052 | 101 return NGX_ERROR; |
| 102 } | |
| 103 | |
| 104 ngx_memzero(&aio->aiocb, sizeof(struct aiocb)); | |
| 105 | |
| 106 aio->aiocb.aio_fildes = file->fd; | |
| 107 aio->aiocb.aio_offset = offset; | |
| 108 aio->aiocb.aio_buf = buf; | |
| 109 aio->aiocb.aio_nbytes = size; | |
| 110 #if (NGX_HAVE_KQUEUE) | |
| 111 aio->aiocb.aio_sigevent.sigev_notify_kqueue = ngx_kqueue; | |
| 112 aio->aiocb.aio_sigevent.sigev_notify = SIGEV_KEVENT; | |
| 113 aio->aiocb.aio_sigevent.sigev_value.sigval_ptr = ev; | |
| 114 #endif | |
| 115 ev->handler = ngx_file_aio_event_handler; | |
| 116 | |
| 117 n = aio_read(&aio->aiocb); | |
| 118 | |
| 119 if (n == -1) { | |
| 120 n = ngx_errno; | |
| 121 | |
| 122 if (n == NGX_EAGAIN) { | |
| 123 return ngx_read_file(file, buf, size, offset); | |
| 124 } | |
| 125 | |
| 126 ngx_log_error(NGX_LOG_CRIT, file->log, n, | |
| 127 "aio_read(\"%V\") failed", &file->name); | |
| 128 | |
| 129 if (n == NGX_ENOSYS) { | |
|
3294
04cfc09b8b8d
export aio presence knowledge to prevent using "aio sendfile",
Igor Sysoev <igor@sysoev.ru>
parents:
3065
diff
changeset
|
130 ngx_file_aio = 0; |
| 3052 | 131 return ngx_read_file(file, buf, size, offset); |
| 132 } | |
| 133 | |
| 134 return NGX_ERROR; | |
| 135 } | |
| 136 | |
| 137 ngx_log_debug2(NGX_LOG_DEBUG_CORE, file->log, 0, | |
| 138 "aio_read: fd:%d %d", file->fd, n); | |
| 139 | |
| 140 ev->active = 1; | |
| 141 ev->ready = 0; | |
| 142 ev->complete = 0; | |
| 143 | |
| 144 return ngx_file_aio_result(aio->file, aio, ev); | |
| 145 } | |
| 146 | |
| 147 | |
| 148 static ssize_t | |
| 149 ngx_file_aio_result(ngx_file_t *file, ngx_event_aio_t *aio, ngx_event_t *ev) | |
| 150 { | |
| 151 int n; | |
| 152 ngx_err_t err; | |
| 153 | |
| 154 n = aio_error(&aio->aiocb); | |
| 155 | |
| 156 ngx_log_debug2(NGX_LOG_DEBUG_CORE, file->log, 0, | |
| 157 "aio_error: fd:%d %d", file->fd, n); | |
| 158 | |
| 159 if (n == -1) { | |
| 160 err = ngx_errno; | |
| 161 aio->err = err; | |
| 162 | |
| 163 ngx_log_error(NGX_LOG_ALERT, file->log, err, | |
| 164 "aio_error(\"%V\") failed", &file->name); | |
| 165 return NGX_ERROR; | |
| 166 } | |
| 167 | |
|
4415
30eff7580d0c
Fixed AIO error handling on FreeBSD.
Maxim Dounin <mdounin@mdounin.ru>
parents:
4412
diff
changeset
|
168 if (n == NGX_EINPROGRESS) { |
|
30eff7580d0c
Fixed AIO error handling on FreeBSD.
Maxim Dounin <mdounin@mdounin.ru>
parents:
4412
diff
changeset
|
169 if (ev->ready) { |
|
30eff7580d0c
Fixed AIO error handling on FreeBSD.
Maxim Dounin <mdounin@mdounin.ru>
parents:
4412
diff
changeset
|
170 ev->ready = 0; |
|
30eff7580d0c
Fixed AIO error handling on FreeBSD.
Maxim Dounin <mdounin@mdounin.ru>
parents:
4412
diff
changeset
|
171 ngx_log_error(NGX_LOG_ALERT, file->log, n, |
|
30eff7580d0c
Fixed AIO error handling on FreeBSD.
Maxim Dounin <mdounin@mdounin.ru>
parents:
4412
diff
changeset
|
172 "aio_read(\"%V\") still in progress", |
|
30eff7580d0c
Fixed AIO error handling on FreeBSD.
Maxim Dounin <mdounin@mdounin.ru>
parents:
4412
diff
changeset
|
173 &file->name); |
| 3052 | 174 } |
| 175 | |
|
4415
30eff7580d0c
Fixed AIO error handling on FreeBSD.
Maxim Dounin <mdounin@mdounin.ru>
parents:
4412
diff
changeset
|
176 return NGX_AGAIN; |
| 3052 | 177 } |
| 178 | |
| 179 n = aio_return(&aio->aiocb); | |
| 180 | |
| 181 if (n == -1) { | |
| 182 err = ngx_errno; | |
| 183 aio->err = err; | |
|
4415
30eff7580d0c
Fixed AIO error handling on FreeBSD.
Maxim Dounin <mdounin@mdounin.ru>
parents:
4412
diff
changeset
|
184 ev->ready = 1; |
| 3052 | 185 |
|
4415
30eff7580d0c
Fixed AIO error handling on FreeBSD.
Maxim Dounin <mdounin@mdounin.ru>
parents:
4412
diff
changeset
|
186 ngx_log_error(NGX_LOG_CRIT, file->log, err, |
| 3052 | 187 "aio_return(\"%V\") failed", &file->name); |
| 188 return NGX_ERROR; | |
| 189 } | |
| 190 | |
| 191 aio->err = 0; | |
| 192 aio->nbytes = n; | |
| 193 ev->ready = 1; | |
| 194 ev->active = 0; | |
| 195 | |
| 196 ngx_log_debug2(NGX_LOG_DEBUG_CORE, file->log, 0, | |
| 197 "aio_return: fd:%d %d", file->fd, n); | |
| 198 | |
| 199 return n; | |
| 200 } | |
| 201 | |
| 202 | |
| 203 static void | |
| 204 ngx_file_aio_event_handler(ngx_event_t *ev) | |
| 205 { | |
| 206 ngx_event_aio_t *aio; | |
| 207 | |
| 208 aio = ev->data; | |
| 209 | |
| 210 ngx_log_debug2(NGX_LOG_DEBUG_CORE, ev->log, 0, | |
| 211 "aio event handler fd:%d %V", aio->fd, &aio->file->name); | |
| 212 | |
| 213 if (ngx_file_aio_result(aio->file, aio, ev) != NGX_AGAIN) { | |
| 214 aio->handler(ev); | |
| 215 } | |
| 216 } |