Mercurial > hg > nginx

changeset 5980:ccad84a174e0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Refactored sendfile() AIO preload. This reduces layering violation and simplifies the logic of AIO preread, since it's now triggered by the send chain function itself without falling back to the copy filter. The context of AIO operation is now stored per file buffer, which makes it possible to properly handle cases when multiple buffers come from different locations, each with its own configuration.
author Valentin Bartenev <vbart@nginx.com>
date Wed, 11 Feb 2015 17:52:15 +0300
parents b2920b517490
children 0f234ee664f7
files src/core/ngx_buf.h src/core/ngx_connection.h src/core/ngx_output_chain.c src/event/ngx_event.h src/http/ngx_http_copy_filter_module.c src/http/ngx_http_request.h src/os/unix/ngx_file_aio_read.c src/os/unix/ngx_files.h src/os/unix/ngx_freebsd_sendfile_chain.c src/os/unix/ngx_linux_aio_read.c
diffstat 10 files changed, 187 insertions(+), 119 deletions(-) [+]
line wrap: on
line diff
--- a/src/core/ngx_buf.h
+++ b/src/core/ngx_buf.h
@@ -94,6 +94,9 @@ struct ngx_output_chain_ctx_s {
     unsigned                     aio:1;
 
     ngx_output_chain_aio_pt      aio_handler;
+#if (NGX_HAVE_FILE_AIO)
+    ssize_t                    (*aio_preload)(ngx_buf_t *file);
+#endif
 #endif
 
     off_t                        alignment;
--- a/src/core/ngx_connection.h
+++ b/src/core/ngx_connection.h
@@ -181,9 +181,7 @@ struct ngx_connection_s {
 #endif
 
 #if (NGX_HAVE_AIO_SENDFILE)
-    unsigned            aio_sendfile:1;
     unsigned            busy_count:2;
-    ngx_buf_t          *busy_sendfile;
 #endif
 
 #if (NGX_THREADS)
--- a/src/core/ngx_output_chain.c
+++ b/src/core/ngx_output_chain.c
@@ -29,6 +29,10 @@
 
 static ngx_inline ngx_int_t
     ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf);
+#if (NGX_HAVE_AIO_SENDFILE)
+static ngx_int_t ngx_output_chain_aio_setup(ngx_output_chain_ctx_t *ctx,
+    ngx_file_t *file);
+#endif
 static ngx_int_t ngx_output_chain_add_copy(ngx_pool_t *pool,
     ngx_chain_t **chain, ngx_chain_t *in);
 static ngx_int_t ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx,
@@ -252,6 +256,12 @@ ngx_output_chain_as_is(ngx_output_chain_
         buf->in_file = 0;
     }
 
+#if (NGX_HAVE_AIO_SENDFILE)
+    if (ctx->aio_preload && buf->in_file) {
+        (void) ngx_output_chain_aio_setup(ctx, buf->file);
+    }
+#endif
+
     if (ctx->need_in_memory && !ngx_buf_in_memory(buf)) {
         return 0;
     }
@@ -264,6 +274,28 @@ ngx_output_chain_as_is(ngx_output_chain_
 }
 
 
+#if (NGX_HAVE_AIO_SENDFILE)
+
+static ngx_int_t
+ngx_output_chain_aio_setup(ngx_output_chain_ctx_t *ctx, ngx_file_t *file)
+{
+    ngx_event_aio_t  *aio;
+
+    if (file->aio == NULL && ngx_file_aio_init(file, ctx->pool) != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    aio = file->aio;
+
+    aio->data = ctx->filter_ctx;
+    aio->preload_handler = ctx->aio_preload;
+
+    return NGX_OK;
+}
+
+#endif
+
+
 static ngx_int_t
 ngx_output_chain_add_copy(ngx_pool_t *pool, ngx_chain_t **chain,
     ngx_chain_t *in)
--- a/src/event/ngx_event.h
+++ b/src/event/ngx_event.h
@@ -168,6 +168,10 @@ struct ngx_event_aio_s {
     ngx_event_handler_pt       handler;
     ngx_file_t                *file;
 
+#if (NGX_HAVE_AIO_SENDFILE)
+    ssize_t                  (*preload_handler)(ngx_buf_t *file);
+#endif
+
     ngx_fd_t                   fd;
 
 #if (NGX_HAVE_EVENTFD)
@@ -181,10 +185,6 @@ struct ngx_event_aio_s {
     size_t                     nbytes;
 #endif
 
-#if (NGX_HAVE_AIO_SENDFILE)
-    off_t                      last_offset;
-#endif
-
     ngx_aiocb_t                aiocb;
     ngx_event_t                event;
 };
--- a/src/http/ngx_http_copy_filter_module.c
+++ b/src/http/ngx_http_copy_filter_module.c
@@ -20,6 +20,7 @@ static void ngx_http_copy_aio_handler(ng
     ngx_file_t *file);
 static void ngx_http_copy_aio_event_handler(ngx_event_t *ev);
 #if (NGX_HAVE_AIO_SENDFILE)
+static ssize_t ngx_http_copy_aio_sendfile_preload(ngx_buf_t *file);
 static void ngx_http_copy_aio_sendfile_event_handler(ngx_event_t *ev);
 #endif
 #endif
@@ -125,7 +126,9 @@ ngx_http_copy_filter(ngx_http_request_t 
                 ctx->aio_handler = ngx_http_copy_aio_handler;
             }
 #if (NGX_HAVE_AIO_SENDFILE)
-            c->aio_sendfile = (clcf->aio == NGX_HTTP_AIO_SENDFILE);
+            if (clcf->aio == NGX_HTTP_AIO_SENDFILE) {
+                ctx->aio_preload = ngx_http_copy_aio_sendfile_preload;
+            }
 #endif
         }
 #endif
@@ -139,72 +142,19 @@ ngx_http_copy_filter(ngx_http_request_t 
     ctx->aio = r->aio;
 #endif
 
-    for ( ;; ) {
-        rc = ngx_output_chain(ctx, in);
-
-        if (ctx->in == NULL) {
-            r->buffered &= ~NGX_HTTP_COPY_BUFFERED;
-
-        } else {
-            r->buffered |= NGX_HTTP_COPY_BUFFERED;
-        }
-
-        ngx_log_debug3(NGX_LOG_DEBUG_HTTP, c->log, 0,
-                       "http copy filter: %i \"%V?%V\"", rc, &r->uri, &r->args);
+    rc = ngx_output_chain(ctx, in);
 
-#if (NGX_HAVE_FILE_AIO && NGX_HAVE_AIO_SENDFILE)
-
-        if (c->busy_sendfile) {
-            ssize_t                n;
-            off_t                  offset;
-            ngx_file_t            *file;
-            ngx_http_ephemeral_t  *e;
-
-            if (r->aio) {
-                c->busy_sendfile = NULL;
-                return rc;
-            }
-
-            file = c->busy_sendfile->file;
-            offset = c->busy_sendfile->file_pos;
+    if (ctx->in == NULL) {
+        r->buffered &= ~NGX_HTTP_COPY_BUFFERED;
 
-            if (file->aio) {
-                c->busy_count = (offset == file->aio->last_offset) ?
-                                c->busy_count + 1 : 0;
-                file->aio->last_offset = offset;
-
-                if (c->busy_count > 2) {
-                    ngx_log_error(NGX_LOG_ALERT, c->log, 0,
-                                  "sendfile(%V) returned busy again",
-                                  &file->name);
-                    c->aio_sendfile = 0;
-                }
-            }
-
-            c->busy_sendfile = NULL;
-            e = (ngx_http_ephemeral_t *) &r->uri_start;
-
-            n = ngx_file_aio_read(file, &e->aio_preload, 1, offset, r->pool);
+    } else {
+        r->buffered |= NGX_HTTP_COPY_BUFFERED;
+    }
 
-            if (n > 0) {
-                in = NULL;
-                continue;
-            }
-
-            rc = n;
+    ngx_log_debug3(NGX_LOG_DEBUG_HTTP, c->log, 0,
+                   "http copy filter: %i \"%V?%V\"", rc, &r->uri, &r->args);
 
-            if (rc == NGX_AGAIN) {
-                file->aio->data = r;
-                file->aio->handler = ngx_http_copy_aio_sendfile_event_handler;
-
-                r->main->blocked++;
-                r->aio = 1;
-            }
-        }
-#endif
-
-        return rc;
-    }
+    return rc;
 }
 
 
@@ -244,6 +194,29 @@ ngx_http_copy_aio_event_handler(ngx_even
 
 #if (NGX_HAVE_AIO_SENDFILE)
 
+static ssize_t
+ngx_http_copy_aio_sendfile_preload(ngx_buf_t *file)
+{
+    ssize_t              n;
+    static u_char        buf[1];
+    ngx_event_aio_t     *aio;
+    ngx_http_request_t  *r;
+
+    n = ngx_file_aio_read(file->file, buf, 1, file->file_pos, NULL);
+
+    if (n == NGX_AGAIN) {
+        aio = file->file->aio;
+        aio->handler = ngx_http_copy_aio_sendfile_event_handler;
+
+        r = aio->data;
+        r->main->blocked++;
+        r->aio = 1;
+    }
+
+    return n;
+}
+
+
 static void
 ngx_http_copy_aio_sendfile_event_handler(ngx_event_t *ev)
 {
--- a/src/http/ngx_http_request.h
+++ b/src/http/ngx_http_request.h
@@ -574,9 +574,6 @@ struct ngx_http_request_s {
 
 typedef struct {
     ngx_http_posted_request_t         terminal_posted_request;
-#if (NGX_HAVE_AIO_SENDFILE)
-    u_char                            aio_preload;
-#endif
 } ngx_http_ephemeral_t;
 
 
--- a/src/os/unix/ngx_file_aio_read.c
+++ b/src/os/unix/ngx_file_aio_read.c
@@ -36,6 +36,28 @@ static ssize_t ngx_file_aio_result(ngx_f
 static void ngx_file_aio_event_handler(ngx_event_t *ev);
 
 
+ngx_int_t
+ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool)
+{
+    ngx_event_aio_t  *aio;
+
+    aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t));
+    if (aio == NULL) {
+        return NGX_ERROR;
+    }
+
+    aio->file = file;
+    aio->fd = file->fd;
+    aio->event.data = aio;
+    aio->event.ready = 1;
+    aio->event.log = file->log;
+
+    file->aio = aio;
+
+    return NGX_OK;
+}
+
+
 ssize_t
 ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset,
     ngx_pool_t *pool)
@@ -48,25 +70,11 @@ ngx_file_aio_read(ngx_file_t *file, u_ch
         return ngx_read_file(file, buf, size, offset);
     }
 
-    aio = file->aio;
-
-    if (aio == NULL) {
-        aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t));
-        if (aio == NULL) {
-            return NGX_ERROR;
-        }
-
-        aio->file = file;
-        aio->fd = file->fd;
-        aio->event.data = aio;
-        aio->event.ready = 1;
-        aio->event.log = file->log;
-#if (NGX_HAVE_AIO_SENDFILE)
-        aio->last_offset = -1;
-#endif
-        file->aio = aio;
+    if (file->aio == NULL && ngx_file_aio_init(file, pool) != NGX_OK) {
+        return NGX_ERROR;
     }
 
+    aio = file->aio;
     ev = &aio->event;
 
     if (!ev->ready) {
--- a/src/os/unix/ngx_files.h
+++ b/src/os/unix/ngx_files.h
@@ -375,6 +375,7 @@ size_t ngx_fs_bsize(u_char *name);
 
 #if (NGX_HAVE_FILE_AIO)
 
+ngx_int_t ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool);
 ssize_t ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size,
     off_t offset, ngx_pool_t *pool);
 
--- a/src/os/unix/ngx_freebsd_sendfile_chain.c
+++ b/src/os/unix/ngx_freebsd_sendfile_chain.c
@@ -32,19 +32,23 @@
 ngx_chain_t *
 ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
 {
-    int              rc, flags;
-    off_t            send, prev_send, sent;
-    size_t           file_size;
-    ssize_t          n;
-    ngx_uint_t       eintr, eagain;
-    ngx_err_t        err;
-    ngx_buf_t       *file;
-    ngx_event_t     *wev;
-    ngx_chain_t     *cl;
-    ngx_iovec_t      header, trailer;
-    struct sf_hdtr   hdtr;
-    struct iovec     headers[NGX_IOVS_PREALLOCATE];
-    struct iovec     trailers[NGX_IOVS_PREALLOCATE];
+    int               rc, flags;
+    off_t             send, prev_send, sent;
+    size_t            file_size;
+    ssize_t           n;
+    ngx_uint_t        eintr, eagain;
+    ngx_err_t         err;
+    ngx_buf_t        *file;
+    ngx_event_t      *wev;
+    ngx_chain_t      *cl;
+    ngx_iovec_t       header, trailer;
+    struct sf_hdtr    hdtr;
+    struct iovec      headers[NGX_IOVS_PREALLOCATE];
+    struct iovec      trailers[NGX_IOVS_PREALLOCATE];
+#if (NGX_HAVE_AIO_SENDFILE)
+    ngx_uint_t        ebusy;
+    ngx_event_aio_t  *aio;
+#endif
 
     wev = c->write;
 
@@ -73,6 +77,11 @@ ngx_freebsd_sendfile_chain(ngx_connectio
     eagain = 0;
     flags = 0;
 
+#if (NGX_HAVE_AIO_SENDFILE && NGX_SUPPRESS_WARN)
+    aio = NULL;
+    file = NULL;
+#endif
+
     header.iovs = headers;
     header.nalloc = NGX_IOVS_PREALLOCATE;
 
@@ -81,6 +90,9 @@ ngx_freebsd_sendfile_chain(ngx_connectio
 
     for ( ;; ) {
         eintr = 0;
+#if (NGX_HAVE_AIO_SENDFILE)
+        ebusy = 0;
+#endif
         prev_send = send;
 
         /* create the header iovec and coalesce the neighbouring bufs */
@@ -160,7 +172,8 @@ ngx_freebsd_sendfile_chain(ngx_connectio
             sent = 0;
 
 #if (NGX_HAVE_AIO_SENDFILE)
-            flags = c->aio_sendfile ? SF_NODISKIO : 0;
+            aio = file->file->aio;
+            flags = (aio && aio->preload_handler) ? SF_NODISKIO : 0;
 #endif
 
             rc = sendfile(file->file->fd, c->fd, file->file_pos,
@@ -180,7 +193,7 @@ ngx_freebsd_sendfile_chain(ngx_connectio
 
 #if (NGX_HAVE_AIO_SENDFILE)
                 case NGX_EBUSY:
-                    c->busy_sendfile = file;
+                    ebusy = 1;
                     break;
 #endif
 
@@ -232,9 +245,41 @@ ngx_freebsd_sendfile_chain(ngx_connectio
         in = ngx_chain_update_sent(in, sent);
 
 #if (NGX_HAVE_AIO_SENDFILE)
-        if (c->busy_sendfile) {
+
+        if (ebusy) {
+            if (sent == 0) {
+                c->busy_count++;
+
+                if (c->busy_count > 2) {
+                    ngx_log_error(NGX_LOG_ALERT, c->log, 0,
+                                  "sendfile(%V) returned busy again",
+                                  &file->file->name);
+
+                    c->busy_count = 0;
+                    aio->preload_handler = NULL;
+
+                    send = prev_send;
+                    continue;
+                }
+
+            } else {
+                c->busy_count = 0;
+            }
+
+            rc = aio->preload_handler(file);
+
+            if (rc > 0) {
+                send = prev_send + sent;
+                continue;
+            }
+
             return in;
         }
+
+        if (flags == SF_NODISKIO) {
+            c->busy_count = 0;
+        }
+
 #endif
 
         if (eagain) {
--- a/src/os/unix/ngx_linux_aio_read.c
+++ b/src/os/unix/ngx_linux_aio_read.c
@@ -24,6 +24,28 @@ io_submit(aio_context_t ctx, long n, str
 }
 
 
+ngx_int_t
+ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool)
+{
+    ngx_event_aio_t  *aio;
+
+    aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t));
+    if (aio == NULL) {
+        return NGX_ERROR;
+    }
+
+    aio->file = file;
+    aio->fd = file->fd;
+    aio->event.data = aio;
+    aio->event.ready = 1;
+    aio->event.log = file->log;
+
+    file->aio = aio;
+
+    return NGX_OK;
+}
+
+
 ssize_t
 ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset,
     ngx_pool_t *pool)
@@ -37,22 +59,11 @@ ngx_file_aio_read(ngx_file_t *file, u_ch
         return ngx_read_file(file, buf, size, offset);
     }
 
-    aio = file->aio;
-
-    if (aio == NULL) {
-        aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t));
-        if (aio == NULL) {
-            return NGX_ERROR;
-        }
-
-        aio->file = file;
-        aio->fd = file->fd;
-        aio->event.data = aio;
-        aio->event.ready = 1;
-        aio->event.log = file->log;
-        file->aio = aio;
+    if (file->aio == NULL && ngx_file_aio_init(file, pool) != NGX_OK) {
+        return NGX_ERROR;
     }
 
+    aio = file->aio;
     ev = &aio->event;
 
     if (!ev->ready) {