Mercurial > hg > nginx
changeset 5913:8e903522c17a
Introduced the ngx_output_chain_to_iovec() function.
It deduplicates code of the send chain functions and uses only preallocated
memory, which completely solves the problem mentioned in d1bde5c3c5d2.
| author | Valentin Bartenev <vbart@nginx.com> |
|---|---|
| date | Tue, 07 Oct 2014 11:38:57 +0400 |
| parents | de68ed551bfb |
| children | 4dd67e5d958e |
| files | src/os/unix/ngx_darwin_sendfile_chain.c src/os/unix/ngx_freebsd_sendfile_chain.c src/os/unix/ngx_linux_sendfile_chain.c src/os/unix/ngx_os.h src/os/unix/ngx_writev_chain.c |
| diffstat | 5 files changed, 171 insertions(+), 319 deletions(-) [+] |
line wrap: on
line diff
--- a/src/os/unix/ngx_darwin_sendfile_chain.c +++ b/src/os/unix/ngx_darwin_sendfile_chain.c @@ -31,17 +31,15 @@ ngx_chain_t * ngx_darwin_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) { int rc; - u_char *prev; off_t size, send, prev_send, aligned, sent, fprev; - off_t header_size, file_size; + off_t file_size; ngx_uint_t eintr; ngx_err_t err; ngx_buf_t *file; - ngx_array_t header, trailer; ngx_event_t *wev; ngx_chain_t *cl; + ngx_iovec_t header, trailer; struct sf_hdtr hdtr; - struct iovec *iov; struct iovec headers[NGX_IOVS_PREALLOCATE]; struct iovec trailers[NGX_IOVS_PREALLOCATE]; @@ -70,69 +68,27 @@ ngx_darwin_sendfile_chain(ngx_connection send = 0; - header.elts = headers; - header.size = sizeof(struct iovec); + header.iovs = headers; header.nalloc = NGX_IOVS_PREALLOCATE; - header.pool = c->pool; - trailer.elts = trailers; - trailer.size = sizeof(struct iovec); + trailer.iovs = trailers; trailer.nalloc = NGX_IOVS_PREALLOCATE; - trailer.pool = c->pool; for ( ;; ) { file = NULL; file_size = 0; - header_size = 0; eintr = 0; prev_send = send; - header.nelts = 0; - trailer.nelts = 0; - /* create the header iovec and coalesce the neighbouring bufs */ - prev = NULL; - iov = NULL; - - for (cl = in; cl && send < limit; cl = cl->next) { - - if (ngx_buf_special(cl->buf)) { - continue; - } - - if (!ngx_buf_in_memory_only(cl->buf)) { - break; - } - - size = cl->buf->last - cl->buf->pos; - - if (send + size > limit) { - size = limit - send; - } + cl = ngx_output_chain_to_iovec(&header, in, limit - send, c->log); - if (prev == cl->buf->pos) { - iov->iov_len += (size_t) size; - - } else { - if (header.nelts >= IOV_MAX) { - break; - } - - iov = ngx_array_push(&header); - if (iov == NULL) { - return NGX_CHAIN_ERROR; - } - - iov->iov_base = (void *) cl->buf->pos; - iov->iov_len = (size_t) size; - } - - prev = cl->buf->pos + (size_t) size; - header_size += size; - send += size; + if (cl == NGX_CHAIN_ERROR) { + return NGX_CHAIN_ERROR; } + send += header.size; if (cl && cl->buf->in_file && send < limit) { file = cl->buf; @@ -165,51 +121,17 @@ ngx_darwin_sendfile_chain(ngx_connection && fprev == cl->buf->file_pos); } - if (file && header.nelts == 0) { + if (file && header.count == 0) { /* create the trailer iovec and coalesce the neighbouring bufs */ - prev = NULL; - iov = NULL; - - while (cl && send < limit) { - - if (ngx_buf_special(cl->buf)) { - cl = cl->next; - continue; - } - - if (!ngx_buf_in_memory_only(cl->buf)) { - break; - } - - size = cl->buf->last - cl->buf->pos; - - if (send + size > limit) { - size = limit - send; - } + cl = ngx_output_chain_to_iovec(&trailer, cl, limit - send, c->log); - if (prev == cl->buf->pos) { - iov->iov_len += (size_t) size; - - } else { - if (trailer.nelts >= IOV_MAX) { - break; - } + if (cl == NGX_CHAIN_ERROR) { + return NGX_CHAIN_ERROR; + } - iov = ngx_array_push(&trailer); - if (iov == NULL) { - return NGX_CHAIN_ERROR; - } - - iov->iov_base = (void *) cl->buf->pos; - iov->iov_len = (size_t) size; - } - - prev = cl->buf->pos + (size_t) size; - send += size; - cl = cl->next; - } + send += trailer.size; } if (file) { @@ -219,16 +141,16 @@ ngx_darwin_sendfile_chain(ngx_connection * but corresponding pointer is not NULL */ - hdtr.headers = header.nelts ? (struct iovec *) header.elts: NULL; - hdtr.hdr_cnt = header.nelts; - hdtr.trailers = trailer.nelts ? (struct iovec *) trailer.elts: NULL; - hdtr.trl_cnt = trailer.nelts; + hdtr.headers = header.count ? header.iovs : NULL; + hdtr.hdr_cnt = header.count; + hdtr.trailers = trailer.count ? trailer.iovs : NULL; + hdtr.trl_cnt = trailer.count; - sent = header_size + file_size; + sent = header.size + file_size; ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, - "sendfile: @%O %O h:%O", - file->file_pos, sent, header_size); + "sendfile: @%O %O h:%uz", + file->file_pos, sent, header.size); rc = sendfile(file->file->fd, c->fd, file->file_pos, &sent, &hdtr, 0); @@ -271,13 +193,13 @@ ngx_darwin_sendfile_chain(ngx_connection ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, "sendfile: %d, @%O %O:%O", - rc, file->file_pos, sent, file_size + header_size); + rc, file->file_pos, sent, file_size + header.size); } else { - rc = writev(c->fd, header.elts, header.nelts); + rc = writev(c->fd, header.iovs, header.count); ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, - "writev: %d of %O", rc, header_size); + "writev: %d of %uz", rc, header.size); if (rc == -1) { err = ngx_errno;
--- a/src/os/unix/ngx_freebsd_sendfile_chain.c +++ b/src/os/unix/ngx_freebsd_sendfile_chain.c @@ -33,17 +33,15 @@ ngx_chain_t * ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) { int rc, flags; - u_char *prev; off_t size, send, prev_send, aligned, sent, fprev; - size_t header_size, file_size; + size_t file_size; ngx_uint_t eintr, eagain; ngx_err_t err; ngx_buf_t *file; - ngx_array_t header, trailer; ngx_event_t *wev; ngx_chain_t *cl; + ngx_iovec_t header, trailer; struct sf_hdtr hdtr; - struct iovec *iov; struct iovec headers[NGX_IOVS_PREALLOCATE]; struct iovec trailers[NGX_IOVS_PREALLOCATE]; @@ -74,69 +72,27 @@ ngx_freebsd_sendfile_chain(ngx_connectio eagain = 0; flags = 0; - header.elts = headers; - header.size = sizeof(struct iovec); + header.iovs = headers; header.nalloc = NGX_IOVS_PREALLOCATE; - header.pool = c->pool; - trailer.elts = trailers; - trailer.size = sizeof(struct iovec); + trailer.iovs = trailers; trailer.nalloc = NGX_IOVS_PREALLOCATE; - trailer.pool = c->pool; for ( ;; ) { file = NULL; file_size = 0; - header_size = 0; eintr = 0; prev_send = send; - header.nelts = 0; - trailer.nelts = 0; - /* create the header iovec and coalesce the neighbouring bufs */ - prev = NULL; - iov = NULL; - - for (cl = in; cl && send < limit; cl = cl->next) { - - if (ngx_buf_special(cl->buf)) { - continue; - } - - if (!ngx_buf_in_memory_only(cl->buf)) { - break; - } - - size = cl->buf->last - cl->buf->pos; - - if (send + size > limit) { - size = limit - send; - } + cl = ngx_output_chain_to_iovec(&header, in, limit - send, c->log); - if (prev == cl->buf->pos) { - iov->iov_len += (size_t) size; - - } else { - if (header.nelts >= IOV_MAX){ - break; - } - - iov = ngx_array_push(&header); - if (iov == NULL) { - return NGX_CHAIN_ERROR; - } - - iov->iov_base = (void *) cl->buf->pos; - iov->iov_len = (size_t) size; - } - - prev = cl->buf->pos + (size_t) size; - header_size += (size_t) size; - send += size; + if (cl == NGX_CHAIN_ERROR) { + return NGX_CHAIN_ERROR; } + send += header.size; if (cl && cl->buf->in_file && send < limit) { file = cl->buf; @@ -174,47 +130,13 @@ ngx_freebsd_sendfile_chain(ngx_connectio /* create the trailer iovec and coalesce the neighbouring bufs */ - prev = NULL; - iov = NULL; - - while (cl && send < limit) { - - if (ngx_buf_special(cl->buf)) { - cl = cl->next; - continue; - } - - if (!ngx_buf_in_memory_only(cl->buf)) { - break; - } - - size = cl->buf->last - cl->buf->pos; - - if (send + size > limit) { - size = limit - send; - } + cl = ngx_output_chain_to_iovec(&trailer, cl, limit - send, c->log); - if (prev == cl->buf->pos) { - iov->iov_len += (size_t) size; - - } else { - if (trailer.nelts >= IOV_MAX){ - break; - } + if (cl == NGX_CHAIN_ERROR) { + return NGX_CHAIN_ERROR; + } - iov = ngx_array_push(&trailer); - if (iov == NULL) { - return NGX_CHAIN_ERROR; - } - - iov->iov_base = (void *) cl->buf->pos; - iov->iov_len = (size_t) size; - } - - prev = cl->buf->pos + (size_t) size; - send += size; - cl = cl->next; - } + send += trailer.size; } if (file) { @@ -250,10 +172,10 @@ ngx_freebsd_sendfile_chain(ngx_connectio * but corresponding pointer is not NULL */ - hdtr.headers = header.nelts ? (struct iovec *) header.elts: NULL; - hdtr.hdr_cnt = header.nelts; - hdtr.trailers = trailer.nelts ? (struct iovec *) trailer.elts: NULL; - hdtr.trl_cnt = trailer.nelts; + hdtr.headers = header.count ? header.iovs : NULL; + hdtr.hdr_cnt = header.count; + hdtr.trailers = trailer.count ? trailer.iovs : NULL; + hdtr.trl_cnt = trailer.count; /* * the "nbytes bug" of the old sendfile() syscall: @@ -261,7 +183,7 @@ ngx_freebsd_sendfile_chain(ngx_connectio */ if (!ngx_freebsd_sendfile_nbytes_bug) { - header_size = 0; + header.size = 0; } sent = 0; @@ -271,7 +193,7 @@ ngx_freebsd_sendfile_chain(ngx_connectio #endif rc = sendfile(file->file->fd, c->fd, file->file_pos, - file_size + header_size, &hdtr, &sent, flags); + file_size + header.size, &hdtr, &sent, flags); if (rc == -1) { err = ngx_errno; @@ -322,13 +244,13 @@ ngx_freebsd_sendfile_chain(ngx_connectio ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, "sendfile: %d, @%O %O:%uz", - rc, file->file_pos, sent, file_size + header_size); + rc, file->file_pos, sent, file_size + header.size); } else { - rc = writev(c->fd, header.elts, header.nelts); + rc = writev(c->fd, header.iovs, header.count); ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, - "writev: %d of %uz", rc, header_size); + "writev: %d of %uz", rc, header.size); if (rc == -1) { err = ngx_errno;
--- a/src/os/unix/ngx_linux_sendfile_chain.c +++ b/src/os/unix/ngx_linux_sendfile_chain.c @@ -32,15 +32,14 @@ ngx_linux_sendfile_chain(ngx_connection_ { int rc, tcp_nodelay; off_t size, send, prev_send, aligned, sent, fprev; - u_char *prev; size_t file_size; ngx_err_t err; ngx_buf_t *file; ngx_uint_t eintr; - ngx_array_t header; ngx_event_t *wev; ngx_chain_t *cl; - struct iovec *iov, headers[NGX_IOVS_PREALLOCATE]; + ngx_iovec_t header; + struct iovec headers[NGX_IOVS_PREALLOCATE]; #if (NGX_HAVE_SENDFILE64) off_t offset; #else @@ -63,10 +62,8 @@ ngx_linux_sendfile_chain(ngx_connection_ send = 0; - header.elts = headers; - header.size = sizeof(struct iovec); + header.iovs = headers; header.nalloc = NGX_IOVS_PREALLOCATE; - header.pool = c->pool; for ( ;; ) { file = NULL; @@ -74,75 +71,20 @@ ngx_linux_sendfile_chain(ngx_connection_ eintr = 0; prev_send = send; - header.nelts = 0; - - prev = NULL; - iov = NULL; - /* create the iovec and coalesce the neighbouring bufs */ - for (cl = in; cl && send < limit; cl = cl->next) { - - if (ngx_buf_special(cl->buf)) { - continue; - } - -#if 1 - if (!ngx_buf_in_memory(cl->buf) && !cl->buf->in_file) { - ngx_log_error(NGX_LOG_ALERT, c->log, 0, - "zero size buf in sendfile " - "t:%d r:%d f:%d %p %p-%p %p %O-%O", - cl->buf->temporary, - cl->buf->recycled, - cl->buf->in_file, - cl->buf->start, - cl->buf->pos, - cl->buf->last, - cl->buf->file, - cl->buf->file_pos, - cl->buf->file_last); - - ngx_debug_point(); - - return NGX_CHAIN_ERROR; - } -#endif + cl = ngx_output_chain_to_iovec(&header, in, limit - send, c->log); - if (!ngx_buf_in_memory_only(cl->buf)) { - break; - } - - size = cl->buf->last - cl->buf->pos; - - if (send + size > limit) { - size = limit - send; - } - - if (prev == cl->buf->pos) { - iov->iov_len += (size_t) size; + if (cl == NGX_CHAIN_ERROR) { + return NGX_CHAIN_ERROR; + } - } else { - if (header.nelts >= IOV_MAX) { - break; - } - - iov = ngx_array_push(&header); - if (iov == NULL) { - return NGX_CHAIN_ERROR; - } - - iov->iov_base = (void *) cl->buf->pos; - iov->iov_len = (size_t) size; - } - - prev = cl->buf->pos + (size_t) size; - send += size; - } + send += header.size; /* set TCP_CORK if there is a header before a file */ if (c->tcp_nopush == NGX_TCP_NOPUSH_UNSET - && header.nelts != 0 + && header.count != 0 && cl && cl->buf->in_file) { @@ -206,7 +148,7 @@ ngx_linux_sendfile_chain(ngx_connection_ /* get the file buf */ - if (header.nelts == 0 && cl && cl->buf->in_file && send < limit) { + if (header.count == 0 && cl && cl->buf->in_file && send < limit) { file = cl->buf; /* coalesce the neighbouring file bufs */ @@ -283,7 +225,7 @@ ngx_linux_sendfile_chain(ngx_connection_ rc, file->file_pos, sent, file_size); } else { - rc = writev(c->fd, header.elts, header.nelts); + rc = writev(c->fd, header.iovs, header.count); if (rc == -1) { err = ngx_errno;
--- a/src/os/unix/ngx_os.h +++ b/src/os/unix/ngx_os.h @@ -64,6 +64,17 @@ ngx_chain_t *ngx_aio_write_chain(ngx_con #endif +typedef struct { + struct iovec *iovs; + ngx_uint_t count; + size_t size; + ngx_uint_t nalloc; +} ngx_iovec_t; + +ngx_chain_t *ngx_output_chain_to_iovec(ngx_iovec_t *vec, ngx_chain_t *in, + size_t limit, ngx_log_t *log); + + extern ngx_os_io_t ngx_os_io; extern ngx_int_t ngx_ncpu; extern ngx_int_t ngx_max_sockets;
--- a/src/os/unix/ngx_writev_chain.c +++ b/src/os/unix/ngx_writev_chain.c @@ -13,15 +13,14 @@ ngx_chain_t * ngx_writev_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) { - u_char *prev; - ssize_t n, size, sent; + ssize_t n, sent; off_t send, prev_send; ngx_uint_t eintr; ngx_err_t err; - ngx_array_t vec; ngx_chain_t *cl; ngx_event_t *wev; - struct iovec *iov, iovs[NGX_IOVS_PREALLOCATE]; + ngx_iovec_t vec; + struct iovec iovs[NGX_IOVS_PREALLOCATE]; wev = c->write; @@ -48,61 +47,43 @@ ngx_writev_chain(ngx_connection_t *c, ng send = 0; - vec.elts = iovs; - vec.size = sizeof(struct iovec); + vec.iovs = iovs; vec.nalloc = NGX_IOVS_PREALLOCATE; - vec.pool = c->pool; for ( ;; ) { - prev = NULL; - iov = NULL; eintr = 0; prev_send = send; - vec.nelts = 0; - /* create the iovec and coalesce the neighbouring bufs */ - for (cl = in; cl && send < limit; cl = cl->next) { - - if (ngx_buf_special(cl->buf)) { - continue; - } - -#if 1 - if (!ngx_buf_in_memory(cl->buf)) { - ngx_debug_point(); - } -#endif - - size = cl->buf->last - cl->buf->pos; - - if (send + size > limit) { - size = (ssize_t) (limit - send); - } + cl = ngx_output_chain_to_iovec(&vec, in, limit - send, c->log); - if (prev == cl->buf->pos) { - iov->iov_len += size; - - } else { - if (vec.nelts >= IOV_MAX) { - break; - } - - iov = ngx_array_push(&vec); - if (iov == NULL) { - return NGX_CHAIN_ERROR; - } - - iov->iov_base = (void *) cl->buf->pos; - iov->iov_len = size; - } - - prev = cl->buf->pos + size; - send += size; + if (cl == NGX_CHAIN_ERROR) { + return NGX_CHAIN_ERROR; } - n = writev(c->fd, vec.elts, vec.nelts); + if (cl && cl->buf->in_file) { + ngx_log_error(NGX_LOG_ALERT, c->log, 0, + "file buf in writev " + "t:%d r:%d f:%d %p %p-%p %p %O-%O", + cl->buf->temporary, + cl->buf->recycled, + cl->buf->in_file, + cl->buf->start, + cl->buf->pos, + cl->buf->last, + cl->buf->file, + cl->buf->file_pos, + cl->buf->file_last); + + ngx_debug_point(); + + return NGX_CHAIN_ERROR; + } + + send += vec.size; + + n = writev(c->fd, vec.iovs, vec.count); if (n == -1) { err = ngx_errno; @@ -148,3 +129,77 @@ ngx_writev_chain(ngx_connection_t *c, ng } } } + + +ngx_chain_t * +ngx_output_chain_to_iovec(ngx_iovec_t *vec, ngx_chain_t *in, size_t limit, + ngx_log_t *log) +{ + size_t total, size; + u_char *prev; + ngx_uint_t n; + struct iovec *iov; + + iov = NULL; + prev = NULL; + total = 0; + n = 0; + + for ( /* void */ ; in && total < limit; in = in->next) { + + if (ngx_buf_special(in->buf)) { + continue; + } + + if (in->buf->in_file) { + break; + } + + if (!ngx_buf_in_memory(in->buf)) { + ngx_log_error(NGX_LOG_ALERT, log, 0, + "bad buf in output chain " + "t:%d r:%d f:%d %p %p-%p %p %O-%O", + in->buf->temporary, + in->buf->recycled, + in->buf->in_file, + in->buf->start, + in->buf->pos, + in->buf->last, + in->buf->file, + in->buf->file_pos, + in->buf->file_last); + + ngx_debug_point(); + + return NGX_CHAIN_ERROR; + } + + size = in->buf->last - in->buf->pos; + + if (size > limit - total) { + size = limit - total; + } + + if (prev == in->buf->pos) { + iov->iov_len += size; + + } else { + if (n == vec->nalloc) { + break; + } + + iov = &vec->iovs[n++]; + + iov->iov_base = (void *) in->buf->pos; + iov->iov_len = size; + } + + prev = in->buf->pos + size; + total += size; + } + + vec->count = n; + vec->size = total; + + return in; +}