Mercurial > hg > nginx
diff src/http/modules/ngx_http_chunked_filter_module.c @ 4166:525a34d74a0a stable-1.0
Merging r4114, r4129:
Buffers reuse in chunked filter.
There were 2 buffers allocated on each buffer chain sent through chunked
filter (one buffer for chunk size, another one for trailing CRLF, about
120 bytes in total on 32-bit platforms). This resulted in large memory
consumption with long-lived requests sending many buffer chains. Usual
example of problematic scenario is streaming though proxy with
proxy_buffering set to off.
Introduced buffers reuse reduces memory consumption in the above problematic
scenario.
See here for initial report:
http://mailman.nginx.org/pipermail/nginx/2010-April/019814.html
author | Igor Sysoev <igor@sysoev.ru> |
---|---|
date | Fri, 30 Sep 2011 14:20:52 +0000 |
parents | 81457372d938 |
children | 4919fb357a5d |
line wrap: on
line diff
--- a/src/http/modules/ngx_http_chunked_filter_module.c +++ b/src/http/modules/ngx_http_chunked_filter_module.c @@ -9,6 +9,12 @@ #include <ngx_http.h> +typedef struct { + ngx_chain_t *free; + ngx_chain_t *busy; +} ngx_http_chunked_filter_ctx_t; + + static ngx_int_t ngx_http_chunked_filter_init(ngx_conf_t *cf); @@ -50,7 +56,8 @@ static ngx_http_output_body_filter_pt static ngx_int_t ngx_http_chunked_header_filter(ngx_http_request_t *r) { - ngx_http_core_loc_conf_t *clcf; + ngx_http_core_loc_conf_t *clcf; + ngx_http_chunked_filter_ctx_t *ctx; if (r->headers_out.status == NGX_HTTP_NOT_MODIFIED || r->headers_out.status == NGX_HTTP_NO_CONTENT @@ -70,6 +77,14 @@ ngx_http_chunked_header_filter(ngx_http_ if (clcf->chunked_transfer_encoding) { r->chunked = 1; + ctx = ngx_pcalloc(r->pool, + sizeof(ngx_http_chunked_filter_ctx_t)); + if (ctx == NULL) { + return NGX_ERROR; + } + + ngx_http_set_ctx(r, ctx, ngx_http_chunked_filter_module); + } else { r->keepalive = 0; } @@ -83,17 +98,21 @@ ngx_http_chunked_header_filter(ngx_http_ static ngx_int_t ngx_http_chunked_body_filter(ngx_http_request_t *r, ngx_chain_t *in) { - u_char *chunk; - off_t size; - ngx_buf_t *b; - ngx_chain_t out, tail, *cl, *tl, **ll; + u_char *chunk; + off_t size; + ngx_int_t rc; + ngx_buf_t *b; + ngx_chain_t *out, *cl, *tl, **ll; + ngx_http_chunked_filter_ctx_t *ctx; if (in == NULL || !r->chunked || r->header_only) { return ngx_http_next_body_filter(r, in); } - out.buf = NULL; - ll = &out.next; + ctx = ngx_http_get_module_ctx(r, ngx_http_chunked_filter_module); + + out = NULL; + ll = &out; size = 0; cl = in; @@ -127,31 +146,46 @@ ngx_http_chunked_body_filter(ngx_http_re } if (size) { - b = ngx_calloc_buf(r->pool); - if (b == NULL) { + tl = ngx_chain_get_free_buf(r->pool, &ctx->free); + if (tl == NULL) { return NGX_ERROR; } - /* the "0000000000000000" is 64-bit hexadimal string */ + b = tl->buf; + chunk = b->start; + + if (chunk == NULL) { + /* the "0000000000000000" is 64-bit hexadecimal string */ - chunk = ngx_palloc(r->pool, sizeof("0000000000000000" CRLF) - 1); - if (chunk == NULL) { - return NGX_ERROR; + chunk = ngx_palloc(r->pool, sizeof("0000000000000000" CRLF) - 1); + if (chunk == NULL) { + return NGX_ERROR; + } + + b->start = chunk; + b->end = chunk + sizeof("0000000000000000" CRLF) - 1; } + b->tag = (ngx_buf_tag_t) &ngx_http_chunked_filter_module; + b->memory = 0; b->temporary = 1; b->pos = chunk; b->last = ngx_sprintf(chunk, "%xO" CRLF, size); - out.buf = b; + tl->next = out; + out = tl; } if (cl->buf->last_buf) { - b = ngx_calloc_buf(r->pool); - if (b == NULL) { + tl = ngx_chain_get_free_buf(r->pool, &ctx->free); + if (tl == NULL) { return NGX_ERROR; } + b = tl->buf; + + b->tag = (ngx_buf_tag_t) &ngx_http_chunked_filter_module; + b->temporary = 0; b->memory = 1; b->last_buf = 1; b->pos = (u_char *) CRLF "0" CRLF CRLF; @@ -159,35 +193,38 @@ ngx_http_chunked_body_filter(ngx_http_re cl->buf->last_buf = 0; + *ll = tl; + if (size == 0) { b->pos += 2; - out.buf = b; - out.next = NULL; - - return ngx_http_next_body_filter(r, &out); } - } else { - if (size == 0) { - *ll = NULL; - return ngx_http_next_body_filter(r, out.next); - } - - b = ngx_calloc_buf(r->pool); - if (b == NULL) { + } else if (size > 0) { + tl = ngx_chain_get_free_buf(r->pool, &ctx->free); + if (tl == NULL) { return NGX_ERROR; } + b = tl->buf; + + b->tag = (ngx_buf_tag_t) &ngx_http_chunked_filter_module; + b->temporary = 0; b->memory = 1; b->pos = (u_char *) CRLF; b->last = b->pos + 2; + + *ll = tl; + + } else { + *ll = NULL; } - tail.buf = b; - tail.next = NULL; - *ll = &tail; + rc = ngx_http_next_body_filter(r, out); - return ngx_http_next_body_filter(r, &out); + ngx_chain_update_chains(&ctx->free, &ctx->busy, &out, + (ngx_buf_tag_t) &ngx_http_chunked_filter_module); + + return rc; }