# HG changeset patch # User Igor Sysoev # Date 1229443779 0 # Node ID bbfaeb220e902aba26bd8c0f178ed93e3721edf0 # Parent 428244458039f03930c640b3f81aa9cc51ae4787 postpone zlib memory allocation diff --git a/src/http/modules/ngx_http_gzip_filter_module.c b/src/http/modules/ngx_http_gzip_filter_module.c --- a/src/http/modules/ngx_http_gzip_filter_module.c +++ b/src/http/modules/ngx_http_gzip_filter_module.c @@ -34,21 +34,27 @@ typedef struct { ngx_chain_t *busy; ngx_chain_t *out; ngx_chain_t **last_out; + + ngx_chain_t *copied; + ngx_chain_t *copy_buf; + ngx_buf_t *in_buf; ngx_buf_t *out_buf; ngx_int_t bufs; - off_t length; - void *preallocated; char *free_mem; ngx_uint_t allocated; + int wbits; + int memlevel; + unsigned flush:4; unsigned redo:1; unsigned done:1; unsigned nomem:1; unsigned gzheader:1; + unsigned buffering:1; size_t zin; size_t zout; @@ -76,6 +82,10 @@ struct gztrailer { #endif +static void ngx_http_gzip_filter_memory(ngx_http_request_t *r, + ngx_http_gzip_ctx_t *ctx); +static ngx_int_t ngx_http_gzip_filter_copy_recycled(ngx_http_gzip_ctx_t *ctx, + ngx_chain_t *in); static ngx_int_t ngx_http_gzip_filter_deflate_start(ngx_http_request_t *r, ngx_http_gzip_ctx_t *ctx); static ngx_int_t ngx_http_gzip_filter_gzheader(ngx_http_request_t *r, @@ -92,6 +102,8 @@ static ngx_int_t ngx_http_gzip_filter_de static void *ngx_http_gzip_filter_alloc(void *opaque, u_int items, u_int size); static void ngx_http_gzip_filter_free(void *opaque, void *address); +static void ngx_http_gzip_filter_free_copy_buf(ngx_http_request_t *r, + ngx_http_gzip_ctx_t *ctx); static ngx_int_t ngx_http_gzip_add_variables(ngx_conf_t *cf); static ngx_int_t ngx_http_gzip_ratio_variable(ngx_http_request_t *r, @@ -245,6 +257,9 @@ ngx_http_gzip_header_filter(ngx_http_req ngx_http_set_ctx(r, ctx, ngx_http_gzip_filter_module); ctx->request = r; + ctx->buffering = 1; + + ngx_http_gzip_filter_memory(r, ctx); h = ngx_list_push(&r->headers_out.headers); if (h == NULL) { @@ -259,8 +274,6 @@ ngx_http_gzip_header_filter(ngx_http_req r->headers_out.content_encoding = h; - ctx->length = r->headers_out.content_length_n; - r->main_filter_need_in_memory = 1; ngx_http_clear_content_length(r); @@ -283,6 +296,30 @@ ngx_http_gzip_body_filter(ngx_http_reque return ngx_http_next_body_filter(r, in); } + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http gzip filter"); + + if (ctx->buffering) { + + if (in) { + switch (ngx_http_gzip_filter_copy_recycled(ctx, in)) { + + case NGX_OK: + return NGX_OK; + + case NGX_DONE: + in = NULL; + break; + + default: /* NGX_ERROR */ + goto failed; + } + + } else { + ctx->buffering = 0; + } + } + if (ctx->preallocated == NULL) { if (ngx_http_gzip_filter_deflate_start(r, ctx) != NGX_OK) { goto failed; @@ -356,6 +393,8 @@ ngx_http_gzip_body_filter(ngx_http_reque } if (ctx->out == NULL) { + ngx_http_gzip_filter_free_copy_buf(r, ctx); + return ctx->busy ? NGX_AGAIN : NGX_OK; } @@ -371,6 +410,8 @@ ngx_http_gzip_body_filter(ngx_http_reque goto failed; } + ngx_http_gzip_filter_free_copy_buf(r, ctx); + ngx_chain_update_chains(&ctx->free, &ctx->busy, &ctx->out, (ngx_buf_tag_t) &ngx_http_gzip_filter_module); ctx->last_out = &ctx->out; @@ -394,15 +435,16 @@ failed: ngx_pfree(r->pool, ctx->preallocated); } + ngx_http_gzip_filter_free_copy_buf(r, ctx); + return NGX_ERROR; } -static ngx_int_t -ngx_http_gzip_filter_deflate_start(ngx_http_request_t *r, - ngx_http_gzip_ctx_t *ctx) +static void +ngx_http_gzip_filter_memory(ngx_http_request_t *r, ngx_http_gzip_ctx_t *ctx) { - int rc, wbits, memlevel; + int wbits, memlevel; ngx_http_gzip_conf_t *conf; conf = ngx_http_get_module_loc_conf(r, ngx_http_gzip_filter_module); @@ -410,16 +452,19 @@ ngx_http_gzip_filter_deflate_start(ngx_h wbits = conf->wbits; memlevel = conf->memlevel; - if (ctx->length > 0) { + if (r->headers_out.content_length_n > 0) { /* the actual zlib window size is smaller by 262 bytes */ - while (ctx->length < ((1 << (wbits - 1)) - 262)) { + while (r->headers_out.content_length_n < ((1 << (wbits - 1)) - 262)) { wbits--; memlevel--; } } + ctx->wbits = wbits; + ctx->memlevel = memlevel; + /* * We preallocate a memory for zlib in one buffer (200K-400K), this * decreases a number of malloc() and free() calls and also probably @@ -433,6 +478,96 @@ ngx_http_gzip_filter_deflate_start(ngx_h */ ctx->allocated = 8192 + (1 << (wbits + 2)) + (1 << (memlevel + 9)); +} + + +static ngx_int_t +ngx_http_gzip_filter_copy_recycled(ngx_http_gzip_ctx_t *ctx, ngx_chain_t *in) +{ + size_t size, buffered; + ngx_buf_t *b, *buf; + ngx_chain_t *cl, **ll; + ngx_http_request_t *r; + + r = ctx->request; + + r->connection->buffered |= NGX_HTTP_GZIP_BUFFERED; + + buffered = 0; + ll = &ctx->in; + + for (cl = ctx->in; cl; cl = cl->next) { + buffered += cl->buf->last - cl->buf->pos; + ll = &cl->next; + } + + while (in) { + cl = ngx_alloc_chain_link(r->pool); + if (cl == NULL) { + return NGX_ERROR; + } + + b = in->buf; + + size = b->last - b->pos; + buffered += size; + + if (b->flush || b->last_buf) { + ctx->buffering = 0; + + } else if (buffered > ctx->allocated / 2) { + + /* + * With default memory settings zlib starts to output gzipped data + * only after it has got about 90K, so it makes sense to allocate + * zlib memory (200-400K) only after we have enough data + * to compress. Although we copy recycled buffers, nevertheless + * for responses up to 120K this allows to allocate zlib memory, + * to compress and to output the response in one step + * using hot CPU cache. + */ + + ctx->buffering = 0; + } + + if (ctx->buffering && size && b->recycled) { + + buf = ngx_create_temp_buf(r->pool, size); + if (buf == NULL) { + return NGX_ERROR; + } + + buf->last = ngx_cpymem(buf->pos, b->pos, size); + b->pos = b->last; + + buf->last_buf = b->last_buf; + buf->tag = (ngx_buf_tag_t) &ngx_http_gzip_filter_module; + + cl->buf = buf; + + } else { + cl->buf = b; + } + + *ll = cl; + ll = &cl->next; + in = in->next; + } + + *ll = NULL; + + return ctx->buffering ? NGX_OK : NGX_DONE; +} + + +static ngx_int_t +ngx_http_gzip_filter_deflate_start(ngx_http_request_t *r, + ngx_http_gzip_ctx_t *ctx) +{ + int rc; + ngx_http_gzip_conf_t *conf; + + conf = ngx_http_get_module_loc_conf(r, ngx_http_gzip_filter_module); ctx->preallocated = ngx_palloc(r->pool, ctx->allocated); if (ctx->preallocated == NULL) { @@ -446,7 +581,7 @@ ngx_http_gzip_filter_deflate_start(ngx_h ctx->zstream.opaque = ctx; rc = deflateInit2(&ctx->zstream, (int) conf->level, Z_DEFLATED, - -wbits, memlevel, Z_DEFAULT_STRATEGY); + - ctx->wbits, ctx->memlevel, Z_DEFAULT_STRATEGY); if (rc != Z_OK) { ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, @@ -454,8 +589,6 @@ ngx_http_gzip_filter_deflate_start(ngx_h return NGX_ERROR; } - r->connection->buffered |= NGX_HTTP_GZIP_BUFFERED; - ctx->last_out = &ctx->out; ctx->crc32 = crc32(0L, Z_NULL, 0); ctx->flush = Z_NO_FLUSH; @@ -510,7 +643,24 @@ ngx_http_gzip_filter_add_data(ngx_http_r return NGX_DECLINED; } + if (ctx->copy_buf) { + + /* + * to avoid CPU cache trashing we do not free() just quit buf, + * but postpone free()ing after zlib compressing and data output + */ + + ctx->copy_buf->next = ctx->copied; + ctx->copied = ctx->copy_buf; + ctx->copy_buf = NULL; + } + ctx->in_buf = ctx->in->buf; + + if (ctx->in_buf->tag == (ngx_buf_tag_t) &ngx_http_gzip_filter_module) { + ctx->copy_buf = ctx->in; + } + ctx->in = ctx->in->next; ctx->zstream.next_in = ctx->in_buf->pos; @@ -832,6 +982,20 @@ ngx_http_gzip_filter_free(void *opaque, } +static void +ngx_http_gzip_filter_free_copy_buf(ngx_http_request_t *r, + ngx_http_gzip_ctx_t *ctx) +{ + ngx_chain_t *cl; + + for (cl = ctx->copied; cl; cl = cl->next) { + ngx_pfree(r->pool, cl->buf->start); + } + + ctx->copied = NULL; +} + + static ngx_int_t ngx_http_gzip_add_variables(ngx_conf_t *cf) {