# HG changeset patch # User Maxim Dounin # Date 1360593240 0 # Node ID 004af18ddb86e1adc9a8131e3413a96ebbf95633 # Parent f2dcb25987dff40245159b3da8e8ee112add8211 Merge of r4985, r4986, r4987, r4988, r4989, r5002: access_log gzip. *) Access log: fixed redundant buffer reallocation. Previously a new buffer was allocated for every "access_log" directive with the same file path and "buffer=" parameters, while only one buffer per file is used. *) Reopening log files code moved to a separate function. The code refactored in a way to call custom handler that can do appropriate cleanup work (if any), like flushing buffers, finishing compress streams, finalizing connections to log daemon, etc.. *) Access log: the "flush" parameter of the "access_log" directive. *) Configure: added the NGX_ZLIB define. This was introduced for conditional compilation of the code that requires the zlib library. *) Access log: the "gzip" parameter of the "access_log" directive. Note: this requires zlib version 1.2.0.4 or above to work. *) The data pointer in ngx_open_file_t objects must be initialized. Uninitialized pointer may result in arbitrary segfaults if access_log is used without buffer and without variables in file path. Patch by Tatsuhiko Kubo (ticket #268). diff --git a/auto/lib/zlib/conf b/auto/lib/zlib/conf --- a/auto/lib/zlib/conf +++ b/auto/lib/zlib/conf @@ -9,11 +9,13 @@ if [ $ZLIB != NONE ]; then case "$NGX_CC_NAME" in msvc* | owc* | bcc) + have=NGX_ZLIB . auto/have LINK_DEPS="$LINK_DEPS $ZLIB/zlib.lib" CORE_LIBS="$CORE_LIBS $ZLIB/zlib.lib" ;; icc*) + have=NGX_ZLIB . auto/have LINK_DEPS="$LINK_DEPS $ZLIB/libz.a" # to allow -ipo optimization we link with the *.o but not library @@ -30,6 +32,7 @@ if [ $ZLIB != NONE ]; then ;; *) + have=NGX_ZLIB . auto/have LINK_DEPS="$LINK_DEPS $ZLIB/libz.a" CORE_LIBS="$CORE_LIBS $ZLIB/libz.a" #CORE_LIBS="$CORE_LIBS -L $ZLIB -lz" @@ -45,7 +48,7 @@ else # FreeBSD, Solaris, Linux ngx_feature="zlib library" - ngx_feature_name= + ngx_feature_name="NGX_ZLIB" ngx_feature_run=no ngx_feature_incs="#include " ngx_feature_path= diff --git a/src/core/ngx_conf_file.c b/src/core/ngx_conf_file.c --- a/src/core/ngx_conf_file.c +++ b/src/core/ngx_conf_file.c @@ -945,7 +945,8 @@ ngx_conf_open_file(ngx_cycle_t *cycle, n file->name = *name; } - file->buffer = NULL; + file->flush = NULL; + file->data = NULL; return file; } @@ -954,7 +955,6 @@ ngx_conf_open_file(ngx_cycle_t *cycle, n static void ngx_conf_flush_files(ngx_cycle_t *cycle) { - ssize_t n, len; ngx_uint_t i; ngx_list_part_t *part; ngx_open_file_t *file; @@ -975,23 +975,8 @@ ngx_conf_flush_files(ngx_cycle_t *cycle) i = 0; } - len = file[i].pos - file[i].buffer; - - if (file[i].buffer == NULL || len == 0) { - continue; - } - - n = ngx_write_fd(file[i].fd, file[i].buffer, len); - - if (n == -1) { - ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, - ngx_write_fd_n " to \"%s\" failed", - file[i].name.data); - - } else if (n != len) { - ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, - ngx_write_fd_n " to \"%s\" was incomplete: %z of %uz", - file[i].name.data, n, len); + if (file[i].flush) { + file[i].flush(&file[i], cycle->log); } } } diff --git a/src/core/ngx_conf_file.h b/src/core/ngx_conf_file.h --- a/src/core/ngx_conf_file.h +++ b/src/core/ngx_conf_file.h @@ -91,17 +91,8 @@ struct ngx_open_file_s { ngx_fd_t fd; ngx_str_t name; - u_char *buffer; - u_char *pos; - u_char *last; - -#if 0 - /* e.g. append mode, error_log */ - ngx_uint_t flags; - /* e.g. reopen db file */ - ngx_uint_t (*handler)(void *data, ngx_open_file_t *file); + void (*flush)(ngx_open_file_t *file, ngx_log_t *log); void *data; -#endif }; diff --git a/src/core/ngx_cycle.c b/src/core/ngx_cycle.c --- a/src/core/ngx_cycle.c +++ b/src/core/ngx_cycle.c @@ -1115,7 +1115,6 @@ ngx_test_lockfile(u_char *file, ngx_log_ void ngx_reopen_files(ngx_cycle_t *cycle, ngx_uid_t user) { - ssize_t n, len; ngx_fd_t fd; ngx_uint_t i; ngx_list_part_t *part; @@ -1139,24 +1138,8 @@ ngx_reopen_files(ngx_cycle_t *cycle, ngx continue; } - len = file[i].pos - file[i].buffer; - - if (file[i].buffer && len != 0) { - - n = ngx_write_fd(file[i].fd, file[i].buffer, len); - - if (n == -1) { - ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, - ngx_write_fd_n " to \"%s\" failed", - file[i].name.data); - - } else if (n != len) { - ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, - ngx_write_fd_n " to \"%s\" was incomplete: %z of %uz", - file[i].name.data, n, len); - } - - file[i].pos = file[i].buffer; + if (file[i].flush) { + file[i].flush(&file[i], cycle->log); } fd = ngx_open_file(file[i].name.data, NGX_FILE_APPEND, diff --git a/src/http/modules/ngx_http_log_module.c b/src/http/modules/ngx_http_log_module.c --- a/src/http/modules/ngx_http_log_module.c +++ b/src/http/modules/ngx_http_log_module.c @@ -9,6 +9,10 @@ #include #include +#if (NGX_ZLIB) +#include +#endif + typedef struct ngx_http_log_op_s ngx_http_log_op_t; @@ -41,6 +45,17 @@ typedef struct { typedef struct { + u_char *start; + u_char *pos; + u_char *last; + + ngx_event_t *event; + ngx_msec_t flush; + ngx_int_t gzip; +} ngx_http_log_buf_t; + + +typedef struct { ngx_array_t *lengths; ngx_array_t *values; } ngx_http_log_script_t; @@ -78,6 +93,17 @@ static void ngx_http_log_write(ngx_http_ static ssize_t ngx_http_log_script_write(ngx_http_request_t *r, ngx_http_log_script_t *script, u_char **name, u_char *buf, size_t len); +#if (NGX_ZLIB) +static ssize_t ngx_http_log_gzip(ngx_fd_t fd, u_char *buf, size_t len, + ngx_int_t level, ngx_log_t *log); + +static void *ngx_http_log_gzip_alloc(void *opaque, u_int items, u_int size); +static void ngx_http_log_gzip_free(void *opaque, void *address); +#endif + +static void ngx_http_log_flush(ngx_open_file_t *file, ngx_log_t *log); +static void ngx_http_log_flush_handler(ngx_event_t *ev); + static u_char *ngx_http_log_pipe(ngx_http_request_t *r, u_char *buf, ngx_http_log_op_t *op); static u_char *ngx_http_log_time(ngx_http_request_t *r, u_char *buf, @@ -132,7 +158,7 @@ static ngx_command_t ngx_http_log_comma { ngx_string("access_log"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF - |NGX_HTTP_LMT_CONF|NGX_CONF_TAKE123, + |NGX_HTTP_LMT_CONF|NGX_CONF_1MORE, ngx_http_log_set_log, NGX_HTTP_LOC_CONF_OFFSET, 0, @@ -216,8 +242,8 @@ ngx_http_log_handler(ngx_http_request_t size_t len; ngx_uint_t i, l; ngx_http_log_t *log; - ngx_open_file_t *file; ngx_http_log_op_t *op; + ngx_http_log_buf_t *buffer; ngx_http_log_loc_conf_t *lcf; ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, @@ -258,21 +284,25 @@ ngx_http_log_handler(ngx_http_request_t len += NGX_LINEFEED_SIZE; - file = log[l].file; + buffer = log[l].file ? log[l].file->data : NULL; - if (file && file->buffer) { + if (buffer) { - if (len > (size_t) (file->last - file->pos)) { + if (len > (size_t) (buffer->last - buffer->pos)) { - ngx_http_log_write(r, &log[l], file->buffer, - file->pos - file->buffer); + ngx_http_log_write(r, &log[l], buffer->start, + buffer->pos - buffer->start); - file->pos = file->buffer; + buffer->pos = buffer->start; } - if (len <= (size_t) (file->last - file->pos)) { + if (len <= (size_t) (buffer->last - buffer->pos)) { + + p = buffer->pos; - p = file->pos; + if (buffer->event && p == buffer->start) { + ngx_add_timer(buffer->event, buffer->flush); + } for (i = 0; i < log[l].format->ops->nelts; i++) { p = op[i].run(r, p, &op[i]); @@ -280,10 +310,14 @@ ngx_http_log_handler(ngx_http_request_t ngx_linefeed(p); - file->pos = p; + buffer->pos = p; continue; } + + if (buffer->event && buffer->event->timer_set) { + ngx_del_timer(buffer->event); + } } line = ngx_pnalloc(r->pool, len); @@ -310,14 +344,29 @@ static void ngx_http_log_write(ngx_http_request_t *r, ngx_http_log_t *log, u_char *buf, size_t len) { - u_char *name; - time_t now; - ssize_t n; - ngx_err_t err; + u_char *name; + time_t now; + ssize_t n; + ngx_err_t err; +#if (NGX_ZLIB) + ngx_http_log_buf_t *buffer; +#endif if (log->script == NULL) { name = log->file->name.data; + +#if (NGX_ZLIB) + buffer = log->file->data; + + if (buffer && buffer->gzip) { + n = ngx_http_log_gzip(log->file->fd, buf, len, buffer->gzip, + r->connection->log); + } else { + n = ngx_write_fd(log->file->fd, buf, len); + } +#else n = ngx_write_fd(log->file->fd, buf, len); +#endif } else { name = NULL; @@ -465,6 +514,194 @@ ngx_http_log_script_write(ngx_http_reque } +#if (NGX_ZLIB) + +static ssize_t +ngx_http_log_gzip(ngx_fd_t fd, u_char *buf, size_t len, ngx_int_t level, + ngx_log_t *log) +{ + int rc, wbits, memlevel; + u_char *out; + size_t size; + ssize_t n; + z_stream zstream; + ngx_err_t err; + ngx_pool_t *pool; + + wbits = MAX_WBITS; + memlevel = MAX_MEM_LEVEL - 1; + + while ((ssize_t) len < ((1 << (wbits - 1)) - 262)) { + wbits--; + memlevel--; + } + + /* + * This is a formula from deflateBound() for conservative upper bound of + * compressed data plus 18 bytes of gzip wrapper. + */ + + size = len + ((len + 7) >> 3) + ((len + 63) >> 6) + 5 + 18; + + ngx_memzero(&zstream, sizeof(z_stream)); + + pool = ngx_create_pool(256, log); + if (pool == NULL) { + /* simulate successful logging */ + return len; + } + + pool->log = log; + + zstream.zalloc = ngx_http_log_gzip_alloc; + zstream.zfree = ngx_http_log_gzip_free; + zstream.opaque = pool; + + out = ngx_pnalloc(pool, size); + if (out == NULL) { + goto done; + } + + zstream.next_in = buf; + zstream.avail_in = len; + zstream.next_out = out; + zstream.avail_out = size; + + rc = deflateInit2(&zstream, (int) level, Z_DEFLATED, wbits + 16, memlevel, + Z_DEFAULT_STRATEGY); + + if (rc != Z_OK) { + ngx_log_error(NGX_LOG_ALERT, log, 0, "deflateInit2() failed: %d", rc); + goto done; + } + + ngx_log_debug4(NGX_LOG_DEBUG_HTTP, log, 0, + "deflate in: ni:%p no:%p ai:%ud ao:%ud", + zstream.next_in, zstream.next_out, + zstream.avail_in, zstream.avail_out); + + rc = deflate(&zstream, Z_FINISH); + + if (rc != Z_STREAM_END) { + ngx_log_error(NGX_LOG_ALERT, log, 0, + "deflate(Z_FINISH) failed: %d", rc); + goto done; + } + + ngx_log_debug5(NGX_LOG_DEBUG_HTTP, log, 0, + "deflate out: ni:%p no:%p ai:%ud ao:%ud rc:%d", + zstream.next_in, zstream.next_out, + zstream.avail_in, zstream.avail_out, + rc); + + size -= zstream.avail_out; + + rc = deflateEnd(&zstream); + + if (rc != Z_OK) { + ngx_log_error(NGX_LOG_ALERT, log, 0, "deflateEnd() failed: %d", rc); + goto done; + } + + n = ngx_write_fd(fd, out, size); + + if (n != (ssize_t) size) { + err = (n == -1) ? ngx_errno : 0; + + ngx_destroy_pool(pool); + + ngx_set_errno(err); + return -1; + } + +done: + + ngx_destroy_pool(pool); + + /* simulate successful logging */ + return len; +} + + +static void * +ngx_http_log_gzip_alloc(void *opaque, u_int items, u_int size) +{ + ngx_pool_t *pool = opaque; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pool->log, 0, + "gzip alloc: n:%ud s:%ud", items, size); + + return ngx_palloc(pool, items * size); +} + + +static void +ngx_http_log_gzip_free(void *opaque, void *address) +{ +#if 0 + ngx_pool_t *pool = opaque; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pool->log, 0, "gzip free: %p", address); +#endif +} + +#endif + + +static void +ngx_http_log_flush(ngx_open_file_t *file, ngx_log_t *log) +{ + size_t len; + ssize_t n; + ngx_http_log_buf_t *buffer; + + buffer = file->data; + + len = buffer->pos - buffer->start; + + if (len == 0) { + return; + } + +#if (NGX_ZLIB) + if (buffer->gzip) { + n = ngx_http_log_gzip(file->fd, buffer->start, len, buffer->gzip, log); + } else { + n = ngx_write_fd(file->fd, buffer->start, len); + } +#else + n = ngx_write_fd(file->fd, buffer->start, len); +#endif + + if (n == -1) { + ngx_log_error(NGX_LOG_ALERT, log, ngx_errno, + ngx_write_fd_n " to \"%s\" failed", + file->name.data); + + } else if ((size_t) n != len) { + ngx_log_error(NGX_LOG_ALERT, log, 0, + ngx_write_fd_n " to \"%s\" was incomplete: %z of %uz", + file->name.data, n, len); + } + + buffer->pos = buffer->start; + + if (buffer->event && buffer->event->timer_set) { + ngx_del_timer(buffer->event); + } +} + + +static void +ngx_http_log_flush_handler(ngx_event_t *ev) +{ + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, + "http log buffer flush handler"); + + ngx_http_log_flush(ev->data, ev->log); +} + + static u_char * ngx_http_log_copy_short(ngx_http_request_t *r, u_char *buf, ngx_http_log_op_t *op) @@ -848,10 +1085,13 @@ ngx_http_log_set_log(ngx_conf_t *cf, ngx { ngx_http_log_loc_conf_t *llcf = conf; - ssize_t buf; + ssize_t size; + ngx_int_t gzip; ngx_uint_t i, n; - ngx_str_t *value, name; + ngx_msec_t flush; + ngx_str_t *value, name, s; ngx_http_log_t *log; + ngx_http_log_buf_t *buffer; ngx_http_log_fmt_t *fmt; ngx_http_log_main_conf_t *lmcf; ngx_http_script_compile_t sc; @@ -936,54 +1176,152 @@ ngx_http_log_set_log(ngx_conf_t *cf, ngx && ngx_strcasecmp(fmt[i].name.data, name.data) == 0) { log->format = &fmt[i]; - goto buffer; + break; } } - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "unknown log format \"%V\"", &name); - return NGX_CONF_ERROR; + if (log->format == NULL) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "unknown log format \"%V\"", &name); + return NGX_CONF_ERROR; + } + + size = 0; + flush = 0; + gzip = 0; + + for (i = 3; i < cf->args->nelts; i++) { + + if (ngx_strncmp(value[i].data, "buffer=", 7) == 0) { + s.len = value[i].len - 7; + s.data = value[i].data + 7; + + size = ngx_parse_size(&s); -buffer: + if (size == NGX_ERROR || size == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid buffer size \"%V\"", &s); + return NGX_CONF_ERROR; + } + + continue; + } + + if (ngx_strncmp(value[i].data, "flush=", 6) == 0) { + s.len = value[i].len - 6; + s.data = value[i].data + 6; + + flush = ngx_parse_time(&s, 0); + + if (flush == (ngx_msec_t) NGX_ERROR || flush == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid flush time \"%V\"", &s); + return NGX_CONF_ERROR; + } + + continue; + } - if (cf->args->nelts == 4) { - if (ngx_strncmp(value[3].data, "buffer=", 7) != 0) { + if (ngx_strncmp(value[i].data, "gzip", 4) == 0 + && (value[i].len == 4 || value[i].data[4] == '=')) + { +#if (NGX_ZLIB) + if (size == 0) { + size = 64 * 1024; + } + + if (value[i].len == 4) { + gzip = Z_BEST_SPEED; + continue; + } + + s.len = value[i].len - 5; + s.data = value[i].data + 5; + + gzip = ngx_atoi(s.data, s.len); + + if (gzip < 1 || gzip > 9) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid compression level \"%V\"", &s); + return NGX_CONF_ERROR; + } + + continue; + +#else ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "invalid parameter \"%V\"", &value[3]); + "nginx was built without zlib support"); return NGX_CONF_ERROR; +#endif } + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid parameter \"%V\"", &value[i]); + return NGX_CONF_ERROR; + } + + if (flush && size == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "no buffer is defined for access_log \"%V\"", + &value[1]); + return NGX_CONF_ERROR; + } + + if (size) { + if (log->script) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "buffered logs cannot have variables in name"); return NGX_CONF_ERROR; } - name.len = value[3].len - 7; - name.data = value[3].data + 7; + if (log->file->data) { + buffer = log->file->data; - buf = ngx_parse_size(&name); + if (buffer->last - buffer->start != size + || buffer->flush != flush + || buffer->gzip != gzip) + { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "access_log \"%V\" already defined " + "with conflicting parameters", + &value[1]); + return NGX_CONF_ERROR; + } - if (buf == NGX_ERROR) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "invalid buffer value \"%V\"", &name); + return NGX_CONF_OK; + } + + buffer = ngx_pcalloc(cf->pool, sizeof(ngx_http_log_buf_t)); + if (buffer == NULL) { return NGX_CONF_ERROR; } - if (log->file->buffer && log->file->last - log->file->pos != buf) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "access_log \"%V\" already defined " - "with different buffer size", &value[1]); + buffer->start = ngx_pnalloc(cf->pool, size); + if (buffer->start == NULL) { return NGX_CONF_ERROR; } - log->file->buffer = ngx_palloc(cf->pool, buf); - if (log->file->buffer == NULL) { - return NGX_CONF_ERROR; + buffer->pos = buffer->start; + buffer->last = buffer->start + size; + + if (flush) { + buffer->event = ngx_pcalloc(cf->pool, sizeof(ngx_event_t)); + if (buffer->event == NULL) { + return NGX_CONF_ERROR; + } + + buffer->event->data = log->file; + buffer->event->handler = ngx_http_log_flush_handler; + buffer->event->log = &cf->cycle->new_log; + + buffer->flush = flush; } - log->file->pos = log->file->buffer; - log->file->last = log->file->buffer + buf; + buffer->gzip = gzip; + + log->file->flush = ngx_http_log_flush; + log->file->data = buffer; } return NGX_CONF_OK;