# HG changeset patch # User Igor Sysoev # Date 1353960000 -14400 # Node ID 6db6e93f55eecaefdba69af347c1ecc7be993d3f # Parent acfd484db0ca1908b9ab94f1bbb57123a655997b nginx 1.3.9 *) Feature: support for chunked transfer encoding while reading client request body. *) Feature: the $request_time and $msec variables can now be used not only in the "log_format" directive. *) Bugfix: cache manager and cache loader processes might not be able to start if more than 512 listen sockets were used. *) Bugfix: in the ngx_http_dav_module. diff --git a/CHANGES b/CHANGES --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,18 @@ +Changes with nginx 1.3.9 27 Nov 2012 + + *) Feature: support for chunked transfer encoding while reading client + request body. + + *) Feature: the $request_time and $msec variables can now be used not + only in the "log_format" directive. + + *) Bugfix: cache manager and cache loader processes might not be able to + start if more than 512 listen sockets were used. + + *) Bugfix: in the ngx_http_dav_module. + + Changes with nginx 1.3.8 30 Oct 2012 *) Feature: the "optional_no_ca" parameter of the "ssl_verify_client" diff --git a/CHANGES.ru b/CHANGES.ru --- a/CHANGES.ru +++ b/CHANGES.ru @@ -1,4 +1,18 @@ +Изменения в nginx 1.3.9 27.11.2012 + + *) Добавление: поддержка chunked transfer encoding при получении тела + запроса. + + *) Добавление: переменные $request_time и $msec теперь можно + использовать не только в директиве log_format. + + *) Исправление: cache manager и cache loader могли не запускаться, если + использовалось более 512 listen-сокетов. + + *) Исправление: в модуле ngx_http_dav_module. + + Изменения в nginx 1.3.8 30.10.2012 *) Добавление: параметр optional_no_ca директивы ssl_verify_client. diff --git a/src/core/nginx.h b/src/core/nginx.h --- a/src/core/nginx.h +++ b/src/core/nginx.h @@ -9,8 +9,8 @@ #define _NGINX_H_INCLUDED_ -#define nginx_version 1003008 -#define NGINX_VERSION "1.3.8" +#define nginx_version 1003009 +#define NGINX_VERSION "1.3.9" #define NGINX_VER "nginx/" NGINX_VERSION #define NGINX_VAR "NGINX" diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c --- a/src/core/ngx_connection.c +++ b/src/core/ngx_connection.c @@ -749,6 +749,8 @@ ngx_close_listening_sockets(ngx_cycle_t ls[i].fd = (ngx_socket_t) -1; } + + cycle->listening.nelts = 0; } diff --git a/src/core/ngx_cycle.c b/src/core/ngx_cycle.c --- a/src/core/ngx_cycle.c +++ b/src/core/ngx_cycle.c @@ -447,7 +447,9 @@ ngx_init_cycle(ngx_cycle_t *old_cycle) continue; } - if (shm_zone[i].shm.size == oshm_zone[n].shm.size) { + if (shm_zone[i].tag == oshm_zone[n].tag + && shm_zone[i].shm.size == oshm_zone[n].shm.size) + { shm_zone[i].shm.addr = oshm_zone[n].shm.addr; if (shm_zone[i].init(&shm_zone[i], oshm_zone[n].data) diff --git a/src/core/ngx_shmtx.c b/src/core/ngx_shmtx.c --- a/src/core/ngx_shmtx.c +++ b/src/core/ngx_shmtx.c @@ -117,10 +117,10 @@ ngx_shmtx_lock(ngx_shmtx_t *mtx) "sem_wait() failed while waiting on shmtx"); break; } + } - ngx_log_debug0(NGX_LOG_DEBUG_CORE, ngx_cycle->log, 0, - "shmtx awoke"); - } + ngx_log_debug0(NGX_LOG_DEBUG_CORE, ngx_cycle->log, 0, + "shmtx awoke"); continue; } diff --git a/src/http/modules/ngx_http_dav_module.c b/src/http/modules/ngx_http_dav_module.c --- a/src/http/modules/ngx_http_dav_module.c +++ b/src/http/modules/ngx_http_dav_module.c @@ -209,6 +209,11 @@ ngx_http_dav_put_handler(ngx_http_reques ngx_ext_rename_file_t ext; ngx_http_dav_loc_conf_t *dlcf; + if (r->request_body == NULL || r->request_body->temp_file == NULL) { + ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); + return; + } + ngx_http_map_uri_to_path(r, &path, &root, 0); path.len--; diff --git a/src/http/modules/ngx_http_gunzip_filter_module.c b/src/http/modules/ngx_http_gunzip_filter_module.c --- a/src/http/modules/ngx_http_gunzip_filter_module.c +++ b/src/http/modules/ngx_http_gunzip_filter_module.c @@ -165,6 +165,7 @@ ngx_http_gunzip_header_filter(ngx_http_r ngx_http_clear_content_length(r); ngx_http_clear_accept_ranges(r); + ngx_http_clear_etag(r); return ngx_http_next_header_filter(r); } diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c --- a/src/http/modules/ngx_http_proxy_module.c +++ b/src/http/modules/ngx_http_proxy_module.c @@ -81,12 +81,9 @@ typedef struct { typedef struct { ngx_http_status_t status; + ngx_http_chunked_t chunked; ngx_http_proxy_vars_t vars; - size_t internal_body_length; - - ngx_uint_t state; - off_t size; - off_t length; + off_t internal_body_length; ngx_uint_t head; /* unsigned head:1 */ } ngx_http_proxy_ctx_t; @@ -558,6 +555,8 @@ static char ngx_http_proxy_version_11[] static ngx_keyval_t ngx_http_proxy_headers[] = { { ngx_string("Host"), ngx_string("$proxy_host") }, { ngx_string("Connection"), ngx_string("close") }, + { ngx_string("Content-Length"), ngx_string("$proxy_internal_body_length") }, + { ngx_string("Transfer-Encoding"), ngx_string("") }, { ngx_string("Keep-Alive"), ngx_string("") }, { ngx_string("Expect"), ngx_string("") }, { ngx_string("Upgrade"), ngx_string("") }, @@ -583,6 +582,8 @@ static ngx_str_t ngx_http_proxy_hide_he static ngx_keyval_t ngx_http_proxy_cache_headers[] = { { ngx_string("Host"), ngx_string("$proxy_host") }, { ngx_string("Connection"), ngx_string("close") }, + { ngx_string("Content-Length"), ngx_string("$proxy_internal_body_length") }, + { ngx_string("Transfer-Encoding"), ngx_string("") }, { ngx_string("Keep-Alive"), ngx_string("") }, { ngx_string("Expect"), ngx_string("") }, { ngx_string("Upgrade"), ngx_string("") }, @@ -1006,6 +1007,9 @@ ngx_http_proxy_create_request(ngx_http_r ctx->internal_body_length = body_len; len += body_len; + + } else { + ctx->internal_body_length = r->headers_in.content_length_n; } le.ip = plcf->headers_set_len->elts; @@ -1252,7 +1256,7 @@ ngx_http_proxy_reinit_request(ngx_http_r ctx->status.count = 0; ctx->status.start = NULL; ctx->status.end = NULL; - ctx->state = 0; + ctx->chunked.state = 0; r->upstream->process_header = ngx_http_proxy_process_status_line; r->upstream->pipe->input_filter = ngx_http_proxy_copy_filter; @@ -1617,265 +1621,6 @@ ngx_http_proxy_copy_filter(ngx_event_pip } -static ngx_inline ngx_int_t -ngx_http_proxy_parse_chunked(ngx_http_request_t *r, ngx_buf_t *buf) -{ - u_char *pos, ch, c; - ngx_int_t rc; - ngx_http_proxy_ctx_t *ctx; - enum { - sw_chunk_start = 0, - sw_chunk_size, - sw_chunk_extension, - sw_chunk_extension_almost_done, - sw_chunk_data, - sw_after_data, - sw_after_data_almost_done, - sw_last_chunk_extension, - sw_last_chunk_extension_almost_done, - sw_trailer, - sw_trailer_almost_done, - sw_trailer_header, - sw_trailer_header_almost_done - } state; - - ctx = ngx_http_get_module_ctx(r, ngx_http_proxy_module); - - if (ctx == NULL) { - return NGX_ERROR; - } - - state = ctx->state; - - if (state == sw_chunk_data && ctx->size == 0) { - state = sw_after_data; - } - - rc = NGX_AGAIN; - - for (pos = buf->pos; pos < buf->last; pos++) { - - ch = *pos; - - ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, - "http proxy chunked byte: %02Xd s:%d", ch, state); - - switch (state) { - - case sw_chunk_start: - if (ch >= '0' && ch <= '9') { - state = sw_chunk_size; - ctx->size = ch - '0'; - break; - } - - c = (u_char) (ch | 0x20); - - if (c >= 'a' && c <= 'f') { - state = sw_chunk_size; - ctx->size = c - 'a' + 10; - break; - } - - goto invalid; - - case sw_chunk_size: - if (ch >= '0' && ch <= '9') { - ctx->size = ctx->size * 16 + (ch - '0'); - break; - } - - c = (u_char) (ch | 0x20); - - if (c >= 'a' && c <= 'f') { - ctx->size = ctx->size * 16 + (c - 'a' + 10); - break; - } - - if (ctx->size == 0) { - - switch (ch) { - case CR: - state = sw_last_chunk_extension_almost_done; - break; - case LF: - state = sw_trailer; - break; - case ';': - case ' ': - case '\t': - state = sw_last_chunk_extension; - break; - default: - goto invalid; - } - - break; - } - - switch (ch) { - case CR: - state = sw_chunk_extension_almost_done; - break; - case LF: - state = sw_chunk_data; - break; - case ';': - case ' ': - case '\t': - state = sw_chunk_extension; - break; - default: - goto invalid; - } - - break; - - case sw_chunk_extension: - switch (ch) { - case CR: - state = sw_chunk_extension_almost_done; - break; - case LF: - state = sw_chunk_data; - } - break; - - case sw_chunk_extension_almost_done: - if (ch == LF) { - state = sw_chunk_data; - break; - } - goto invalid; - - case sw_chunk_data: - rc = NGX_OK; - goto data; - - case sw_after_data: - switch (ch) { - case CR: - state = sw_after_data_almost_done; - break; - case LF: - state = sw_chunk_start; - } - break; - - case sw_after_data_almost_done: - if (ch == LF) { - state = sw_chunk_start; - break; - } - goto invalid; - - case sw_last_chunk_extension: - switch (ch) { - case CR: - state = sw_last_chunk_extension_almost_done; - break; - case LF: - state = sw_trailer; - } - break; - - case sw_last_chunk_extension_almost_done: - if (ch == LF) { - state = sw_trailer; - break; - } - goto invalid; - - case sw_trailer: - switch (ch) { - case CR: - state = sw_trailer_almost_done; - break; - case LF: - goto done; - default: - state = sw_trailer_header; - } - break; - - case sw_trailer_almost_done: - if (ch == LF) { - goto done; - } - goto invalid; - - case sw_trailer_header: - switch (ch) { - case CR: - state = sw_trailer_header_almost_done; - break; - case LF: - state = sw_trailer; - } - break; - - case sw_trailer_header_almost_done: - if (ch == LF) { - state = sw_trailer; - break; - } - goto invalid; - - } - } - -data: - - ctx->state = state; - buf->pos = pos; - - switch (state) { - - case sw_chunk_start: - ctx->length = 3 /* "0" LF LF */; - break; - case sw_chunk_size: - ctx->length = 2 /* LF LF */ - + (ctx->size ? ctx->size + 4 /* LF "0" LF LF */ : 0); - break; - case sw_chunk_extension: - case sw_chunk_extension_almost_done: - ctx->length = 1 /* LF */ + ctx->size + 4 /* LF "0" LF LF */; - break; - case sw_chunk_data: - ctx->length = ctx->size + 4 /* LF "0" LF LF */; - break; - case sw_after_data: - case sw_after_data_almost_done: - ctx->length = 4 /* LF "0" LF LF */; - break; - case sw_last_chunk_extension: - case sw_last_chunk_extension_almost_done: - ctx->length = 2 /* LF LF */; - break; - case sw_trailer: - case sw_trailer_almost_done: - ctx->length = 1 /* LF */; - break; - case sw_trailer_header: - case sw_trailer_header_almost_done: - ctx->length = 2 /* LF LF */; - break; - - } - - return rc; - -done: - - return NGX_DONE; - -invalid: - - return NGX_ERROR; -} - - static ngx_int_t ngx_http_proxy_chunked_filter(ngx_event_pipe_t *p, ngx_buf_t *buf) { @@ -1901,7 +1646,7 @@ ngx_http_proxy_chunked_filter(ngx_event_ for ( ;; ) { - rc = ngx_http_proxy_parse_chunked(r, buf); + rc = ngx_http_parse_chunked(r, buf, &ctx->chunked); if (rc == NGX_OK) { @@ -1952,16 +1697,16 @@ ngx_http_proxy_chunked_filter(ngx_event_ ngx_log_debug2(NGX_LOG_DEBUG_EVENT, p->log, 0, "input buf #%d %p", b->num, b->pos); - if (buf->last - buf->pos >= ctx->size) { - - buf->pos += ctx->size; + if (buf->last - buf->pos >= ctx->chunked.size) { + + buf->pos += ctx->chunked.size; b->last = buf->pos; - ctx->size = 0; + ctx->chunked.size = 0; continue; } - ctx->size -= buf->last - buf->pos; + ctx->chunked.size -= buf->last - buf->pos; buf->pos = buf->last; b->last = buf->last; @@ -1982,7 +1727,7 @@ ngx_http_proxy_chunked_filter(ngx_event_ /* set p->length, minimal amount of data we want to see */ - p->length = ctx->length; + p->length = ctx->chunked.length; break; } @@ -1997,7 +1742,7 @@ ngx_http_proxy_chunked_filter(ngx_event_ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http proxy chunked state %d, length %d", - ctx->state, p->length); + ctx->chunked.state, p->length); if (b) { b->shadow = buf; @@ -2094,7 +1839,7 @@ ngx_http_proxy_non_buffered_chunked_filt for ( ;; ) { - rc = ngx_http_proxy_parse_chunked(r, buf); + rc = ngx_http_parse_chunked(r, buf, &ctx->chunked); if (rc == NGX_OK) { @@ -2116,13 +1861,13 @@ ngx_http_proxy_non_buffered_chunked_filt b->pos = buf->pos; b->tag = u->output.tag; - if (buf->last - buf->pos >= ctx->size) { - buf->pos += ctx->size; + if (buf->last - buf->pos >= ctx->chunked.size) { + buf->pos += ctx->chunked.size; b->last = buf->pos; - ctx->size = 0; + ctx->chunked.size = 0; } else { - ctx->size -= buf->last - buf->pos; + ctx->chunked.size -= buf->last - buf->pos; buf->pos = buf->last; b->last = buf->last; } @@ -2301,7 +2046,7 @@ ngx_http_proxy_internal_body_length_vari ctx = ngx_http_get_module_ctx(r, ngx_http_proxy_module); - if (ctx == NULL) { + if (ctx == NULL || ctx->internal_body_length < 0) { v->not_found = 1; return NGX_OK; } @@ -2310,13 +2055,13 @@ ngx_http_proxy_internal_body_length_vari v->no_cacheable = 0; v->not_found = 0; - v->data = ngx_pnalloc(r->connection->pool, NGX_SIZE_T_LEN); + v->data = ngx_pnalloc(r->connection->pool, NGX_OFF_T_LEN); if (v->data == NULL) { return NGX_ERROR; } - v->len = ngx_sprintf(v->data, "%uz", ctx->internal_body_length) - v->data; + v->len = ngx_sprintf(v->data, "%O", ctx->internal_body_length) - v->data; return NGX_OK; } @@ -3084,8 +2829,6 @@ ngx_http_proxy_merge_headers(ngx_conf_t } if (conf->headers_set_hash.buckets - && ((conf->body_source.data == NULL) - == (prev->body_source.data == NULL)) #if (NGX_HTTP_CACHE) && ((conf->upstream.cache == NULL) == (prev->upstream.cache == NULL)) #endif @@ -3168,16 +2911,6 @@ ngx_http_proxy_merge_headers(ngx_conf_t h++; } - if (conf->body_source.data) { - s = ngx_array_push(&headers_merged); - if (s == NULL) { - return NGX_ERROR; - } - - ngx_str_set(&s->key, "Content-Length"); - ngx_str_set(&s->value, "$proxy_internal_body_length"); - } - src = headers_merged.elts; for (i = 0; i < headers_merged.nelts; i++) { diff --git a/src/http/modules/ngx_http_scgi_module.c b/src/http/modules/ngx_http_scgi_module.c --- a/src/http/modules/ngx_http_scgi_module.c +++ b/src/http/modules/ngx_http_scgi_module.c @@ -533,10 +533,11 @@ ngx_http_scgi_create_key(ngx_http_reques static ngx_int_t ngx_http_scgi_create_request(ngx_http_request_t *r) { + off_t content_length_n; u_char ch, *key, *val, *lowcase_key; size_t len, key_len, val_len, allocated; ngx_buf_t *b; - ngx_str_t *content_length; + ngx_str_t content_length; ngx_uint_t i, n, hash, skip_empty, header_params; ngx_chain_t *cl, *body; ngx_list_part_t *part; @@ -545,12 +546,20 @@ ngx_http_scgi_create_request(ngx_http_re ngx_http_script_engine_t e, le; ngx_http_scgi_loc_conf_t *scf; ngx_http_script_len_code_pt lcode; - static ngx_str_t zero = ngx_string("0"); + u_char buffer[NGX_OFF_T_LEN]; + + content_length_n = 0; + body = r->upstream->request_bufs; - content_length = r->headers_in.content_length ? - &r->headers_in.content_length->value : &zero; + while (body) { + content_length_n += ngx_buf_size(body->buf); + body = body->next; + } - len = sizeof("CONTENT_LENGTH") + content_length->len + 1; + content_length.data = buffer; + content_length.len = ngx_sprintf(buffer, "%O", content_length_n) - buffer; + + len = sizeof("CONTENT_LENGTH") + content_length.len + 1; header_params = 0; ignored = NULL; @@ -672,11 +681,8 @@ ngx_http_scgi_create_request(ngx_http_re cl->buf = b; - b->last = ngx_snprintf(b->last, - NGX_SIZE_T_LEN + 1 + sizeof("CONTENT_LENGTH") - + NGX_OFF_T_LEN + 1, - "%ui:CONTENT_LENGTH%Z%V%Z", - len, content_length); + b->last = ngx_sprintf(b->last, "%ui:CONTENT_LENGTH%Z%V%Z", + len, &content_length); if (scf->params_len) { ngx_memzero(&e, sizeof(ngx_http_script_engine_t)); diff --git a/src/http/modules/perl/nginx.pm b/src/http/modules/perl/nginx.pm --- a/src/http/modules/perl/nginx.pm +++ b/src/http/modules/perl/nginx.pm @@ -50,7 +50,7 @@ our @EXPORT = qw( HTTP_INSUFFICIENT_STORAGE ); -our $VERSION = '1.3.8'; +our $VERSION = '1.3.9'; require XSLoader; XSLoader::load('nginx', $VERSION); diff --git a/src/http/ngx_http.h b/src/http/ngx_http.h --- a/src/http/ngx_http.h +++ b/src/http/ngx_http.h @@ -18,6 +18,7 @@ typedef struct ngx_http_upstream_s ng typedef struct ngx_http_cache_s ngx_http_cache_t; typedef struct ngx_http_file_cache_s ngx_http_file_cache_t; typedef struct ngx_http_log_ctx_s ngx_http_log_ctx_t; +typedef struct ngx_http_chunked_s ngx_http_chunked_t; typedef ngx_int_t (*ngx_http_header_handler_pt)(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset); @@ -52,6 +53,13 @@ struct ngx_http_log_ctx_s { }; +struct ngx_http_chunked_s { + ngx_uint_t state; + off_t size; + off_t length; +}; + + typedef struct { ngx_uint_t http_version; ngx_uint_t code; @@ -92,6 +100,8 @@ ngx_int_t ngx_http_arg(ngx_http_request_ ngx_str_t *value); void ngx_http_split_args(ngx_http_request_t *r, ngx_str_t *uri, ngx_str_t *args); +ngx_int_t ngx_http_parse_chunked(ngx_http_request_t *r, ngx_buf_t *b, + ngx_http_chunked_t *ctx); ngx_int_t ngx_http_find_server_conf(ngx_http_request_t *r); diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c @@ -848,7 +848,8 @@ ngx_http_handler(ngx_http_request_t *r) break; } - r->lingering_close = (r->headers_in.content_length_n > 0); + r->lingering_close = (r->headers_in.content_length_n > 0 + || r->headers_in.chunked); r->phase_handler = 0; } else { diff --git a/src/http/ngx_http_header_filter_module.c b/src/http/ngx_http_header_filter_module.c --- a/src/http/ngx_http_header_filter_module.c +++ b/src/http/ngx_http_header_filter_module.c @@ -112,7 +112,7 @@ static ngx_str_t ngx_http_status_lines[] #define NGX_HTTP_OFF_5XX (NGX_HTTP_LAST_4XX - 400 + NGX_HTTP_OFF_4XX) ngx_string("500 Internal Server Error"), - ngx_string("501 Method Not Implemented"), + ngx_string("501 Not Implemented"), ngx_string("502 Bad Gateway"), ngx_string("503 Service Temporarily Unavailable"), ngx_string("504 Gateway Time-out"), diff --git a/src/http/ngx_http_parse.c b/src/http/ngx_http_parse.c --- a/src/http/ngx_http_parse.c +++ b/src/http/ngx_http_parse.c @@ -1818,3 +1818,259 @@ ngx_http_split_args(ngx_http_request_t * args->len = 0; } } + + +ngx_int_t +ngx_http_parse_chunked(ngx_http_request_t *r, ngx_buf_t *b, + ngx_http_chunked_t *ctx) +{ + u_char *pos, ch, c; + ngx_int_t rc; + enum { + sw_chunk_start = 0, + sw_chunk_size, + sw_chunk_extension, + sw_chunk_extension_almost_done, + sw_chunk_data, + sw_after_data, + sw_after_data_almost_done, + sw_last_chunk_extension, + sw_last_chunk_extension_almost_done, + sw_trailer, + sw_trailer_almost_done, + sw_trailer_header, + sw_trailer_header_almost_done + } state; + + state = ctx->state; + + if (state == sw_chunk_data && ctx->size == 0) { + state = sw_after_data; + } + + rc = NGX_AGAIN; + + for (pos = b->pos; pos < b->last; pos++) { + + ch = *pos; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http chunked byte: %02Xd s:%d", ch, state); + + switch (state) { + + case sw_chunk_start: + if (ch >= '0' && ch <= '9') { + state = sw_chunk_size; + ctx->size = ch - '0'; + break; + } + + c = (u_char) (ch | 0x20); + + if (c >= 'a' && c <= 'f') { + state = sw_chunk_size; + ctx->size = c - 'a' + 10; + break; + } + + goto invalid; + + case sw_chunk_size: + if (ch >= '0' && ch <= '9') { + ctx->size = ctx->size * 16 + (ch - '0'); + break; + } + + c = (u_char) (ch | 0x20); + + if (c >= 'a' && c <= 'f') { + ctx->size = ctx->size * 16 + (c - 'a' + 10); + break; + } + + if (ctx->size == 0) { + + switch (ch) { + case CR: + state = sw_last_chunk_extension_almost_done; + break; + case LF: + state = sw_trailer; + break; + case ';': + case ' ': + case '\t': + state = sw_last_chunk_extension; + break; + default: + goto invalid; + } + + break; + } + + switch (ch) { + case CR: + state = sw_chunk_extension_almost_done; + break; + case LF: + state = sw_chunk_data; + break; + case ';': + case ' ': + case '\t': + state = sw_chunk_extension; + break; + default: + goto invalid; + } + + break; + + case sw_chunk_extension: + switch (ch) { + case CR: + state = sw_chunk_extension_almost_done; + break; + case LF: + state = sw_chunk_data; + } + break; + + case sw_chunk_extension_almost_done: + if (ch == LF) { + state = sw_chunk_data; + break; + } + goto invalid; + + case sw_chunk_data: + rc = NGX_OK; + goto data; + + case sw_after_data: + switch (ch) { + case CR: + state = sw_after_data_almost_done; + break; + case LF: + state = sw_chunk_start; + } + break; + + case sw_after_data_almost_done: + if (ch == LF) { + state = sw_chunk_start; + break; + } + goto invalid; + + case sw_last_chunk_extension: + switch (ch) { + case CR: + state = sw_last_chunk_extension_almost_done; + break; + case LF: + state = sw_trailer; + } + break; + + case sw_last_chunk_extension_almost_done: + if (ch == LF) { + state = sw_trailer; + break; + } + goto invalid; + + case sw_trailer: + switch (ch) { + case CR: + state = sw_trailer_almost_done; + break; + case LF: + goto done; + default: + state = sw_trailer_header; + } + break; + + case sw_trailer_almost_done: + if (ch == LF) { + goto done; + } + goto invalid; + + case sw_trailer_header: + switch (ch) { + case CR: + state = sw_trailer_header_almost_done; + break; + case LF: + state = sw_trailer; + } + break; + + case sw_trailer_header_almost_done: + if (ch == LF) { + state = sw_trailer; + break; + } + goto invalid; + + } + } + +data: + + ctx->state = state; + b->pos = pos; + + switch (state) { + + case sw_chunk_start: + ctx->length = 3 /* "0" LF LF */; + break; + case sw_chunk_size: + ctx->length = 2 /* LF LF */ + + (ctx->size ? ctx->size + 4 /* LF "0" LF LF */ : 0); + break; + case sw_chunk_extension: + case sw_chunk_extension_almost_done: + ctx->length = 1 /* LF */ + ctx->size + 4 /* LF "0" LF LF */; + break; + case sw_chunk_data: + ctx->length = ctx->size + 4 /* LF "0" LF LF */; + break; + case sw_after_data: + case sw_after_data_almost_done: + ctx->length = 4 /* LF "0" LF LF */; + break; + case sw_last_chunk_extension: + case sw_last_chunk_extension_almost_done: + ctx->length = 2 /* LF LF */; + break; + case sw_trailer: + case sw_trailer_almost_done: + ctx->length = 1 /* LF */; + break; + case sw_trailer_header: + case sw_trailer_header_almost_done: + ctx->length = 2 /* LF LF */; + break; + + } + + return rc; + +done: + + ctx->state = 0; + b->pos = pos + 1; + + return NGX_DONE; + +invalid: + + return NGX_ERROR; +} diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -1574,19 +1574,11 @@ ngx_http_process_request_header(ngx_http if (r->headers_in.content_length_n == NGX_ERROR) { ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, "client sent invalid \"Content-Length\" header"); - ngx_http_finalize_request(r, NGX_HTTP_LENGTH_REQUIRED); + ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); return NGX_ERROR; } } - if (r->method & NGX_HTTP_PUT && r->headers_in.content_length_n == -1) { - ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, - "client sent %V method without \"Content-Length\" header", - &r->method_name); - ngx_http_finalize_request(r, NGX_HTTP_LENGTH_REQUIRED); - return NGX_ERROR; - } - if (r->method & NGX_HTTP_TRACE) { ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, "client sent TRACE method"); @@ -1594,14 +1586,25 @@ ngx_http_process_request_header(ngx_http return NGX_ERROR; } - if (r->headers_in.transfer_encoding - && ngx_strcasestrn(r->headers_in.transfer_encoding->value.data, - "chunked", 7 - 1)) - { - ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, - "client sent \"Transfer-Encoding: chunked\" header"); - ngx_http_finalize_request(r, NGX_HTTP_LENGTH_REQUIRED); - return NGX_ERROR; + if (r->headers_in.transfer_encoding) { + if (r->headers_in.transfer_encoding->value.len == 7 + && ngx_strncasecmp(r->headers_in.transfer_encoding->value.data, + (u_char *) "chunked", 7) == 0) + { + r->headers_in.content_length = NULL; + r->headers_in.content_length_n = -1; + r->headers_in.chunked = 1; + + } else if (r->headers_in.transfer_encoding->value.len != 8 + || ngx_strncasecmp(r->headers_in.transfer_encoding->value.data, + (u_char *) "identity", 8) != 0) + { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent unknown \"Transfer-Encoding\": \"%V\"", + &r->headers_in.transfer_encoding->value); + ngx_http_finalize_request(r, NGX_HTTP_NOT_IMPLEMENTED); + return NGX_ERROR; + } } if (r->headers_in.connection_type == NGX_HTTP_CONNECTION_KEEP_ALIVE) { diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h --- a/src/http/ngx_http_request.h +++ b/src/http/ngx_http_request.h @@ -224,6 +224,7 @@ typedef struct { time_t keep_alive_n; unsigned connection_type:2; + unsigned chunked:1; unsigned msie:1; unsigned msie6:1; unsigned opera:1; @@ -276,7 +277,9 @@ typedef struct { ngx_chain_t *bufs; ngx_buf_t *buf; off_t rest; - ngx_chain_t *to_write; + ngx_chain_t *free; + ngx_chain_t *busy; + ngx_http_chunked_t *chunked; ngx_http_client_body_handler_pt post_handler; } ngx_http_request_body_t; diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c --- a/src/http/ngx_http_request_body.c +++ b/src/http/ngx_http_request_body.c @@ -12,18 +12,21 @@ static void ngx_http_read_client_request_body_handler(ngx_http_request_t *r); static ngx_int_t ngx_http_do_read_client_request_body(ngx_http_request_t *r); -static ngx_int_t ngx_http_write_request_body(ngx_http_request_t *r, - ngx_chain_t *body); +static ngx_int_t ngx_http_write_request_body(ngx_http_request_t *r); static ngx_int_t ngx_http_read_discarded_request_body(ngx_http_request_t *r); +static ngx_int_t ngx_http_discard_request_body_filter(ngx_http_request_t *r, + ngx_buf_t *b); static ngx_int_t ngx_http_test_expect(ngx_http_request_t *r); +static ngx_int_t ngx_http_request_body_filter(ngx_http_request_t *r, + ngx_chain_t *in); +static ngx_int_t ngx_http_request_body_length_filter(ngx_http_request_t *r, + ngx_chain_t *in); +static ngx_int_t ngx_http_request_body_chunked_filter(ngx_http_request_t *r, + ngx_chain_t *in); +static ngx_int_t ngx_http_request_body_save_filter(ngx_http_request_t *r, + ngx_chain_t *in); -/* - * on completion ngx_http_read_client_request_body() adds to - * r->request_body->bufs one or two bufs: - * *) one memory buf that was preread in r->header_in; - * *) one memory or file buf that contains the rest of the body - */ ngx_int_t ngx_http_read_client_request_body(ngx_http_request_t *r, @@ -31,9 +34,8 @@ ngx_http_read_client_request_body(ngx_ht { size_t preread; ssize_t size; - ngx_buf_t *b; - ngx_chain_t *cl, **next; - ngx_temp_file_t *tf; + ngx_int_t rc; + ngx_chain_t out; ngx_http_request_body_t *rb; ngx_http_core_loc_conf_t *clcf; @@ -45,69 +47,36 @@ ngx_http_read_client_request_body(ngx_ht } if (ngx_http_test_expect(r) != NGX_OK) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; + rc = NGX_HTTP_INTERNAL_SERVER_ERROR; + goto done; } rb = ngx_pcalloc(r->pool, sizeof(ngx_http_request_body_t)); if (rb == NULL) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; - } - - r->request_body = rb; - - if (r->headers_in.content_length_n < 0) { - post_handler(r); - return NGX_OK; + rc = NGX_HTTP_INTERNAL_SERVER_ERROR; + goto done; } - clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); - - if (r->headers_in.content_length_n == 0) { - - if (r->request_body_in_file_only) { - tf = ngx_pcalloc(r->pool, sizeof(ngx_temp_file_t)); - if (tf == NULL) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; - } - - tf->file.fd = NGX_INVALID_FILE; - tf->file.log = r->connection->log; - tf->path = clcf->client_body_temp_path; - tf->pool = r->pool; - tf->warn = "a client request body is buffered to a temporary file"; - tf->log_level = r->request_body_file_log_level; - tf->persistent = r->request_body_in_persistent_file; - tf->clean = r->request_body_in_clean_file; - - if (r->request_body_file_group_access) { - tf->access = 0660; - } - - rb->temp_file = tf; - - if (ngx_create_temp_file(&tf->file, tf->path, tf->pool, - tf->persistent, tf->clean, tf->access) - != NGX_OK) - { - return NGX_HTTP_INTERNAL_SERVER_ERROR; - } - } - - post_handler(r); - - return NGX_OK; - } - - rb->post_handler = post_handler; - /* * set by ngx_pcalloc(): * * rb->bufs = NULL; * rb->buf = NULL; - * rb->rest = 0; + * rb->free = NULL; + * rb->busy = NULL; + * rb->chunked = NULL; */ + rb->rest = -1; + rb->post_handler = post_handler; + + r->request_body = rb; + + if (r->headers_in.content_length_n < 0 && !r->headers_in.chunked) { + post_handler(r); + return NGX_OK; + } + preread = r->header_in->last - r->header_in->pos; if (preread) { @@ -117,79 +86,70 @@ ngx_http_read_client_request_body(ngx_ht ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http client request body preread %uz", preread); - b = ngx_calloc_buf(r->pool); - if (b == NULL) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; + out.buf = r->header_in; + out.next = NULL; + + rc = ngx_http_request_body_filter(r, &out); + + if (rc != NGX_OK) { + goto done; } - b->temporary = 1; - b->start = r->header_in->pos; - b->pos = r->header_in->pos; - b->last = r->header_in->last; - b->end = r->header_in->end; + r->request_length += preread - (r->header_in->last - r->header_in->pos); - rb->bufs = ngx_alloc_chain_link(r->pool); - if (rb->bufs == NULL) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; + if (!r->headers_in.chunked + && rb->rest > 0 + && rb->rest <= (off_t) (r->header_in->end - r->header_in->last)) + { + /* the whole request body may be placed in r->header_in */ + + rb->buf = r->header_in; + r->read_event_handler = ngx_http_read_client_request_body_handler; + r->write_event_handler = ngx_http_request_empty_handler; + + rc = ngx_http_do_read_client_request_body(r); + goto done; } - rb->bufs->buf = b; - rb->bufs->next = NULL; + } else { + /* set rb->rest */ - rb->buf = b; - - if ((off_t) preread >= r->headers_in.content_length_n) { - - /* the whole request body was pre-read */ + if (ngx_http_request_body_filter(r, NULL) != NGX_OK) { + rc = NGX_HTTP_INTERNAL_SERVER_ERROR; + goto done; + } + } - r->header_in->pos += (size_t) r->headers_in.content_length_n; - r->request_length += r->headers_in.content_length_n; - b->last = r->header_in->pos; + if (rb->rest == 0) { + /* the whole request body was pre-read */ - if (r->request_body_in_file_only) { - if (ngx_http_write_request_body(r, rb->bufs) != NGX_OK) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; - } + if (r->request_body_in_file_only) { + if (ngx_http_write_request_body(r) != NGX_OK) { + rc = NGX_HTTP_INTERNAL_SERVER_ERROR; + goto done; } - - post_handler(r); - - return NGX_OK; } - /* - * to not consider the body as pipelined request in - * ngx_http_set_keepalive() - */ - r->header_in->pos = r->header_in->last; + post_handler(r); - r->request_length += preread; - - rb->rest = r->headers_in.content_length_n - preread; - - if (rb->rest <= (off_t) (b->end - b->last)) { - - /* the whole request body may be placed in r->header_in */ + return NGX_OK; + } - rb->to_write = rb->bufs; - - r->read_event_handler = ngx_http_read_client_request_body_handler; - - return ngx_http_do_read_client_request_body(r); - } + if (rb->rest < 0) { + ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, + "negative request body rest"); + rc = NGX_HTTP_INTERNAL_SERVER_ERROR; + goto done; + } - next = &rb->bufs->next; - - } else { - b = NULL; - rb->rest = r->headers_in.content_length_n; - next = &rb->bufs; - } + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); size = clcf->client_body_buffer_size; size += size >> 2; - if (rb->rest < size) { + /* TODO: honor r->request_body_in_single_buf */ + + if (!r->headers_in.chunked && rb->rest < size) { size = (ssize_t) rb->rest; if (r->request_body_in_single_buf) { @@ -198,44 +158,26 @@ ngx_http_read_client_request_body(ngx_ht } else { size = clcf->client_body_buffer_size; - - /* disable copying buffer for r->request_body_in_single_buf */ - b = NULL; } rb->buf = ngx_create_temp_buf(r->pool, size); if (rb->buf == NULL) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; - } - - cl = ngx_alloc_chain_link(r->pool); - if (cl == NULL) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; - } - - cl->buf = rb->buf; - cl->next = NULL; - - if (b && r->request_body_in_single_buf) { - size = b->last - b->pos; - ngx_memcpy(rb->buf->pos, b->pos, size); - rb->buf->last += size; - - next = &rb->bufs; - } - - *next = cl; - - if (r->request_body_in_file_only || r->request_body_in_single_buf) { - rb->to_write = rb->bufs; - - } else { - rb->to_write = rb->bufs->next ? rb->bufs->next : rb->bufs; + rc = NGX_HTTP_INTERNAL_SERVER_ERROR; + goto done; } r->read_event_handler = ngx_http_read_client_request_body_handler; + r->write_event_handler = ngx_http_request_empty_handler; - return ngx_http_do_read_client_request_body(r); + rc = ngx_http_do_read_client_request_body(r); + +done: + + if (rc >= NGX_HTTP_SPECIAL_RESPONSE) { + r->main->count--; + } + + return rc; } @@ -261,9 +203,12 @@ ngx_http_read_client_request_body_handle static ngx_int_t ngx_http_do_read_client_request_body(ngx_http_request_t *r) { + off_t rest; size_t size; ssize_t n; + ngx_int_t rc; ngx_buf_t *b; + ngx_chain_t *cl, out; ngx_connection_t *c; ngx_http_request_body_t *rb; ngx_http_core_loc_conf_t *clcf; @@ -278,18 +223,44 @@ ngx_http_do_read_client_request_body(ngx for ( ;; ) { if (rb->buf->last == rb->buf->end) { - if (ngx_http_write_request_body(r, rb->to_write) != NGX_OK) { + /* pass buffer to request body filter chain */ + + out.buf = rb->buf; + out.next = NULL; + + rc = ngx_http_request_body_filter(r, &out); + + if (rc != NGX_OK) { + return rc; + } + + /* write to file */ + + if (ngx_http_write_request_body(r) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } - rb->to_write = rb->bufs->next ? rb->bufs->next : rb->bufs; + /* update chains */ + + rc = ngx_http_request_body_filter(r, NULL); + + if (rc != NGX_OK) { + return rc; + } + + if (rb->busy != NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + rb->buf->pos = rb->buf->start; rb->buf->last = rb->buf->start; } size = rb->buf->end - rb->buf->last; + rest = rb->rest - (rb->buf->last - rb->buf->pos); - if ((off_t) size > rb->rest) { - size = (size_t) rb->rest; + if ((off_t) size > rest) { + size = (size_t) rest; } n = c->recv(c, rb->buf->last, size); @@ -312,9 +283,21 @@ ngx_http_do_read_client_request_body(ngx } rb->buf->last += n; - rb->rest -= n; r->request_length += n; + if (n == rest) { + /* pass buffer to request body filter chain */ + + out.buf = rb->buf; + out.next = NULL; + + rc = ngx_http_request_body_filter(r, &out); + + if (rc != NGX_OK) { + return rc; + } + } + if (rb->rest == 0) { break; } @@ -351,32 +334,24 @@ ngx_http_do_read_client_request_body(ngx /* save the last part */ - if (ngx_http_write_request_body(r, rb->to_write) != NGX_OK) { + if (ngx_http_write_request_body(r) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } - b = ngx_calloc_buf(r->pool); - if (b == NULL) { + cl = ngx_chain_get_free_buf(r->pool, &rb->free); + if (cl == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } + b = cl->buf; + + ngx_memzero(b, sizeof(ngx_buf_t)); + b->in_file = 1; - b->file_pos = 0; b->file_last = rb->temp_file->file.offset; b->file = &rb->temp_file->file; - if (rb->bufs->next) { - rb->bufs->next->buf = b; - - } else { - rb->bufs->buf = b; - } - } - - if (rb->bufs->next - && (r->request_body_in_file_only || r->request_body_in_single_buf)) - { - rb->bufs = rb->bufs->next; + rb->bufs = cl; } r->read_event_handler = ngx_http_block_reading; @@ -388,15 +363,19 @@ ngx_http_do_read_client_request_body(ngx static ngx_int_t -ngx_http_write_request_body(ngx_http_request_t *r, ngx_chain_t *body) +ngx_http_write_request_body(ngx_http_request_t *r) { ssize_t n; + ngx_chain_t *cl; ngx_temp_file_t *tf; ngx_http_request_body_t *rb; ngx_http_core_loc_conf_t *clcf; rb = r->request_body; + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http write client request body, bufs %p", rb->bufs); + if (rb->temp_file == NULL) { tf = ngx_pcalloc(r->pool, sizeof(ngx_temp_file_t)); if (tf == NULL) { @@ -419,9 +398,26 @@ ngx_http_write_request_body(ngx_http_req } rb->temp_file = tf; + + if (rb->bufs == NULL) { + /* empty body with r->request_body_in_file_only */ + + if (ngx_create_temp_file(&tf->file, tf->path, tf->pool, + tf->persistent, tf->clean, tf->access) + != NGX_OK) + { + return NGX_ERROR; + } + + return NGX_OK; + } } - n = ngx_write_chain_to_temp_file(rb->temp_file, body); + if (rb->bufs == NULL) { + return NGX_OK; + } + + n = ngx_write_chain_to_temp_file(rb->temp_file, rb->bufs); /* TODO: n == 0 or not complete and level event */ @@ -431,6 +427,14 @@ ngx_http_write_request_body(ngx_http_req rb->temp_file->offset += n; + /* mark all buffers as written */ + + for (cl = rb->bufs; cl; cl = cl->next) { + cl->buf->pos = cl->buf->last; + } + + rb->bufs = NULL; + return NGX_OK; } @@ -439,9 +443,10 @@ ngx_int_t ngx_http_discard_request_body(ngx_http_request_t *r) { ssize_t size; + ngx_int_t rc; ngx_event_t *rev; - if (r != r->main || r->discard_body) { + if (r != r->main || r->discard_body || r->request_body) { return NGX_OK; } @@ -457,37 +462,45 @@ ngx_http_discard_request_body(ngx_http_r ngx_del_timer(rev); } - if (r->headers_in.content_length_n <= 0 || r->request_body) { + if (r->headers_in.content_length_n <= 0 && !r->headers_in.chunked) { return NGX_OK; } size = r->header_in->last - r->header_in->pos; - if (size) { - if (r->headers_in.content_length_n > size) { - r->header_in->pos += size; - r->headers_in.content_length_n -= size; + if (size || r->headers_in.chunked) { + rc = ngx_http_discard_request_body_filter(r, r->header_in); - } else { - r->header_in->pos += (size_t) r->headers_in.content_length_n; - r->headers_in.content_length_n = 0; + if (rc != NGX_OK) { + return rc; + } + + if (r->headers_in.content_length_n == 0) { return NGX_OK; } } + rc = ngx_http_read_discarded_request_body(r); + + if (rc == NGX_OK) { + r->lingering_close = 0; + return NGX_OK; + } + + if (rc >= NGX_HTTP_SPECIAL_RESPONSE) { + return rc; + } + + /* rc == NGX_AGAIN */ + r->read_event_handler = ngx_http_discarded_request_body_handler; if (ngx_handle_read_event(rev, 0) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } - if (ngx_http_read_discarded_request_body(r) == NGX_OK) { - r->lingering_close = 0; - - } else { - r->count++; - r->discard_body = 1; - } + r->count++; + r->discard_body = 1; return NGX_OK; } @@ -535,6 +548,12 @@ ngx_http_discarded_request_body_handler( return; } + if (rc >= NGX_HTTP_SPECIAL_RESPONSE) { + c->error = 1; + ngx_http_finalize_request(r, NGX_ERROR); + return; + } + /* rc == NGX_AGAIN */ if (ngx_handle_read_event(rev, 0) != NGX_OK) { @@ -561,13 +580,19 @@ ngx_http_discarded_request_body_handler( static ngx_int_t ngx_http_read_discarded_request_body(ngx_http_request_t *r) { - size_t size; - ssize_t n; - u_char buffer[NGX_HTTP_DISCARD_BUFFER_SIZE]; + size_t size; + ssize_t n; + ngx_int_t rc; + ngx_buf_t b; + u_char buffer[NGX_HTTP_DISCARD_BUFFER_SIZE]; ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http read discarded body"); + ngx_memzero(&b, sizeof(ngx_buf_t)); + + b.temporary = 1; + for ( ;; ) { if (r->headers_in.content_length_n == 0) { r->read_event_handler = ngx_http_block_reading; @@ -578,9 +603,8 @@ ngx_http_read_discarded_request_body(ngx return NGX_AGAIN; } - size = (r->headers_in.content_length_n > NGX_HTTP_DISCARD_BUFFER_SIZE) ? - NGX_HTTP_DISCARD_BUFFER_SIZE: - (size_t) r->headers_in.content_length_n; + size = (size_t) ngx_min(r->headers_in.content_length_n, + NGX_HTTP_DISCARD_BUFFER_SIZE); n = r->connection->recv(r->connection, buffer, size); @@ -597,12 +621,108 @@ ngx_http_read_discarded_request_body(ngx return NGX_OK; } - r->headers_in.content_length_n -= n; + b.pos = buffer; + b.last = buffer + n; + + rc = ngx_http_discard_request_body_filter(r, &b); + + if (rc != NGX_OK) { + return rc; + } } } static ngx_int_t +ngx_http_discard_request_body_filter(ngx_http_request_t *r, ngx_buf_t *b) +{ + size_t size; + ngx_int_t rc; + ngx_http_request_body_t *rb; + + if (r->headers_in.chunked) { + + rb = r->request_body; + + if (rb == NULL) { + + rb = ngx_pcalloc(r->pool, sizeof(ngx_http_request_body_t)); + if (rb == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + rb->chunked = ngx_pcalloc(r->pool, sizeof(ngx_http_chunked_t)); + if (rb->chunked == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + r->request_body = rb; + } + + for ( ;; ) { + + rc = ngx_http_parse_chunked(r, b, rb->chunked); + + if (rc == NGX_OK) { + + /* a chunk has been parsed successfully */ + + size = b->last - b->pos; + + if ((off_t) size > rb->chunked->size) { + b->pos += rb->chunked->size; + rb->chunked->size = 0; + + } else { + rb->chunked->size -= size; + b->pos = b->last; + } + + continue; + } + + if (rc == NGX_DONE) { + + /* a whole response has been parsed successfully */ + + r->headers_in.content_length_n = 0; + break; + } + + if (rc == NGX_AGAIN) { + + /* set amount of data we want to see next time */ + + r->headers_in.content_length_n = rb->chunked->length; + break; + } + + /* invalid */ + + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "client sent invalid chunked body"); + + return NGX_HTTP_BAD_REQUEST; + } + + } else { + size = b->last - b->pos; + + if ((off_t) size > r->headers_in.content_length_n) { + b->pos += r->headers_in.content_length_n; + r->headers_in.content_length_n = 0; + + } else { + b->pos = b->last; + r->headers_in.content_length_n -= size; + } + } + + return NGX_OK; +} + + +static ngx_int_t ngx_http_test_expect(ngx_http_request_t *r) { ngx_int_t n; @@ -642,3 +762,278 @@ ngx_http_test_expect(ngx_http_request_t return NGX_ERROR; } + + +static ngx_int_t +ngx_http_request_body_filter(ngx_http_request_t *r, ngx_chain_t *in) +{ + if (r->headers_in.chunked) { + return ngx_http_request_body_chunked_filter(r, in); + + } else { + return ngx_http_request_body_length_filter(r, in); + } +} + + +static ngx_int_t +ngx_http_request_body_length_filter(ngx_http_request_t *r, ngx_chain_t *in) +{ + size_t size; + ngx_int_t rc; + ngx_buf_t *b; + ngx_chain_t *cl, *tl, *out, **ll; + ngx_http_request_body_t *rb; + + rb = r->request_body; + + if (rb->rest == -1) { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http request body content length filter"); + + rb->rest = r->headers_in.content_length_n; + } + + out = NULL; + ll = &out; + + for (cl = in; cl; cl = cl->next) { + + tl = ngx_chain_get_free_buf(r->pool, &rb->free); + if (tl == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + b = tl->buf; + + ngx_memzero(b, sizeof(ngx_buf_t)); + + b->temporary = 1; + b->tag = (ngx_buf_tag_t) &ngx_http_read_client_request_body; + b->start = cl->buf->start; + b->pos = cl->buf->pos; + b->last = cl->buf->last; + b->end = cl->buf->end; + + size = cl->buf->last - cl->buf->pos; + + if ((off_t) size < rb->rest) { + cl->buf->pos = cl->buf->last; + rb->rest -= size; + + } else { + cl->buf->pos += rb->rest; + rb->rest = 0; + b->last = cl->buf->pos; + b->last_buf = 1; + } + + *ll = tl; + ll = &tl->next; + } + + rc = ngx_http_request_body_save_filter(r, out); + + ngx_chain_update_chains(r->pool, &rb->free, &rb->busy, &out, + (ngx_buf_tag_t) &ngx_http_read_client_request_body); + + return rc; +} + + +static ngx_int_t +ngx_http_request_body_chunked_filter(ngx_http_request_t *r, ngx_chain_t *in) +{ + size_t size; + ngx_int_t rc; + ngx_buf_t *b; + ngx_chain_t *cl, *out, *tl, **ll; + ngx_http_request_body_t *rb; + ngx_http_core_loc_conf_t *clcf; + + rb = r->request_body; + + if (rb->rest == -1) { + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http request body chunked filter"); + + rb->chunked = ngx_pcalloc(r->pool, sizeof(ngx_http_chunked_t)); + if (rb->chunked == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + r->headers_in.content_length_n = 0; + rb->rest = 3; + } + + out = NULL; + ll = &out; + + for (cl = in; cl; cl = cl->next) { + + for ( ;; ) { + + ngx_log_debug7(NGX_LOG_DEBUG_EVENT, r->connection->log, 0, + "http body chunked buf " + "t:%d f:%d %p, pos %p, size: %z file: %O, size: %z", + cl->buf->temporary, cl->buf->in_file, + cl->buf->start, cl->buf->pos, + cl->buf->last - cl->buf->pos, + cl->buf->file_pos, + cl->buf->file_last - cl->buf->file_pos); + + rc = ngx_http_parse_chunked(r, cl->buf, rb->chunked); + + if (rc == NGX_OK) { + + /* a chunk has been parsed successfully */ + + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + + if (clcf->client_max_body_size + && clcf->client_max_body_size + < r->headers_in.content_length_n + rb->chunked->size) + { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "client intended to send too large chunked " + "body: %O bytes", + r->headers_in.content_length_n + + rb->chunked->size); + + r->lingering_close = 1; + + return NGX_HTTP_REQUEST_ENTITY_TOO_LARGE; + } + + tl = ngx_chain_get_free_buf(r->pool, &rb->free); + if (tl == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + b = tl->buf; + + ngx_memzero(b, sizeof(ngx_buf_t)); + + b->temporary = 1; + b->tag = (ngx_buf_tag_t) &ngx_http_read_client_request_body; + b->start = cl->buf->start; + b->pos = cl->buf->pos; + b->last = cl->buf->last; + b->end = cl->buf->end; + + *ll = tl; + ll = &tl->next; + + size = cl->buf->last - cl->buf->pos; + + if ((off_t) size > rb->chunked->size) { + cl->buf->pos += rb->chunked->size; + r->headers_in.content_length_n += rb->chunked->size; + rb->chunked->size = 0; + + } else { + rb->chunked->size -= size; + r->headers_in.content_length_n += size; + cl->buf->pos = cl->buf->last; + } + + b->last = cl->buf->pos; + + continue; + } + + if (rc == NGX_DONE) { + + /* a whole response has been parsed successfully */ + + rb->rest = 0; + + tl = ngx_chain_get_free_buf(r->pool, &rb->free); + if (tl == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + b = tl->buf; + + ngx_memzero(b, sizeof(ngx_buf_t)); + + b->last_buf = 1; + + *ll = tl; + ll = &tl->next; + + break; + } + + if (rc == NGX_AGAIN) { + + /* set rb->rest, amount of data we want to see next time */ + + rb->rest = rb->chunked->length; + + break; + } + + /* invalid */ + + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "client sent invalid chunked body"); + + return NGX_HTTP_BAD_REQUEST; + } + } + + rc = ngx_http_request_body_save_filter(r, out); + + ngx_chain_update_chains(r->pool, &rb->free, &rb->busy, &out, + (ngx_buf_tag_t) &ngx_http_read_client_request_body); + + return rc; +} + + +static ngx_int_t +ngx_http_request_body_save_filter(ngx_http_request_t *r, ngx_chain_t *in) +{ +#if (NGX_DEBUG) + ngx_chain_t *cl; +#endif + ngx_http_request_body_t *rb; + + rb = r->request_body; + +#if (NGX_DEBUG) + + for (cl = rb->bufs; cl; cl = cl->next) { + ngx_log_debug7(NGX_LOG_DEBUG_EVENT, r->connection->log, 0, + "http body old buf t:%d f:%d %p, pos %p, size: %z " + "file: %O, size: %z", + cl->buf->temporary, cl->buf->in_file, + cl->buf->start, cl->buf->pos, + cl->buf->last - cl->buf->pos, + cl->buf->file_pos, + cl->buf->file_last - cl->buf->file_pos); + } + + for (cl = in; cl; cl = cl->next) { + ngx_log_debug7(NGX_LOG_DEBUG_EVENT, r->connection->log, 0, + "http body new buf t:%d f:%d %p, pos %p, size: %z " + "file: %O, size: %z", + cl->buf->temporary, cl->buf->in_file, + cl->buf->start, cl->buf->pos, + cl->buf->last - cl->buf->pos, + cl->buf->file_pos, + cl->buf->file_last - cl->buf->file_pos); + } + +#endif + + /* TODO: coalesce neighbouring buffers */ + + if (ngx_chain_add_copy(r->pool, &rb->bufs, in) != NGX_OK) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + return NGX_OK; +} diff --git a/src/http/ngx_http_special_response.c b/src/http/ngx_http_special_response.c --- a/src/http/ngx_http_special_response.c +++ b/src/http/ngx_http_special_response.c @@ -260,9 +260,9 @@ static char ngx_http_error_500_page[] = static char ngx_http_error_501_page[] = "" CRLF -"501 Method Not Implemented" CRLF +"501 Not Implemented" CRLF "" CRLF -"

501 Method Not Implemented

" CRLF +"

501 Not Implemented

" CRLF ; @@ -384,6 +384,7 @@ ngx_http_special_response_handler(ngx_ht case NGX_HTTPS_CERT_ERROR: case NGX_HTTPS_NO_CERT: case NGX_HTTP_INTERNAL_SERVER_ERROR: + case NGX_HTTP_NOT_IMPLEMENTED: r->keepalive = 0; } } @@ -420,7 +421,7 @@ ngx_http_special_response_handler(ngx_ht r->expect_tested = 1; if (ngx_http_discard_request_body(r) != NGX_OK) { - error = NGX_HTTP_INTERNAL_SERVER_ERROR; + r->keepalive = 0; } if (clcf->msie_refresh diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -1809,9 +1809,16 @@ ngx_http_upstream_test_connect(ngx_conne #if (NGX_HAVE_KQUEUE) if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) { - if (c->write->pending_eof) { + if (c->write->pending_eof || c->read->pending_eof) { + if (c->write->pending_eof) { + err = c->write->kq_errno; + + } else { + err = c->read->kq_errno; + } + c->log->action = "connecting to upstream"; - (void) ngx_connection_error(c, c->write->kq_errno, + (void) ngx_connection_error(c, err, "kevent() reported that connect() failed"); return NGX_ERROR; } diff --git a/src/http/ngx_http_upstream_round_robin.c b/src/http/ngx_http_upstream_round_robin.c --- a/src/http/ngx_http_upstream_round_robin.c +++ b/src/http/ngx_http_upstream_round_robin.c @@ -430,6 +430,10 @@ ngx_http_upstream_get_round_robin_peer(n if (rrp->peers->single) { peer = &rrp->peers->peer[0]; + if (peer->down) { + goto failed; + } + } else { /* there are several peers */ diff --git a/src/http/ngx_http_variables.c b/src/http/ngx_http_variables.c --- a/src/http/ngx_http_variables.c +++ b/src/http/ngx_http_variables.c @@ -39,6 +39,8 @@ static ngx_int_t ngx_http_variable_tcpin ngx_http_variable_value_t *v, uintptr_t data); #endif +static ngx_int_t ngx_http_variable_content_length(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_http_variable_host(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_http_variable_binary_remote_addr(ngx_http_request_t *r, @@ -79,6 +81,8 @@ static ngx_int_t ngx_http_variable_reque ngx_http_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_http_variable_request_body_file(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data); +static ngx_int_t ngx_http_variable_request_time(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_http_variable_status(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data); @@ -108,6 +112,8 @@ static ngx_int_t ngx_http_variable_hostn ngx_http_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_http_variable_pid(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data); +static ngx_int_t ngx_http_variable_msec(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data); /* * TODO: @@ -149,8 +155,8 @@ static ngx_http_variable_t ngx_http_cor { ngx_string("http_cookie"), NULL, ngx_http_variable_headers, offsetof(ngx_http_request_t, headers_in.cookies), 0, 0 }, - { ngx_string("content_length"), NULL, ngx_http_variable_header, - offsetof(ngx_http_request_t, headers_in.content_length), 0, 0 }, + { ngx_string("content_length"), NULL, ngx_http_variable_content_length, + 0, 0, 0 }, { ngx_string("content_type"), NULL, ngx_http_variable_header, offsetof(ngx_http_request_t, headers_in.content_type), 0, 0 }, @@ -237,6 +243,9 @@ static ngx_http_variable_t ngx_http_cor ngx_http_variable_request_body_file, 0, 0, 0 }, + { ngx_string("request_time"), NULL, ngx_http_variable_request_time, + 0, NGX_HTTP_VAR_NOCACHEABLE, 0 }, + { ngx_string("status"), NULL, ngx_http_variable_status, 0, NGX_HTTP_VAR_NOCACHEABLE, 0 }, @@ -285,6 +294,9 @@ static ngx_http_variable_t ngx_http_cor { ngx_string("pid"), NULL, ngx_http_variable_pid, 0, 0, 0 }, + { ngx_string("msec"), NULL, ngx_http_variable_msec, + 0, NGX_HTTP_VAR_NOCACHEABLE, 0 }, + #if (NGX_HAVE_TCP_INFO) { ngx_string("tcpinfo_rtt"), NULL, ngx_http_variable_tcpinfo, 0, NGX_HTTP_VAR_NOCACHEABLE, 0 }, @@ -980,6 +992,39 @@ ngx_http_variable_tcpinfo(ngx_http_reque static ngx_int_t +ngx_http_variable_content_length(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data) +{ + u_char *p; + + if (r->headers_in.content_length) { + v->len = r->headers_in.content_length->value.len; + v->data = r->headers_in.content_length->value.data; + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + + } else if (r->headers_in.content_length_n >= 0) { + p = ngx_pnalloc(r->pool, NGX_OFF_T_LEN); + if (p == NULL) { + return NGX_ERROR; + } + + v->len = ngx_sprintf(p, "%O", r->headers_in.content_length_n) - p; + v->data = p; + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + + } else { + v->not_found = 1; + } + + return NGX_OK; +} + + +static ngx_int_t ngx_http_variable_host(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data) { @@ -1757,7 +1802,7 @@ ngx_http_variable_request_body(ngx_http_ { u_char *p; size_t len; - ngx_buf_t *buf, *next; + ngx_buf_t *buf; ngx_chain_t *cl; if (r->request_body == NULL @@ -1782,8 +1827,13 @@ ngx_http_variable_request_body(ngx_http_ return NGX_OK; } - next = cl->next->buf; - len = (buf->last - buf->pos) + (next->last - next->pos); + len = buf->last - buf->pos; + cl = cl->next; + + for ( /* void */ ; cl; cl = cl->next) { + buf = cl->buf; + len += buf->last - buf->pos; + } p = ngx_pnalloc(r->pool, len); if (p == NULL) { @@ -1791,9 +1841,12 @@ ngx_http_variable_request_body(ngx_http_ } v->data = p; - - p = ngx_cpymem(p, buf->pos, buf->last - buf->pos); - ngx_memcpy(p, next->pos, next->last - next->pos); + cl = r->request_body->bufs; + + for ( /* void */ ; cl; cl = cl->next) { + buf = cl->buf; + p = ngx_cpymem(p, buf->pos, buf->last - buf->pos); + } v->len = len; v->valid = 1; @@ -1825,6 +1878,35 @@ ngx_http_variable_request_body_file(ngx_ static ngx_int_t +ngx_http_variable_request_time(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data) +{ + u_char *p; + ngx_time_t *tp; + ngx_msec_int_t ms; + + p = ngx_pnalloc(r->pool, NGX_TIME_T_LEN + 4); + if (p == NULL) { + return NGX_ERROR; + } + + tp = ngx_timeofday(); + + ms = (ngx_msec_int_t) + ((tp->sec - r->start_sec) * 1000 + (tp->msec - r->start_msec)); + ms = ngx_max(ms, 0); + + v->len = ngx_sprintf(p, "%T.%03M", ms / 1000, ms % 1000) - p; + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + v->data = p; + + return NGX_OK; +} + + +static ngx_int_t ngx_http_variable_connection(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data) { @@ -1915,6 +1997,30 @@ ngx_http_variable_pid(ngx_http_request_t } +static ngx_int_t +ngx_http_variable_msec(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data) +{ + u_char *p; + ngx_time_t *tp; + + p = ngx_pnalloc(r->pool, NGX_TIME_T_LEN + 4); + if (p == NULL) { + return NGX_ERROR; + } + + tp = ngx_timeofday(); + + v->len = ngx_sprintf(p, "%T.%03M", tp->sec, tp->msec) - p; + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + v->data = p; + + return NGX_OK; +} + + void * ngx_http_map_find(ngx_http_request_t *r, ngx_http_map_t *map, ngx_str_t *match) { diff --git a/src/os/unix/ngx_files.c b/src/os/unix/ngx_files.c --- a/src/os/unix/ngx_files.c +++ b/src/os/unix/ngx_files.c @@ -241,8 +241,12 @@ ngx_write_chain_to_file(ngx_file_t *file return NGX_ERROR; } + ngx_log_debug2(NGX_LOG_DEBUG_CORE, file->log, 0, + "writev: %d, %z", file->fd, n); + file->sys_offset += n; file->offset += n; + offset += n; total += n; } while (cl); diff --git a/src/os/unix/ngx_process_cycle.c b/src/os/unix/ngx_process_cycle.c --- a/src/os/unix/ngx_process_cycle.c +++ b/src/os/unix/ngx_process_cycle.c @@ -20,7 +20,7 @@ static void ngx_signal_worker_processes( static ngx_uint_t ngx_reap_children(ngx_cycle_t *cycle); static void ngx_master_process_exit(ngx_cycle_t *cycle); static void ngx_worker_process_cycle(ngx_cycle_t *cycle, void *data); -static void ngx_worker_process_init(ngx_cycle_t *cycle, ngx_uint_t priority); +static void ngx_worker_process_init(ngx_cycle_t *cycle, ngx_int_t worker); static void ngx_worker_process_exit(ngx_cycle_t *cycle); static void ngx_channel_handler(ngx_event_t *ev); #if (NGX_THREADS) @@ -62,7 +62,6 @@ ngx_int_t ngx_threads_n; #endif -uint64_t cpu_affinity; static u_char master_process[] = "master process"; @@ -360,10 +359,8 @@ ngx_start_worker_processes(ngx_cycle_t * for (i = 0; i < n; i++) { - cpu_affinity = ngx_get_cpu_affinity(i); - - ngx_spawn_process(cycle, ngx_worker_process_cycle, NULL, - "worker process", type); + ngx_spawn_process(cycle, ngx_worker_process_cycle, + (void *) (intptr_t) i, "worker process", type); ch.pid = ngx_processes[ngx_process_slot].pid; ch.slot = ngx_process_slot; @@ -371,8 +368,6 @@ ngx_start_worker_processes(ngx_cycle_t * ngx_pass_open_channel(cycle, &ch); } - - cpu_affinity = 0; } @@ -726,12 +721,14 @@ ngx_master_process_exit(ngx_cycle_t *cyc static void ngx_worker_process_cycle(ngx_cycle_t *cycle, void *data) { + ngx_int_t worker = (intptr_t) data; + ngx_uint_t i; ngx_connection_t *c; ngx_process = NGX_PROCESS_WORKER; - ngx_worker_process_init(cycle, 1); + ngx_worker_process_init(cycle, worker); ngx_setproctitle("worker process"); @@ -837,9 +834,10 @@ ngx_worker_process_cycle(ngx_cycle_t *cy static void -ngx_worker_process_init(ngx_cycle_t *cycle, ngx_uint_t priority) +ngx_worker_process_init(ngx_cycle_t *cycle, ngx_int_t worker) { sigset_t set; + uint64_t cpu_affinity; ngx_int_t n; ngx_uint_t i; struct rlimit rlmt; @@ -853,7 +851,7 @@ ngx_worker_process_init(ngx_cycle_t *cyc ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module); - if (priority && ccf->priority != 0) { + if (worker >= 0 && ccf->priority != 0) { if (setpriority(PRIO_PROCESS, 0, ccf->priority) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "setpriority(%d) failed", ccf->priority); @@ -917,8 +915,12 @@ ngx_worker_process_init(ngx_cycle_t *cyc } } - if (cpu_affinity) { - ngx_setaffinity(cpu_affinity, cycle->log); + if (worker >= 0) { + cpu_affinity = ngx_get_cpu_affinity(worker); + + if (cpu_affinity) { + ngx_setaffinity(cpu_affinity, cycle->log); + } } #if (NGX_HAVE_PR_SET_DUMPABLE) @@ -1294,13 +1296,18 @@ ngx_cache_manager_process_cycle(ngx_cycl void *ident[4]; ngx_event_t ev; - cycle->connection_n = 512; - + /* + * Set correct process type since closing listening Unix domain socket + * in a master process also removes the Unix domain socket file. + */ ngx_process = NGX_PROCESS_HELPER; - ngx_worker_process_init(cycle, 0); + ngx_close_listening_sockets(cycle); - ngx_close_listening_sockets(cycle); + /* Set a moderate number of connections for a helper process. */ + cycle->connection_n = 512; + + ngx_worker_process_init(cycle, -1); ngx_memzero(&ev, sizeof(ngx_event_t)); ev.handler = ctx->handler;