# HG changeset patch # User Maxim Dounin # Date 1384793302 -14400 # Node ID 43ccaf8e8728f1709640d6e507752554c0b6115a # Parent cbb9a6c7493c3c01323fbc4a61be4a9f0af55ef2 Upstream: cache revalidation with conditional requests. The following new directives are introduced: proxy_cache_revalidate, fastcgi_cache_revalidate, scgi_cache_revalidate, uwsgi_cache_revalidate. Default is off. When set to on, they enable cache revalidation using conditional requests with If-Modified-Since for expired cache items. As of now, no attempts are made to merge headers given in a 304 response during cache revalidation with headers previously stored in a cache item. Headers in a 304 response are only used to calculate new validity time of a cache item. diff --git a/src/http/modules/ngx_http_fastcgi_module.c b/src/http/modules/ngx_http_fastcgi_module.c --- a/src/http/modules/ngx_http_fastcgi_module.c +++ b/src/http/modules/ngx_http_fastcgi_module.c @@ -405,6 +405,13 @@ static ngx_command_t ngx_http_fastcgi_c offsetof(ngx_http_fastcgi_loc_conf_t, upstream.cache_lock_timeout), NULL }, + { ngx_string("fastcgi_cache_revalidate"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_fastcgi_loc_conf_t, upstream.cache_revalidate), + NULL }, + #endif { ngx_string("fastcgi_temp_path"), @@ -563,7 +570,8 @@ static ngx_str_t ngx_http_fastcgi_hide_ #if (NGX_HTTP_CACHE) static ngx_keyval_t ngx_http_fastcgi_cache_headers[] = { - { ngx_string("HTTP_IF_MODIFIED_SINCE"), ngx_string("") }, + { ngx_string("HTTP_IF_MODIFIED_SINCE"), + ngx_string("$upstream_cache_last_modified") }, { ngx_string("HTTP_IF_UNMODIFIED_SINCE"), ngx_string("") }, { ngx_string("HTTP_IF_NONE_MATCH"), ngx_string("") }, { ngx_string("HTTP_IF_MATCH"), ngx_string("") }, @@ -2336,6 +2344,7 @@ ngx_http_fastcgi_create_loc_conf(ngx_con conf->upstream.cache_valid = NGX_CONF_UNSET_PTR; conf->upstream.cache_lock = NGX_CONF_UNSET; conf->upstream.cache_lock_timeout = NGX_CONF_UNSET_MSEC; + conf->upstream.cache_revalidate = NGX_CONF_UNSET; #endif conf->upstream.hide_headers = NGX_CONF_UNSET_PTR; @@ -2582,6 +2591,9 @@ ngx_http_fastcgi_merge_loc_conf(ngx_conf ngx_conf_merge_msec_value(conf->upstream.cache_lock_timeout, prev->upstream.cache_lock_timeout, 5000); + ngx_conf_merge_value(conf->upstream.cache_revalidate, + prev->upstream.cache_revalidate, 0); + #endif ngx_conf_merge_value(conf->upstream.pass_request_headers, diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c --- a/src/http/modules/ngx_http_proxy_module.c +++ b/src/http/modules/ngx_http_proxy_module.c @@ -465,6 +465,13 @@ static ngx_command_t ngx_http_proxy_com offsetof(ngx_http_proxy_loc_conf_t, upstream.cache_lock_timeout), NULL }, + { ngx_string("proxy_cache_revalidate"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_proxy_loc_conf_t, upstream.cache_revalidate), + NULL }, + #endif { ngx_string("proxy_temp_path"), @@ -622,7 +629,8 @@ static ngx_keyval_t ngx_http_proxy_cach { ngx_string("Keep-Alive"), ngx_string("") }, { ngx_string("Expect"), ngx_string("") }, { ngx_string("Upgrade"), ngx_string("") }, - { ngx_string("If-Modified-Since"), ngx_string("") }, + { ngx_string("If-Modified-Since"), + ngx_string("$upstream_cache_last_modified") }, { ngx_string("If-Unmodified-Since"), ngx_string("") }, { ngx_string("If-None-Match"), ngx_string("") }, { ngx_string("If-Match"), ngx_string("") }, @@ -2454,6 +2462,7 @@ ngx_http_proxy_create_loc_conf(ngx_conf_ conf->upstream.cache_valid = NGX_CONF_UNSET_PTR; conf->upstream.cache_lock = NGX_CONF_UNSET; conf->upstream.cache_lock_timeout = NGX_CONF_UNSET_MSEC; + conf->upstream.cache_revalidate = NGX_CONF_UNSET; #endif conf->upstream.hide_headers = NGX_CONF_UNSET_PTR; @@ -2710,6 +2719,9 @@ ngx_http_proxy_merge_loc_conf(ngx_conf_t ngx_conf_merge_msec_value(conf->upstream.cache_lock_timeout, prev->upstream.cache_lock_timeout, 5000); + ngx_conf_merge_value(conf->upstream.cache_revalidate, + prev->upstream.cache_revalidate, 0); + #endif ngx_conf_merge_str_value(conf->method, prev->method, ""); diff --git a/src/http/modules/ngx_http_scgi_module.c b/src/http/modules/ngx_http_scgi_module.c --- a/src/http/modules/ngx_http_scgi_module.c +++ b/src/http/modules/ngx_http_scgi_module.c @@ -262,6 +262,13 @@ static ngx_command_t ngx_http_scgi_comma offsetof(ngx_http_scgi_loc_conf_t, upstream.cache_lock_timeout), NULL }, + { ngx_string("scgi_cache_revalidate"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_scgi_loc_conf_t, upstream.cache_revalidate), + NULL }, + #endif { ngx_string("scgi_temp_path"), @@ -369,7 +376,8 @@ static ngx_str_t ngx_http_scgi_hide_head #if (NGX_HTTP_CACHE) static ngx_keyval_t ngx_http_scgi_cache_headers[] = { - { ngx_string("HTTP_IF_MODIFIED_SINCE"), ngx_string("") }, + { ngx_string("HTTP_IF_MODIFIED_SINCE"), + ngx_string("$upstream_cache_last_modified") }, { ngx_string("HTTP_IF_UNMODIFIED_SINCE"), ngx_string("") }, { ngx_string("HTTP_IF_NONE_MATCH"), ngx_string("") }, { ngx_string("HTTP_IF_MATCH"), ngx_string("") }, @@ -1093,6 +1101,7 @@ ngx_http_scgi_create_loc_conf(ngx_conf_t conf->upstream.cache_valid = NGX_CONF_UNSET_PTR; conf->upstream.cache_lock = NGX_CONF_UNSET; conf->upstream.cache_lock_timeout = NGX_CONF_UNSET_MSEC; + conf->upstream.cache_revalidate = NGX_CONF_UNSET; #endif conf->upstream.hide_headers = NGX_CONF_UNSET_PTR; @@ -1333,6 +1342,9 @@ ngx_http_scgi_merge_loc_conf(ngx_conf_t ngx_conf_merge_msec_value(conf->upstream.cache_lock_timeout, prev->upstream.cache_lock_timeout, 5000); + ngx_conf_merge_value(conf->upstream.cache_revalidate, + prev->upstream.cache_revalidate, 0); + #endif ngx_conf_merge_value(conf->upstream.pass_request_headers, diff --git a/src/http/modules/ngx_http_uwsgi_module.c b/src/http/modules/ngx_http_uwsgi_module.c --- a/src/http/modules/ngx_http_uwsgi_module.c +++ b/src/http/modules/ngx_http_uwsgi_module.c @@ -289,6 +289,13 @@ static ngx_command_t ngx_http_uwsgi_comm offsetof(ngx_http_uwsgi_loc_conf_t, upstream.cache_lock_timeout), NULL }, + { ngx_string("uwsgi_cache_revalidate"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_uwsgi_loc_conf_t, upstream.cache_revalidate), + NULL }, + #endif { ngx_string("uwsgi_temp_path"), @@ -402,7 +409,8 @@ static ngx_str_t ngx_http_uwsgi_hide_hea #if (NGX_HTTP_CACHE) static ngx_keyval_t ngx_http_uwsgi_cache_headers[] = { - { ngx_string("HTTP_IF_MODIFIED_SINCE"), ngx_string("") }, + { ngx_string("HTTP_IF_MODIFIED_SINCE"), + ngx_string("$upstream_cache_last_modified") }, { ngx_string("HTTP_IF_UNMODIFIED_SINCE"), ngx_string("") }, { ngx_string("HTTP_IF_NONE_MATCH"), ngx_string("") }, { ngx_string("HTTP_IF_MATCH"), ngx_string("") }, @@ -1130,6 +1138,7 @@ ngx_http_uwsgi_create_loc_conf(ngx_conf_ conf->upstream.cache_valid = NGX_CONF_UNSET_PTR; conf->upstream.cache_lock = NGX_CONF_UNSET; conf->upstream.cache_lock_timeout = NGX_CONF_UNSET_MSEC; + conf->upstream.cache_revalidate = NGX_CONF_UNSET; #endif conf->upstream.hide_headers = NGX_CONF_UNSET_PTR; @@ -1370,6 +1379,9 @@ ngx_http_uwsgi_merge_loc_conf(ngx_conf_t ngx_conf_merge_msec_value(conf->upstream.cache_lock_timeout, prev->upstream.cache_lock_timeout, 5000); + ngx_conf_merge_value(conf->upstream.cache_revalidate, + prev->upstream.cache_revalidate, 0); + #endif ngx_conf_merge_value(conf->upstream.pass_request_headers, diff --git a/src/http/ngx_http_cache.h b/src/http/ngx_http_cache.h --- a/src/http/ngx_http_cache.h +++ b/src/http/ngx_http_cache.h @@ -19,8 +19,9 @@ #define NGX_HTTP_CACHE_EXPIRED 3 #define NGX_HTTP_CACHE_STALE 4 #define NGX_HTTP_CACHE_UPDATING 5 -#define NGX_HTTP_CACHE_HIT 6 -#define NGX_HTTP_CACHE_SCARCE 7 +#define NGX_HTTP_CACHE_REVALIDATED 6 +#define NGX_HTTP_CACHE_HIT 7 +#define NGX_HTTP_CACHE_SCARCE 8 #define NGX_HTTP_CACHE_KEY_LEN 16 @@ -143,6 +144,7 @@ void ngx_http_file_cache_create_key(ngx_ ngx_int_t ngx_http_file_cache_open(ngx_http_request_t *r); void ngx_http_file_cache_set_header(ngx_http_request_t *r, u_char *buf); void ngx_http_file_cache_update(ngx_http_request_t *r, ngx_temp_file_t *tf); +void ngx_http_file_cache_update_header(ngx_http_request_t *r); ngx_int_t ngx_http_cache_send(ngx_http_request_t *); void ngx_http_file_cache_free(ngx_http_cache_t *c, ngx_temp_file_t *tf); time_t ngx_http_file_cache_valid(ngx_array_t *cache_valid, ngx_uint_t status); diff --git a/src/http/ngx_http_file_cache.c b/src/http/ngx_http_file_cache.c --- a/src/http/ngx_http_file_cache.c +++ b/src/http/ngx_http_file_cache.c @@ -53,6 +53,7 @@ ngx_str_t ngx_http_cache_status[] = { ngx_string("EXPIRED"), ngx_string("STALE"), ngx_string("UPDATING"), + ngx_string("REVALIDATED"), ngx_string("HIT") }; @@ -971,6 +972,116 @@ ngx_http_file_cache_update(ngx_http_requ } +void +ngx_http_file_cache_update_header(ngx_http_request_t *r) +{ + ssize_t n; + ngx_err_t err; + ngx_file_t file; + ngx_file_info_t fi; + ngx_http_cache_t *c; + ngx_http_file_cache_header_t h; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http file cache update header"); + + c = r->cache; + + ngx_memzero(&file, sizeof(ngx_file_t)); + + file.name = c->file.name; + file.log = r->connection->log; + file.fd = ngx_open_file(file.name.data, NGX_FILE_RDWR, NGX_FILE_OPEN, 0); + + if (file.fd == NGX_INVALID_FILE) { + err = ngx_errno; + + /* cache file may have been deleted */ + + if (err == NGX_ENOENT) { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http file cache \"%s\" not found", + file.name.data); + return; + } + + ngx_log_error(NGX_LOG_CRIT, r->connection->log, err, + ngx_open_file_n " \"%s\" failed", file.name.data); + return; + } + + /* + * make sure cache file wasn't replaced; + * if it was, do nothing + */ + + if (ngx_fd_info(file.fd, &fi) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_CRIT, r->connection->log, ngx_errno, + ngx_fd_info_n " \"%s\" failed", file.name.data); + goto done; + } + + if (c->uniq != ngx_file_uniq(&fi) + || c->length != ngx_file_size(&fi)) + { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http file cache \"%s\" changed", + file.name.data); + goto done; + } + + n = ngx_read_file(&file, (u_char *) &h, + sizeof(ngx_http_file_cache_header_t), 0); + + if (n == NGX_ERROR) { + goto done; + } + + if ((size_t) n != sizeof(ngx_http_file_cache_header_t)) { + ngx_log_error(NGX_LOG_CRIT, r->connection->log, 0, + ngx_read_file_n " read only %z of %z from \"%s\"", + n, sizeof(ngx_http_file_cache_header_t), file.name.data); + goto done; + } + + if (h.last_modified != c->last_modified + || h.crc32 != c->crc32 + || h.header_start != c->header_start + || h.body_start != c->body_start) + { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http file cache \"%s\" content changed", + file.name.data); + goto done; + } + + /* + * update cache file header with new data, + * notably h.valid_sec and h.date + */ + + ngx_memzero(&h, sizeof(ngx_http_file_cache_header_t)); + + h.valid_sec = c->valid_sec; + h.last_modified = c->last_modified; + h.date = c->date; + h.crc32 = c->crc32; + h.valid_msec = (u_short) c->valid_msec; + h.header_start = (u_short) c->header_start; + h.body_start = (u_short) c->body_start; + + (void) ngx_write_file(&file, (u_char *) &h, + sizeof(ngx_http_file_cache_header_t), 0); + +done: + + if (ngx_close_file(file.fd) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_ALERT, r->connection->log, ngx_errno, + ngx_close_file_n " \"%s\" failed", file.name.data); + } +} + + ngx_int_t ngx_http_cache_send(ngx_http_request_t *r) { diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -17,6 +17,8 @@ static ngx_int_t ngx_http_upstream_cache ngx_http_upstream_t *u); static ngx_int_t ngx_http_upstream_cache_status(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data); +static ngx_int_t ngx_http_upstream_cache_last_modified(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data); #endif static void ngx_http_upstream_init_request(ngx_http_request_t *r); @@ -358,6 +360,10 @@ static ngx_http_variable_t ngx_http_ups ngx_http_upstream_cache_status, 0, NGX_HTTP_VAR_NOCACHEABLE, 0 }, + { ngx_string("upstream_cache_last_modified"), NULL, + ngx_http_upstream_cache_last_modified, 0, + NGX_HTTP_VAR_NOCACHEABLE|NGX_HTTP_VAR_NOHASH, 0 }, + #endif { ngx_null_string, NULL, NULL, 0, 0, 0 } @@ -1806,6 +1812,56 @@ ngx_http_upstream_test_next(ngx_http_req #endif } +#if (NGX_HTTP_CACHE) + + if (status == NGX_HTTP_NOT_MODIFIED + && u->cache_status == NGX_HTTP_CACHE_EXPIRED + && u->conf->cache_revalidate) + { + time_t now, valid; + ngx_int_t rc; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http upstream not modified"); + + now = ngx_time(); + valid = r->cache->valid_sec; + + rc = u->reinit_request(r); + + if (rc != NGX_OK) { + ngx_http_upstream_finalize_request(r, u, rc); + return NGX_OK; + } + + u->cache_status = NGX_HTTP_CACHE_REVALIDATED; + rc = ngx_http_upstream_cache_send(r, u); + + if (valid == 0) { + valid = r->cache->valid_sec; + } + + if (valid == 0) { + valid = ngx_http_file_cache_valid(u->conf->cache_valid, + u->headers_in.status_n); + if (valid) { + valid = now + valid; + } + } + + if (valid) { + r->cache->valid_sec = valid; + r->cache->date = now; + + ngx_http_file_cache_update_header(r); + } + + ngx_http_upstream_finalize_request(r, u, rc); + return NGX_OK; + } + +#endif + return NGX_DECLINED; } @@ -4492,6 +4548,35 @@ ngx_http_upstream_cache_status(ngx_http_ return NGX_OK; } + +static ngx_int_t +ngx_http_upstream_cache_last_modified(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data) +{ + u_char *p; + + if (!r->upstream->conf->cache_revalidate + || r->upstream->cache_status != NGX_HTTP_CACHE_EXPIRED + || r->cache->last_modified == -1) + { + v->not_found = 1; + return NGX_OK; + } + + p = ngx_pnalloc(r->pool, sizeof("Mon, 28 Sep 1970 06:00:00 GMT") - 1); + if (p == NULL) { + return NGX_ERROR; + } + + v->len = ngx_http_time(p, r->cache->last_modified) - p; + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + v->data = p; + + return NGX_OK; +} + #endif diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h --- a/src/http/ngx_http_upstream.h +++ b/src/http/ngx_http_upstream.h @@ -178,6 +178,8 @@ typedef struct { ngx_flag_t cache_lock; ngx_msec_t cache_lock_timeout; + ngx_flag_t cache_revalidate; + ngx_array_t *cache_valid; ngx_array_t *cache_bypass; ngx_array_t *no_cache;