Mercurial > hg > nginx

changeset 5441:43ccaf8e8728

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Upstream: cache revalidation with conditional requests. The following new directives are introduced: proxy_cache_revalidate, fastcgi_cache_revalidate, scgi_cache_revalidate, uwsgi_cache_revalidate. Default is off. When set to on, they enable cache revalidation using conditional requests with If-Modified-Since for expired cache items. As of now, no attempts are made to merge headers given in a 304 response during cache revalidation with headers previously stored in a cache item. Headers in a 304 response are only used to calculate new validity time of a cache item.
author Maxim Dounin <mdounin@mdounin.ru>
date Mon, 18 Nov 2013 20:48:22 +0400
parents cbb9a6c7493c
children 63f960bbc52f
files src/http/modules/ngx_http_fastcgi_module.c src/http/modules/ngx_http_proxy_module.c src/http/modules/ngx_http_scgi_module.c src/http/modules/ngx_http_uwsgi_module.c src/http/ngx_http_cache.h src/http/ngx_http_file_cache.c src/http/ngx_http_upstream.c src/http/ngx_http_upstream.h
diffstat 8 files changed, 254 insertions(+), 6 deletions(-) [+]
line wrap: on
line diff
--- a/src/http/modules/ngx_http_fastcgi_module.c
+++ b/src/http/modules/ngx_http_fastcgi_module.c
@@ -405,6 +405,13 @@ static ngx_command_t  ngx_http_fastcgi_c
       offsetof(ngx_http_fastcgi_loc_conf_t, upstream.cache_lock_timeout),
       NULL },
 
+    { ngx_string("fastcgi_cache_revalidate"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+      ngx_conf_set_flag_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_fastcgi_loc_conf_t, upstream.cache_revalidate),
+      NULL },
+
 #endif
 
     { ngx_string("fastcgi_temp_path"),
@@ -563,7 +570,8 @@ static ngx_str_t  ngx_http_fastcgi_hide_
 #if (NGX_HTTP_CACHE)
 
 static ngx_keyval_t  ngx_http_fastcgi_cache_headers[] = {
-    { ngx_string("HTTP_IF_MODIFIED_SINCE"), ngx_string("") },
+    { ngx_string("HTTP_IF_MODIFIED_SINCE"),
+      ngx_string("$upstream_cache_last_modified") },
     { ngx_string("HTTP_IF_UNMODIFIED_SINCE"), ngx_string("") },
     { ngx_string("HTTP_IF_NONE_MATCH"), ngx_string("") },
     { ngx_string("HTTP_IF_MATCH"), ngx_string("") },
@@ -2336,6 +2344,7 @@ ngx_http_fastcgi_create_loc_conf(ngx_con
     conf->upstream.cache_valid = NGX_CONF_UNSET_PTR;
     conf->upstream.cache_lock = NGX_CONF_UNSET;
     conf->upstream.cache_lock_timeout = NGX_CONF_UNSET_MSEC;
+    conf->upstream.cache_revalidate = NGX_CONF_UNSET;
 #endif
 
     conf->upstream.hide_headers = NGX_CONF_UNSET_PTR;
@@ -2582,6 +2591,9 @@ ngx_http_fastcgi_merge_loc_conf(ngx_conf
     ngx_conf_merge_msec_value(conf->upstream.cache_lock_timeout,
                               prev->upstream.cache_lock_timeout, 5000);
 
+    ngx_conf_merge_value(conf->upstream.cache_revalidate,
+                              prev->upstream.cache_revalidate, 0);
+
 #endif
 
     ngx_conf_merge_value(conf->upstream.pass_request_headers,
--- a/src/http/modules/ngx_http_proxy_module.c
+++ b/src/http/modules/ngx_http_proxy_module.c
@@ -465,6 +465,13 @@ static ngx_command_t  ngx_http_proxy_com
       offsetof(ngx_http_proxy_loc_conf_t, upstream.cache_lock_timeout),
       NULL },
 
+    { ngx_string("proxy_cache_revalidate"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+      ngx_conf_set_flag_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_proxy_loc_conf_t, upstream.cache_revalidate),
+      NULL },
+
 #endif
 
     { ngx_string("proxy_temp_path"),
@@ -622,7 +629,8 @@ static ngx_keyval_t  ngx_http_proxy_cach
     { ngx_string("Keep-Alive"), ngx_string("") },
     { ngx_string("Expect"), ngx_string("") },
     { ngx_string("Upgrade"), ngx_string("") },
-    { ngx_string("If-Modified-Since"), ngx_string("") },
+    { ngx_string("If-Modified-Since"),
+      ngx_string("$upstream_cache_last_modified") },
     { ngx_string("If-Unmodified-Since"), ngx_string("") },
     { ngx_string("If-None-Match"), ngx_string("") },
     { ngx_string("If-Match"), ngx_string("") },
@@ -2454,6 +2462,7 @@ ngx_http_proxy_create_loc_conf(ngx_conf_
     conf->upstream.cache_valid = NGX_CONF_UNSET_PTR;
     conf->upstream.cache_lock = NGX_CONF_UNSET;
     conf->upstream.cache_lock_timeout = NGX_CONF_UNSET_MSEC;
+    conf->upstream.cache_revalidate = NGX_CONF_UNSET;
 #endif
 
     conf->upstream.hide_headers = NGX_CONF_UNSET_PTR;
@@ -2710,6 +2719,9 @@ ngx_http_proxy_merge_loc_conf(ngx_conf_t
     ngx_conf_merge_msec_value(conf->upstream.cache_lock_timeout,
                               prev->upstream.cache_lock_timeout, 5000);
 
+    ngx_conf_merge_value(conf->upstream.cache_revalidate,
+                              prev->upstream.cache_revalidate, 0);
+
 #endif
 
     ngx_conf_merge_str_value(conf->method, prev->method, "");
--- a/src/http/modules/ngx_http_scgi_module.c
+++ b/src/http/modules/ngx_http_scgi_module.c
@@ -262,6 +262,13 @@ static ngx_command_t ngx_http_scgi_comma
       offsetof(ngx_http_scgi_loc_conf_t, upstream.cache_lock_timeout),
       NULL },
 
+    { ngx_string("scgi_cache_revalidate"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+      ngx_conf_set_flag_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_scgi_loc_conf_t, upstream.cache_revalidate),
+      NULL },
+
 #endif
 
     { ngx_string("scgi_temp_path"),
@@ -369,7 +376,8 @@ static ngx_str_t ngx_http_scgi_hide_head
 #if (NGX_HTTP_CACHE)
 
 static ngx_keyval_t  ngx_http_scgi_cache_headers[] = {
-    { ngx_string("HTTP_IF_MODIFIED_SINCE"), ngx_string("") },
+    { ngx_string("HTTP_IF_MODIFIED_SINCE"),
+      ngx_string("$upstream_cache_last_modified") },
     { ngx_string("HTTP_IF_UNMODIFIED_SINCE"), ngx_string("") },
     { ngx_string("HTTP_IF_NONE_MATCH"), ngx_string("") },
     { ngx_string("HTTP_IF_MATCH"), ngx_string("") },
@@ -1093,6 +1101,7 @@ ngx_http_scgi_create_loc_conf(ngx_conf_t
     conf->upstream.cache_valid = NGX_CONF_UNSET_PTR;
     conf->upstream.cache_lock = NGX_CONF_UNSET;
     conf->upstream.cache_lock_timeout = NGX_CONF_UNSET_MSEC;
+    conf->upstream.cache_revalidate = NGX_CONF_UNSET;
 #endif
 
     conf->upstream.hide_headers = NGX_CONF_UNSET_PTR;
@@ -1333,6 +1342,9 @@ ngx_http_scgi_merge_loc_conf(ngx_conf_t 
     ngx_conf_merge_msec_value(conf->upstream.cache_lock_timeout,
                               prev->upstream.cache_lock_timeout, 5000);
 
+    ngx_conf_merge_value(conf->upstream.cache_revalidate,
+                              prev->upstream.cache_revalidate, 0);
+
 #endif
 
     ngx_conf_merge_value(conf->upstream.pass_request_headers,
--- a/src/http/modules/ngx_http_uwsgi_module.c
+++ b/src/http/modules/ngx_http_uwsgi_module.c
@@ -289,6 +289,13 @@ static ngx_command_t ngx_http_uwsgi_comm
       offsetof(ngx_http_uwsgi_loc_conf_t, upstream.cache_lock_timeout),
       NULL },
 
+    { ngx_string("uwsgi_cache_revalidate"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+      ngx_conf_set_flag_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_uwsgi_loc_conf_t, upstream.cache_revalidate),
+      NULL },
+
 #endif
 
     { ngx_string("uwsgi_temp_path"),
@@ -402,7 +409,8 @@ static ngx_str_t ngx_http_uwsgi_hide_hea
 #if (NGX_HTTP_CACHE)
 
 static ngx_keyval_t  ngx_http_uwsgi_cache_headers[] = {
-    { ngx_string("HTTP_IF_MODIFIED_SINCE"), ngx_string("") },
+    { ngx_string("HTTP_IF_MODIFIED_SINCE"),
+      ngx_string("$upstream_cache_last_modified") },
     { ngx_string("HTTP_IF_UNMODIFIED_SINCE"), ngx_string("") },
     { ngx_string("HTTP_IF_NONE_MATCH"), ngx_string("") },
     { ngx_string("HTTP_IF_MATCH"), ngx_string("") },
@@ -1130,6 +1138,7 @@ ngx_http_uwsgi_create_loc_conf(ngx_conf_
     conf->upstream.cache_valid = NGX_CONF_UNSET_PTR;
     conf->upstream.cache_lock = NGX_CONF_UNSET;
     conf->upstream.cache_lock_timeout = NGX_CONF_UNSET_MSEC;
+    conf->upstream.cache_revalidate = NGX_CONF_UNSET;
 #endif
 
     conf->upstream.hide_headers = NGX_CONF_UNSET_PTR;
@@ -1370,6 +1379,9 @@ ngx_http_uwsgi_merge_loc_conf(ngx_conf_t
     ngx_conf_merge_msec_value(conf->upstream.cache_lock_timeout,
                               prev->upstream.cache_lock_timeout, 5000);
 
+    ngx_conf_merge_value(conf->upstream.cache_revalidate,
+                              prev->upstream.cache_revalidate, 0);
+
 #endif
 
     ngx_conf_merge_value(conf->upstream.pass_request_headers,
--- a/src/http/ngx_http_cache.h
+++ b/src/http/ngx_http_cache.h
@@ -19,8 +19,9 @@
 #define NGX_HTTP_CACHE_EXPIRED       3
 #define NGX_HTTP_CACHE_STALE         4
 #define NGX_HTTP_CACHE_UPDATING      5
-#define NGX_HTTP_CACHE_HIT           6
-#define NGX_HTTP_CACHE_SCARCE        7
+#define NGX_HTTP_CACHE_REVALIDATED   6
+#define NGX_HTTP_CACHE_HIT           7
+#define NGX_HTTP_CACHE_SCARCE        8
 
 #define NGX_HTTP_CACHE_KEY_LEN       16
 
@@ -143,6 +144,7 @@ void ngx_http_file_cache_create_key(ngx_
 ngx_int_t ngx_http_file_cache_open(ngx_http_request_t *r);
 void ngx_http_file_cache_set_header(ngx_http_request_t *r, u_char *buf);
 void ngx_http_file_cache_update(ngx_http_request_t *r, ngx_temp_file_t *tf);
+void ngx_http_file_cache_update_header(ngx_http_request_t *r);
 ngx_int_t ngx_http_cache_send(ngx_http_request_t *);
 void ngx_http_file_cache_free(ngx_http_cache_t *c, ngx_temp_file_t *tf);
 time_t ngx_http_file_cache_valid(ngx_array_t *cache_valid, ngx_uint_t status);
--- a/src/http/ngx_http_file_cache.c
+++ b/src/http/ngx_http_file_cache.c
@@ -53,6 +53,7 @@ ngx_str_t  ngx_http_cache_status[] = {
     ngx_string("EXPIRED"),
     ngx_string("STALE"),
     ngx_string("UPDATING"),
+    ngx_string("REVALIDATED"),
     ngx_string("HIT")
 };
 
@@ -971,6 +972,116 @@ ngx_http_file_cache_update(ngx_http_requ
 }
 
 
+void
+ngx_http_file_cache_update_header(ngx_http_request_t *r)
+{
+    ssize_t                        n;
+    ngx_err_t                      err;
+    ngx_file_t                     file;
+    ngx_file_info_t                fi;
+    ngx_http_cache_t              *c;
+    ngx_http_file_cache_header_t   h;
+
+    ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                   "http file cache update header");
+
+    c = r->cache;
+
+    ngx_memzero(&file, sizeof(ngx_file_t));
+
+    file.name = c->file.name;
+    file.log = r->connection->log;
+    file.fd = ngx_open_file(file.name.data, NGX_FILE_RDWR, NGX_FILE_OPEN, 0);
+
+    if (file.fd == NGX_INVALID_FILE) {
+        err = ngx_errno;
+
+        /* cache file may have been deleted */
+
+        if (err == NGX_ENOENT) {
+            ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                           "http file cache \"%s\" not found",
+                           file.name.data);
+            return;
+        }
+
+        ngx_log_error(NGX_LOG_CRIT, r->connection->log, err,
+                      ngx_open_file_n " \"%s\" failed", file.name.data);
+        return;
+    }
+
+    /*
+     * make sure cache file wasn't replaced;
+     * if it was, do nothing
+     */
+
+    if (ngx_fd_info(file.fd, &fi) == NGX_FILE_ERROR) {
+        ngx_log_error(NGX_LOG_CRIT, r->connection->log, ngx_errno,
+                      ngx_fd_info_n " \"%s\" failed", file.name.data);
+        goto done;
+    }
+
+    if (c->uniq != ngx_file_uniq(&fi)
+        || c->length != ngx_file_size(&fi))
+    {
+        ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                       "http file cache \"%s\" changed",
+                       file.name.data);
+        goto done;
+    }
+
+    n = ngx_read_file(&file, (u_char *) &h,
+                      sizeof(ngx_http_file_cache_header_t), 0);
+
+    if (n == NGX_ERROR) {
+        goto done;
+    }
+
+    if ((size_t) n != sizeof(ngx_http_file_cache_header_t)) {
+        ngx_log_error(NGX_LOG_CRIT, r->connection->log, 0,
+                      ngx_read_file_n " read only %z of %z from \"%s\"",
+                      n, sizeof(ngx_http_file_cache_header_t), file.name.data);
+        goto done;
+    }
+
+    if (h.last_modified != c->last_modified
+        || h.crc32 != c->crc32
+        || h.header_start != c->header_start
+        || h.body_start != c->body_start)
+    {
+        ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                       "http file cache \"%s\" content changed",
+                       file.name.data);
+        goto done;
+    }
+
+    /*
+     * update cache file header with new data,
+     * notably h.valid_sec and h.date
+     */
+
+    ngx_memzero(&h, sizeof(ngx_http_file_cache_header_t));
+
+    h.valid_sec = c->valid_sec;
+    h.last_modified = c->last_modified;
+    h.date = c->date;
+    h.crc32 = c->crc32;
+    h.valid_msec = (u_short) c->valid_msec;
+    h.header_start = (u_short) c->header_start;
+    h.body_start = (u_short) c->body_start;
+
+    (void) ngx_write_file(&file, (u_char *) &h,
+                          sizeof(ngx_http_file_cache_header_t), 0);
+
+done:
+
+    if (ngx_close_file(file.fd) == NGX_FILE_ERROR) {
+        ngx_log_error(NGX_LOG_ALERT, r->connection->log, ngx_errno,
+                      ngx_close_file_n " \"%s\" failed", file.name.data);
+    }
+}
+
+
 ngx_int_t
 ngx_http_cache_send(ngx_http_request_t *r)
 {
--- a/src/http/ngx_http_upstream.c
+++ b/src/http/ngx_http_upstream.c
@@ -17,6 +17,8 @@ static ngx_int_t ngx_http_upstream_cache
     ngx_http_upstream_t *u);
 static ngx_int_t ngx_http_upstream_cache_status(ngx_http_request_t *r,
     ngx_http_variable_value_t *v, uintptr_t data);
+static ngx_int_t ngx_http_upstream_cache_last_modified(ngx_http_request_t *r,
+    ngx_http_variable_value_t *v, uintptr_t data);
 #endif
 
 static void ngx_http_upstream_init_request(ngx_http_request_t *r);
@@ -358,6 +360,10 @@ static ngx_http_variable_t  ngx_http_ups
       ngx_http_upstream_cache_status, 0,
       NGX_HTTP_VAR_NOCACHEABLE, 0 },
 
+    { ngx_string("upstream_cache_last_modified"), NULL,
+      ngx_http_upstream_cache_last_modified, 0,
+      NGX_HTTP_VAR_NOCACHEABLE|NGX_HTTP_VAR_NOHASH, 0 },
+
 #endif
 
     { ngx_null_string, NULL, NULL, 0, 0, 0 }
@@ -1806,6 +1812,56 @@ ngx_http_upstream_test_next(ngx_http_req
 #endif
     }
 
+#if (NGX_HTTP_CACHE)
+
+    if (status == NGX_HTTP_NOT_MODIFIED
+        && u->cache_status == NGX_HTTP_CACHE_EXPIRED
+        && u->conf->cache_revalidate)
+    {
+        time_t     now, valid;
+        ngx_int_t  rc;
+
+        ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                       "http upstream not modified");
+
+        now = ngx_time();
+        valid = r->cache->valid_sec;
+
+        rc = u->reinit_request(r);
+
+        if (rc != NGX_OK) {
+            ngx_http_upstream_finalize_request(r, u, rc);
+            return NGX_OK;
+        }
+
+        u->cache_status = NGX_HTTP_CACHE_REVALIDATED;
+        rc = ngx_http_upstream_cache_send(r, u);
+
+        if (valid == 0) {
+            valid = r->cache->valid_sec;
+        }
+
+        if (valid == 0) {
+            valid = ngx_http_file_cache_valid(u->conf->cache_valid,
+                                              u->headers_in.status_n);
+            if (valid) {
+                valid = now + valid;
+            }
+        }
+
+        if (valid) {
+            r->cache->valid_sec = valid;
+            r->cache->date = now;
+
+            ngx_http_file_cache_update_header(r);
+        }
+
+        ngx_http_upstream_finalize_request(r, u, rc);
+        return NGX_OK;
+    }
+
+#endif
+
     return NGX_DECLINED;
 }
 
@@ -4492,6 +4548,35 @@ ngx_http_upstream_cache_status(ngx_http_
     return NGX_OK;
 }
 
+
+static ngx_int_t
+ngx_http_upstream_cache_last_modified(ngx_http_request_t *r,
+    ngx_http_variable_value_t *v, uintptr_t data)
+{
+    u_char  *p;
+
+    if (!r->upstream->conf->cache_revalidate
+        || r->upstream->cache_status != NGX_HTTP_CACHE_EXPIRED
+        || r->cache->last_modified == -1)
+    {
+        v->not_found = 1;
+        return NGX_OK;
+    }
+
+    p = ngx_pnalloc(r->pool, sizeof("Mon, 28 Sep 1970 06:00:00 GMT") - 1);
+    if (p == NULL) {
+        return NGX_ERROR;
+    }
+
+    v->len = ngx_http_time(p, r->cache->last_modified) - p;
+    v->valid = 1;
+    v->no_cacheable = 0;
+    v->not_found = 0;
+    v->data = p;
+
+    return NGX_OK;
+}
+
 #endif
 
 
--- a/src/http/ngx_http_upstream.h
+++ b/src/http/ngx_http_upstream.h
@@ -178,6 +178,8 @@ typedef struct {
     ngx_flag_t                       cache_lock;
     ngx_msec_t                       cache_lock_timeout;
 
+    ngx_flag_t                       cache_revalidate;
+
     ngx_array_t                     *cache_valid;
     ngx_array_t                     *cache_bypass;
     ngx_array_t                     *no_cache;