Mercurial > hg > nginx-vendor-current

comparison src/http/ngx_http_file_cache.c @ 468:56baf312c1b5 NGINX_0_7_46

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
nginx 0.7.46 *) Bugfix: the previous release tarball was incorrect.
author Igor Sysoev <http://sysoev.ru>
date Mon, 30 Mar 2009 00:00:00 +0400
parents 9eda3153223b
children 6866b490272e
comparison
equal deleted inserted replaced
467:d46142e61c30 468:56baf312c1b5
8 #include <ngx_core.h> 8 #include <ngx_core.h>
9 #include <ngx_http.h> 9 #include <ngx_http.h>
10 #include <ngx_md5.h> 10 #include <ngx_md5.h>
11 11
12 12
13 static ngx_int_t ngx_http_file_cache_exists(ngx_http_request_t *r, 13 static ngx_int_t ngx_http_file_cache_exists(ngx_http_file_cache_t *cache,
14 ngx_http_file_cache_t *cache); 14 ngx_http_cache_t *c);
15 static ngx_http_file_cache_node_t * 15 static ngx_http_file_cache_node_t *
16 ngx_http_file_cache_lookup(ngx_http_file_cache_t *cache, u_char *key); 16 ngx_http_file_cache_lookup(ngx_http_file_cache_t *cache, u_char *key);
17 static void ngx_http_file_cache_rbtree_insert_value(ngx_rbtree_node_t *temp, 17 static void ngx_http_file_cache_rbtree_insert_value(ngx_rbtree_node_t *temp,
18 ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel); 18 ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel);
19 static void ngx_http_file_cache_cleanup(void *data); 19 static void ngx_http_file_cache_cleanup(void *data);
20 static time_t ngx_http_file_cache_expire(ngx_http_file_cache_t *cache, 20 static time_t ngx_http_file_cache_forced_expire(ngx_http_file_cache_t *cache);
21 ngx_uint_t force); 21 static time_t ngx_http_file_cache_expire(ngx_http_file_cache_t *cache);
22 static ngx_int_t ngx_http_file_cache_clean_noop(ngx_tree_ctx_t *ctx, 22 static void ngx_http_file_cache_delete(ngx_http_file_cache_t *cache,
23 ngx_queue_t *q, u_char *name);
24 static ngx_int_t
25 ngx_http_file_cache_manager_sleep(ngx_http_file_cache_t *cache);
26 static ngx_int_t ngx_http_file_cache_noop(ngx_tree_ctx_t *ctx,
23 ngx_str_t *path); 27 ngx_str_t *path);
24 static ngx_int_t ngx_http_file_cache_clean_file(ngx_tree_ctx_t *ctx, 28 static ngx_int_t ngx_http_file_cache_manage_file(ngx_tree_ctx_t *ctx,
29 ngx_str_t *path);
30 static ngx_int_t ngx_http_file_cache_add_file(ngx_tree_ctx_t *ctx,
31 ngx_str_t *path);
32 static ngx_int_t ngx_http_file_cache_add(ngx_http_file_cache_t *cache,
33 ngx_http_cache_t *c);
34 static ngx_int_t ngx_http_file_cache_delete_file(ngx_tree_ctx_t *ctx,
25 ngx_str_t *path); 35 ngx_str_t *path);
26 36
27 37
28 static u_char ngx_http_file_cache_key[] = { LF, 'K', 'E', 'Y', ':', ' ' }; 38 static u_char ngx_http_file_cache_key[] = { LF, 'K', 'E', 'Y', ':', ' ' };
29 39
31 static ngx_int_t 41 static ngx_int_t
32 ngx_http_file_cache_init(ngx_shm_zone_t *shm_zone, void *data) 42 ngx_http_file_cache_init(ngx_shm_zone_t *shm_zone, void *data)
33 { 43 {
34 ngx_http_file_cache_t *ocache = data; 44 ngx_http_file_cache_t *ocache = data;
35 45
46 size_t len;
36 ngx_rbtree_node_t *sentinel; 47 ngx_rbtree_node_t *sentinel;
37 ngx_http_file_cache_t *cache; 48 ngx_http_file_cache_t *cache;
38 49
39 cache = shm_zone->data; 50 cache = shm_zone->data;
40 51
50 } 61 }
51 62
52 cache->rbtree = ocache->rbtree; 63 cache->rbtree = ocache->rbtree;
53 cache->queue = ocache->queue; 64 cache->queue = ocache->queue;
54 cache->shpool = ocache->shpool; 65 cache->shpool = ocache->shpool;
55 cache->created = ocache->created; 66 cache->cold = ocache->cold;
67 cache->size = ocache->size;
68 cache->bsize = ocache->bsize;
69
70 cache->max_size /= cache->bsize;
56 71
57 return NGX_OK; 72 return NGX_OK;
58 } 73 }
59 74
60 cache->shpool = (ngx_slab_pool_t *) shm_zone->shm.addr; 75 cache->shpool = (ngx_slab_pool_t *) shm_zone->shm.addr;
77 return NGX_ERROR; 92 return NGX_ERROR;
78 } 93 }
79 94
80 ngx_queue_init(cache->queue); 95 ngx_queue_init(cache->queue);
81 96
82 cache->created = ngx_time(); 97 cache->cold = ngx_slab_alloc(cache->shpool, sizeof(ngx_atomic_t));
98 if (cache->cold == NULL) {
99 return NGX_ERROR;
100 }
101
102 *cache->cold = 1;
103
104 cache->size = ngx_slab_alloc(cache->shpool, sizeof(off_t));
105 if (cache->size == NULL) {
106 return NGX_ERROR;
107 }
108
109 *cache->size = 0;
110
111 cache->bsize = ngx_fs_bsize(cache->path->name.data);
112
113 cache->max_size /= cache->bsize;
114
115 len = sizeof(" in cache keys zone \"\"") + shm_zone->name.len;
116
117 cache->shpool->log_ctx = ngx_slab_alloc(cache->shpool, len);
118 if (cache->shpool->log_ctx == NULL) {
119 return NGX_ERROR;
120 }
121
122 ngx_sprintf(cache->shpool->log_ctx, " in cache keys zone \"%V\"%Z",
123 &shm_zone->name);
83 124
84 return NGX_OK; 125 return NGX_OK;
85 } 126 }
86 127
87 128
142 cln = ngx_pool_cleanup_add(r->pool, 0); 183 cln = ngx_pool_cleanup_add(r->pool, 0);
143 if (cln == NULL) { 184 if (cln == NULL) {
144 return NGX_ERROR; 185 return NGX_ERROR;
145 } 186 }
146 187
147 rc = ngx_http_file_cache_exists(r, cache); 188 rc = ngx_http_file_cache_exists(cache, c);
148 189
149 ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, 190 ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
150 "http file cache exists: %i u:%ui e:%d", 191 "http file cache exists: %i u:%ui e:%d",
151 rc, c->uses, c->exists); 192 rc, c->uses, c->exists);
152 193
159 200
160 if (rc == NGX_AGAIN) { 201 if (rc == NGX_AGAIN) {
161 return rc; 202 return rc;
162 } 203 }
163 204
164 now = ngx_time(); 205 cold = *cache->cold;
165
166 cold = (now - cache->created < cache->inactive) ? 1 : 0;
167 206
168 if (rc == NGX_OK) { 207 if (rc == NGX_OK) {
169 208
170 if (c->error) { 209 if (c->error) {
171 return c->error; 210 return c->error;
291 330
292 if (cold) { 331 if (cold) {
293 332
294 ngx_shmtx_lock(&cache->shpool->mutex); 333 ngx_shmtx_lock(&cache->shpool->mutex);
295 334
296 c->node->uses = c->min_uses; 335 if (!c->node->exists) {
297 c->node->body_start = c->body_start; 336 c->node->uses = 1;
298 c->node->exists = 1; 337 c->node->body_start = c->body_start;
338 c->node->exists = 1;
339
340 *cache->size += (c->length + cache->bsize - 1) / cache->bsize;
341 }
299 342
300 ngx_shmtx_unlock(&cache->shpool->mutex); 343 ngx_shmtx_unlock(&cache->shpool->mutex);
301 } 344 }
345
346 now = ngx_time();
302 347
303 if (c->valid_sec < now) { 348 if (c->valid_sec < now) {
304 349
305 c->uses = c->min_uses; 350 c->uses = c->min_uses;
306 351
315 return NGX_OK; 360 return NGX_OK;
316 } 361 }
317 362
318 363
319 static ngx_int_t 364 static ngx_int_t
320 ngx_http_file_cache_exists(ngx_http_request_t *r, ngx_http_file_cache_t *cache) 365 ngx_http_file_cache_exists(ngx_http_file_cache_t *cache, ngx_http_cache_t *c)
321 { 366 {
322 ngx_int_t rc; 367 ngx_int_t rc;
323 ngx_http_file_cache_node_t *fcn; 368 ngx_http_file_cache_node_t *fcn;
324 369
325 ngx_shmtx_lock(&cache->shpool->mutex); 370 ngx_shmtx_lock(&cache->shpool->mutex);
326 371
327 fcn = ngx_http_file_cache_lookup(cache, r->cache->key); 372 fcn = ngx_http_file_cache_lookup(cache, c->key);
328 373
329 if (fcn) { 374 if (fcn) {
330 ngx_queue_remove(&fcn->queue); 375 ngx_queue_remove(&fcn->queue);
331 376
332 if (fcn->error) { 377 if (fcn->error) {
343 fcn->uses++; 388 fcn->uses++;
344 fcn->count++; 389 fcn->count++;
345 390
346 if (fcn->exists) { 391 if (fcn->exists) {
347 392
348 r->cache->exists = fcn->exists; 393 c->exists = fcn->exists;
349 r->cache->body_start = fcn->body_start; 394 c->body_start = fcn->body_start;
350 395
351 rc = NGX_OK; 396 rc = NGX_OK;
352 397
353 goto done; 398 goto done;
354 } 399 }
355 400
356 if (fcn->uses >= r->cache->min_uses) { 401 if (fcn->uses >= c->min_uses) {
357 402
358 r->cache->exists = fcn->exists; 403 c->exists = fcn->exists;
359 r->cache->body_start = fcn->body_start; 404 c->body_start = fcn->body_start;
360 405
361 rc = NGX_OK; 406 rc = NGX_OK;
362 407
363 } else { 408 } else {
364 rc = NGX_AGAIN; 409 rc = NGX_AGAIN;
370 fcn = ngx_slab_alloc_locked(cache->shpool, 415 fcn = ngx_slab_alloc_locked(cache->shpool,
371 sizeof(ngx_http_file_cache_node_t)); 416 sizeof(ngx_http_file_cache_node_t));
372 if (fcn == NULL) { 417 if (fcn == NULL) {
373 ngx_shmtx_unlock(&cache->shpool->mutex); 418 ngx_shmtx_unlock(&cache->shpool->mutex);
374 419
375 (void) ngx_http_file_cache_expire(cache, 1); 420 ngx_http_file_cache_forced_expire(cache);
376 421
377 ngx_shmtx_lock(&cache->shpool->mutex); 422 ngx_shmtx_lock(&cache->shpool->mutex);
378 423
379 fcn = ngx_slab_alloc_locked(cache->shpool, 424 fcn = ngx_slab_alloc_locked(cache->shpool,
380 sizeof(ngx_http_file_cache_node_t)); 425 sizeof(ngx_http_file_cache_node_t));
382 rc = NGX_ERROR; 427 rc = NGX_ERROR;
383 goto failed; 428 goto failed;
384 } 429 }
385 } 430 }
386 431
387 ngx_memcpy((u_char *) &fcn->node.key, r->cache->key, 432 ngx_memcpy((u_char *) &fcn->node.key, c->key, sizeof(ngx_rbtree_key_t));
388 sizeof(ngx_rbtree_key_t)); 433
389 434 ngx_memcpy(fcn->key, &c->key[sizeof(ngx_rbtree_key_t)],
390 ngx_memcpy(fcn->key, &r->cache->key[sizeof(ngx_rbtree_key_t)],
391 NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t)); 435 NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t));
392 436
393 ngx_rbtree_insert(cache->rbtree, &fcn->node); 437 ngx_rbtree_insert(cache->rbtree, &fcn->node);
394 438
395 renew: 439 renew:
409 453
410 fcn->expire = ngx_time() + cache->inactive; 454 fcn->expire = ngx_time() + cache->inactive;
411 455
412 ngx_queue_insert_head(cache->queue, &fcn->queue); 456 ngx_queue_insert_head(cache->queue, &fcn->queue);
413 457
414 r->cache->uniq = fcn->uniq; 458 c->uniq = fcn->uniq;
415 r->cache->uses = fcn->uses; 459 c->uses = fcn->uses;
416 r->cache->error = fcn->error; 460 c->error = fcn->error;
417 r->cache->node = fcn; 461 c->node = fcn;
418 462
419 failed: 463 failed:
420 464
421 ngx_shmtx_unlock(&cache->shpool->mutex); 465 ngx_shmtx_unlock(&cache->shpool->mutex);
422 466
554 598
555 599
556 void 600 void
557 ngx_http_file_cache_update(ngx_http_request_t *r, ngx_temp_file_t *tf) 601 ngx_http_file_cache_update(ngx_http_request_t *r, ngx_temp_file_t *tf)
558 { 602 {
603 off_t size;
559 ngx_int_t rc; 604 ngx_int_t rc;
560 ngx_file_uniq_t uniq; 605 ngx_file_uniq_t uniq;
561 ngx_file_info_t fi; 606 ngx_file_info_t fi;
562 ngx_http_cache_t *c; 607 ngx_http_cache_t *c;
563 ngx_ext_rename_file_t ext; 608 ngx_ext_rename_file_t ext;
603 } else { 648 } else {
604 uniq = ngx_file_uniq(&fi); 649 uniq = ngx_file_uniq(&fi);
605 } 650 }
606 } 651 }
607 652
653 size = (c->length + cache->bsize - 1) / cache->bsize;
654
608 ngx_shmtx_lock(&cache->shpool->mutex); 655 ngx_shmtx_lock(&cache->shpool->mutex);
609 656
610 c->node->count--; 657 c->node->count--;
611 c->node->uniq = uniq; 658 c->node->uniq = uniq;
612 c->node->body_start = c->body_start; 659 c->node->body_start = c->body_start;
660
661 size = size - (c->node->length + cache->bsize - 1) / cache->bsize;
662
663 c->node->length = c->length;
664
665 *cache->size += size;
613 666
614 if (rc == NGX_OK) { 667 if (rc == NGX_OK) {
615 c->node->exists = 1; 668 c->node->exists = 1;
616 } 669 }
617 670
744 ngx_shmtx_unlock(&cache->shpool->mutex); 797 ngx_shmtx_unlock(&cache->shpool->mutex);
745 } 798 }
746 799
747 800
748 static time_t 801 static time_t
749 ngx_http_file_cache_expire(ngx_http_file_cache_t *cache, ngx_uint_t forced) 802 ngx_http_file_cache_forced_expire(ngx_http_file_cache_t *cache)
750 { 803 {
751 u_char *name, *p; 804 u_char *name;
752 size_t len; 805 size_t len;
753 time_t now, wait; 806 time_t wait;
754 ngx_uint_t tries; 807 ngx_uint_t tries;
755 ngx_path_t *path; 808 ngx_path_t *path;
756 ngx_queue_t *q; 809 ngx_queue_t *q;
757 ngx_http_file_cache_node_t *fcn; 810 ngx_http_file_cache_node_t *fcn;
811
812 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
813 "http file cache forced expire");
814
815 path = cache->path;
816 len = path->name.len + 1 + path->len + 2 * NGX_HTTP_CACHE_KEY_LEN;
817
818 name = ngx_alloc(len + 1, ngx_cycle->log);
819 if (name == NULL) {
820 return 60;
821 }
822
823 ngx_memcpy(name, path->name.data, path->name.len);
824
825 wait = 60;
826 tries = 0;
827
828 ngx_shmtx_lock(&cache->shpool->mutex);
829
830 for (q = ngx_queue_last(cache->queue);
831 q != ngx_queue_sentinel(cache->queue);
832 q = ngx_queue_prev(q))
833 {
834 fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue);
835
836 ngx_log_debug6(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
837 "http file cache forced expire: #%d %d %02xd%02xd%02xd%02xd",
838 fcn->count, fcn->exists,
839 fcn->key[0], fcn->key[1], fcn->key[2], fcn->key[3]);
840
841 if (fcn->count) {
842
843 if (tries++ < 20) {
844 continue;
845 }
846
847 wait = 1;
848
849 break;
850 }
851
852 if (!fcn->exists) {
853
854 ngx_queue_remove(q);
855 ngx_rbtree_delete(cache->rbtree, &fcn->node);
856 ngx_slab_free_locked(cache->shpool, fcn);
857
858 break;
859 }
860
861 ngx_http_file_cache_delete(cache, q, name);
862
863 break;
864 }
865
866 ngx_shmtx_unlock(&cache->shpool->mutex);
867
868 ngx_free(name);
869
870 return wait;
871 }
872
873
874 static time_t
875 ngx_http_file_cache_expire(ngx_http_file_cache_t *cache)
876 {
877 u_char *name, *p;
878 size_t len;
879 time_t now, wait;
880 ngx_path_t *path;
881 ngx_queue_t *q;
882 ngx_http_file_cache_node_t *fcn;
758 u_char key[2 * NGX_HTTP_CACHE_KEY_LEN]; 883 u_char key[2 * NGX_HTTP_CACHE_KEY_LEN];
759 884
760 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, 885 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
761 "http file cache expire"); 886 "http file cache expire");
762 887
763 path = cache->path; 888 path = cache->path;
764 len = path->name.len + 1 + path->len + 2 * NGX_HTTP_CACHE_KEY_LEN; 889 len = path->name.len + 1 + path->len + 2 * NGX_HTTP_CACHE_KEY_LEN;
765 890
891 name = ngx_alloc(len + 1, ngx_cycle->log);
892 if (name == NULL) {
893 return 60;
894 }
895
896 ngx_memcpy(name, path->name.data, path->name.len);
897
766 now = ngx_time(); 898 now = ngx_time();
767 899
768 ngx_shmtx_lock(&cache->shpool->mutex); 900 ngx_shmtx_lock(&cache->shpool->mutex);
769 901
770 tries = 0;
771
772 for ( ;; ) { 902 for ( ;; ) {
773 903
774 if (ngx_queue_empty(cache->queue)) { 904 if (ngx_queue_empty(cache->queue)) {
775 wait = cache->inactive; 905 wait = 60;
776 break; 906 break;
777 } 907 }
778 908
779 q = ngx_queue_last(cache->queue); 909 q = ngx_queue_last(cache->queue);
780 910
781 fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue); 911 fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue);
782 912
783 if (!forced) { 913 wait = fcn->expire - now;
784 wait = fcn->expire - now; 914
785 915 if (wait > 0) {
786 if (wait > 0) { 916 wait = wait > 60 ? 60 : wait;
787 break; 917 break;
788 }
789 } 918 }
790 919
791 ngx_log_debug6(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, 920 ngx_log_debug6(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
792 "http file cache expire: #%d %d %02xd%02xd%02xd%02xd", 921 "http file cache expire: #%d %d %02xd%02xd%02xd%02xd",
793 fcn->count, fcn->exists, 922 fcn->count, fcn->exists,
794 fcn->key[0], fcn->key[1], fcn->key[2], fcn->key[3]); 923 fcn->key[0], fcn->key[1], fcn->key[2], fcn->key[3]);
795 924
796 if (fcn->count) { 925 if (fcn->count) {
797 926
798 if (!forced) { 927 p = ngx_hex_dump(key, (u_char *) &fcn->node.key,
799 928 sizeof(ngx_rbtree_key_t));
800 p = ngx_hex_dump(key, (u_char *) &fcn->node.key, 929
801 sizeof(ngx_rbtree_key_t)); 930 len = NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t);
802 (void) ngx_hex_dump(p, fcn->key, 931 (void) ngx_hex_dump(p, fcn->key, len);
803 NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t)); 932
804 933 /*
805 /* 934 * abnormally exited workers may leave locked cache entries,
806 * abnormally exited workers may leave locked cache entries, 935 * and although it may be safe to remove them completely,
807 * and although it may be safe to remove them completely, 936 * we prefer to remove them from inactive queue and rbtree
808 * we prefer to remove them from inactive queue and rbtree 937 * only, and to allow other leaks
809 * only, and to allow other leaks 938 */
810 */ 939
811 940 ngx_queue_remove(q);
812 ngx_queue_remove(q); 941 ngx_rbtree_delete(cache->rbtree, &fcn->node);
813 942
814 ngx_rbtree_delete(cache->rbtree, &fcn->node); 943 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
815
816 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
817 "ignore long locked inactive cache entry %*s, count:%d", 944 "ignore long locked inactive cache entry %*s, count:%d",
818 2 * NGX_HTTP_CACHE_KEY_LEN, key, fcn->count); 945 2 * NGX_HTTP_CACHE_KEY_LEN, key, fcn->count);
819 } 946
820 947 continue;
821 if (tries++ < 10) { 948 }
822 continue;
823 }
824
825 wait = 1;
826 break;
827 }
828
829 forced = 0;
830 949
831 if (!fcn->exists) { 950 if (!fcn->exists) {
832 951
833 ngx_queue_remove(q); 952 ngx_queue_remove(q);
834
835 ngx_rbtree_delete(cache->rbtree, &fcn->node); 953 ngx_rbtree_delete(cache->rbtree, &fcn->node);
836
837 ngx_slab_free_locked(cache->shpool, fcn); 954 ngx_slab_free_locked(cache->shpool, fcn);
838 955
839 continue; 956 continue;
840 } 957 }
841 958
842 name = ngx_alloc(len + 1, ngx_cycle->log); 959 ngx_http_file_cache_delete(cache, q, name);
843 if (name == NULL) {
844 wait = 60;
845 break;
846 }
847
848 ngx_memcpy(name, path->name.data, path->name.len);
849
850 p = name + path->name.len + 1 + path->len;
851 p = ngx_hex_dump(p, (u_char *) &fcn->node.key,
852 sizeof(ngx_rbtree_key_t));
853 p = ngx_hex_dump(p, fcn->key,
854 NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t));
855 *p = '\0';
856
857 ngx_queue_remove(q);
858
859 ngx_rbtree_delete(cache->rbtree, &fcn->node);
860
861 ngx_slab_free_locked(cache->shpool, fcn);
862
863 ngx_shmtx_unlock(&cache->shpool->mutex);
864
865 ngx_create_hashed_filename(path, name, len);
866
867 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
868 "http file cache expire: \"%s\"", name);
869
870 if (ngx_delete_file(name) == NGX_FILE_ERROR) {
871 ngx_log_error(NGX_LOG_CRIT, ngx_cycle->log, ngx_errno,
872 ngx_delete_file_n " \"%s\" failed", name);
873 }
874
875 ngx_free(name);
876
877 ngx_shmtx_lock(&cache->shpool->mutex);
878 } 960 }
879 961
880 ngx_shmtx_unlock(&cache->shpool->mutex); 962 ngx_shmtx_unlock(&cache->shpool->mutex);
881 963
964 ngx_free(name);
965
882 return wait; 966 return wait;
883 } 967 }
884 968
885 969
886 time_t 970 static void
887 ngx_http_file_cache_cleaner(void *data) 971 ngx_http_file_cache_delete(ngx_http_file_cache_t *cache, ngx_queue_t *q,
972 u_char *name)
973 {
974 u_char *p;
975 size_t len;
976 ngx_path_t *path;
977 ngx_http_file_cache_node_t *fcn;
978
979 fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue);
980
981 *cache->size -= (fcn->length + cache->bsize - 1) / cache->bsize;
982
983 path = cache->path;
984
985 p = name + path->name.len + 1 + path->len;
986
987 p = ngx_hex_dump(p, (u_char *) &fcn->node.key, sizeof(ngx_rbtree_key_t));
988
989 len = NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t);
990 p = ngx_hex_dump(p, fcn->key, len);
991 *p = '\0';
992
993 ngx_queue_remove(q);
994
995 ngx_rbtree_delete(cache->rbtree, &fcn->node);
996
997 ngx_slab_free_locked(cache->shpool, fcn);
998
999 ngx_shmtx_unlock(&cache->shpool->mutex);
1000
1001 len = path->name.len + 1 + path->len + 2 * NGX_HTTP_CACHE_KEY_LEN;
1002
1003 ngx_create_hashed_filename(path, name, len);
1004
1005 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
1006 "http file cache expire: \"%s\"", name);
1007
1008 if (ngx_delete_file(name) == NGX_FILE_ERROR) {
1009 ngx_log_error(NGX_LOG_CRIT, ngx_cycle->log, ngx_errno,
1010 ngx_delete_file_n " \"%s\" failed", name);
1011 }
1012
1013 ngx_shmtx_lock(&cache->shpool->mutex);
1014 }
1015
1016
1017 static time_t
1018 ngx_http_file_cache_manager(void *data)
888 { 1019 {
889 ngx_http_file_cache_t *cache = data; 1020 ngx_http_file_cache_t *cache = data;
890 1021
891 time_t now, next; 1022 off_t size;
1023 time_t next;
892 ngx_tree_ctx_t tree; 1024 ngx_tree_ctx_t tree;
893 1025
894 now = ngx_time(); 1026 if (*cache->cold) {
895 1027
896 if (now >= cache->next_clean_time 1028 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
897 && now >= cache->created + cache->inactive) 1029 "http file cache manager update");
898 {
899 ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0,
900 "clean unused cache files");
901 1030
902 tree.init_handler = NULL; 1031 tree.init_handler = NULL;
903 tree.file_handler = ngx_http_file_cache_clean_file; 1032 tree.file_handler = ngx_http_file_cache_manage_file;
904 tree.pre_tree_handler = ngx_http_file_cache_clean_noop; 1033 tree.pre_tree_handler = ngx_http_file_cache_noop;
905 tree.post_tree_handler = ngx_http_file_cache_clean_noop; 1034 tree.post_tree_handler = ngx_http_file_cache_noop;
906 tree.spec_handler = ngx_http_file_cache_clean_file; 1035 tree.spec_handler = ngx_http_file_cache_delete_file;
907 tree.data = cache; 1036 tree.data = cache;
908 tree.alloc = 0; 1037 tree.alloc = 0;
909 tree.log = ngx_cycle->log; 1038 tree.log = ngx_cycle->log;
910 1039
911 (void) ngx_walk_tree(&tree, &cache->path->name); 1040 cache->last = ngx_current_msec;
1041 cache->files = 0;
1042
1043 if (ngx_walk_tree(&tree, &cache->path->name) == NGX_ABORT) {
1044 return 60;
1045 }
1046
1047 *cache->cold = 0;
1048
1049 ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0,
1050 "http file cache: %V %.3fM, bsize: %uz",
1051 &cache->path->name,
1052 ((double) *cache->size * cache->bsize) / (1024 * 1024),
1053 cache->bsize);
1054 }
1055
1056 next = ngx_http_file_cache_expire(cache);
1057
1058 cache->last = ngx_current_msec;
1059 cache->files = 0;
1060
1061 for ( ;; ) {
1062 ngx_shmtx_lock(&cache->shpool->mutex);
1063
1064 size = *cache->size;
1065
1066 ngx_shmtx_unlock(&cache->shpool->mutex);
1067
1068 if (size < cache->max_size) {
1069 return next;
1070 }
1071
1072 next = ngx_http_file_cache_forced_expire(cache);
1073
1074 if (ngx_http_file_cache_manager_sleep(cache) != NGX_OK) {
1075 return next;
1076 }
1077 }
1078 }
1079
1080
1081 static ngx_int_t
1082 ngx_http_file_cache_manager_sleep(ngx_http_file_cache_t *cache)
1083 {
1084 ngx_msec_t elapsed;
1085
1086 if (cache->files++ > 100) {
912 1087
913 ngx_time_update(0, 0); 1088 ngx_time_update(0, 0);
914 1089
915 next = ngx_next_time(cache->clean_time); 1090 elapsed = ngx_abs((ngx_msec_int_t) (ngx_current_msec - cache->last));
916 1091
917 if (next == -1) { 1092 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
918 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, ngx_errno, 1093 "http file cache manager time: %M", elapsed);
919 ngx_next_time_n " failed"); 1094
920 return 60; 1095 if (elapsed > 200) {
921 } 1096
922 1097 /*
923 cache->next_clean_time = next; 1098 * if processing 100 files takes more than 200ms,
924 } 1099 * it seems that many operations require disk i/o,
925 1100 * therefore sleep 200ms
926 next = ngx_http_file_cache_expire(cache, 0); 1101 */
927 1102
928 now = ngx_time(); 1103 ngx_msleep(200);
929 1104
930 return (now + next < cache->next_clean_time) ? next: 1105 ngx_time_update(0, 0);
931 cache->next_clean_time - now; 1106 }
1107
1108 cache->last = ngx_current_msec;
1109 cache->files = 0;
1110 }
1111
1112 return (ngx_quit || ngx_terminate) ? NGX_ABORT : NGX_OK;
932 } 1113 }
933 1114
934 1115
935 static ngx_int_t 1116 static ngx_int_t
936 ngx_http_file_cache_clean_noop(ngx_tree_ctx_t *ctx, ngx_str_t *path) 1117 ngx_http_file_cache_noop(ngx_tree_ctx_t *ctx, ngx_str_t *path)
937 { 1118 {
938 return NGX_OK; 1119 return NGX_OK;
939 } 1120 }
940 1121
941 1122
942 static ngx_int_t 1123 static ngx_int_t
943 ngx_http_file_cache_clean_file(ngx_tree_ctx_t *ctx, ngx_str_t *path) 1124 ngx_http_file_cache_manage_file(ngx_tree_ctx_t *ctx, ngx_str_t *path)
944 { 1125 {
945 u_char *p, key[2 * NGX_HTTP_CACHE_KEY_LEN]; 1126 ngx_http_file_cache_t *cache;
946 ngx_int_t n; 1127
947 ngx_uint_t i; 1128 cache = ctx->data;
948 ngx_http_file_cache_t *cache; 1129
949 ngx_http_file_cache_node_t *node; 1130 if (ngx_http_file_cache_add_file(ctx, path) != NGX_OK) {
950 1131 (void) ngx_http_file_cache_delete_file(ctx, path);
951 if (path->len < 2 * NGX_HTTP_CACHE_KEY_LEN) { 1132 }
952 goto clean; 1133
953 } 1134 return ngx_http_file_cache_manager_sleep(cache);
954 1135 }
955 p = &path->data[path->len - 2 * NGX_HTTP_CACHE_KEY_LEN]; 1136
1137
1138 static ngx_int_t
1139 ngx_http_file_cache_add_file(ngx_tree_ctx_t *ctx, ngx_str_t *name)
1140 {
1141 u_char *p;
1142 ngx_fd_t fd;
1143 ngx_int_t n;
1144 ngx_uint_t i;
1145 ngx_file_info_t fi;
1146 ngx_http_cache_t c;
1147 ngx_http_file_cache_t *cache;
1148 ngx_http_file_cache_header_t h;
1149
1150 if (name->len < 2 * NGX_HTTP_CACHE_KEY_LEN) {
1151 return NGX_ERROR;
1152 }
1153
1154 ngx_memzero(&c, sizeof(ngx_http_cache_t));
1155
1156 fd = ngx_open_file(name->data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0);
1157
1158 if (fd == NGX_INVALID_FILE) {
1159 ngx_log_error(NGX_LOG_CRIT, ctx->log, ngx_errno,
1160 ngx_open_file_n " \"%s\" failed", name->data);
1161 return NGX_ERROR;
1162 }
1163
1164 c.file.fd = fd;
1165 c.file.name = *name;
1166 c.file.log = ctx->log;
1167
1168 n = ngx_read_file(&c.file, (u_char *) &h,
1169 sizeof(ngx_http_file_cache_header_t), 0);
1170 if (n == NGX_ERROR) {
1171 return NGX_ERROR;
1172 }
1173
1174 if ((size_t) n < sizeof(ngx_http_file_cache_header_t)) {
1175 ngx_log_error(NGX_LOG_CRIT, ctx->log, 0,
1176 "cache file \"%s\" is too small", name->data);
1177 return NGX_ERROR;
1178 }
1179
1180 if (ngx_fd_info(fd, &fi) == NGX_FILE_ERROR) {
1181 ngx_log_error(NGX_LOG_CRIT, ctx->log, ngx_errno,
1182 ngx_fd_info_n " \"%s\" failed", name->data);
1183
1184 } else {
1185 c.uniq = ngx_file_uniq(&fi);
1186 c.valid_sec = h.valid_sec;
1187 c.valid_msec = h.valid_msec;
1188 c.body_start = h.body_start;
1189 c.length = ngx_file_size(&fi);
1190 }
1191
1192 if (ngx_close_file(fd) == NGX_FILE_ERROR) {
1193 ngx_log_error(NGX_LOG_ALERT, ctx->log, ngx_errno,
1194 ngx_close_file_n " \"%s\" failed", name->data);
1195 }
1196
1197 if (c.body_start == 0) {
1198 return NGX_ERROR;
1199 }
1200
1201 p = &name->data[name->len - 2 * NGX_HTTP_CACHE_KEY_LEN];
956 1202
957 for (i = 0; i < NGX_HTTP_CACHE_KEY_LEN; i++) { 1203 for (i = 0; i < NGX_HTTP_CACHE_KEY_LEN; i++) {
958 n = ngx_hextoi(p, 2); 1204 n = ngx_hextoi(p, 2);
959 1205
960 if (n == NGX_ERROR) { 1206 if (n == NGX_ERROR) {
961 goto clean; 1207 return NGX_ERROR;
962 } 1208 }
963 1209
964 p += 2; 1210 p += 2;
965 1211
966 key[i] = (u_char) n; 1212 c.key[i] = (u_char) n;
967 } 1213 }
968 1214
969 cache = ctx->data; 1215 cache = ctx->data;
970 1216
1217 return ngx_http_file_cache_add(cache, &c);
1218 }
1219
1220
1221 static ngx_int_t
1222 ngx_http_file_cache_add(ngx_http_file_cache_t *cache, ngx_http_cache_t *c)
1223 {
1224 ngx_http_file_cache_node_t *fcn;
1225
971 ngx_shmtx_lock(&cache->shpool->mutex); 1226 ngx_shmtx_lock(&cache->shpool->mutex);
972 1227
973 node = ngx_http_file_cache_lookup(cache, key); 1228 fcn = ngx_http_file_cache_lookup(cache, c->key);
1229
1230 if (fcn == NULL) {
1231
1232 fcn = ngx_slab_alloc_locked(cache->shpool,
1233 sizeof(ngx_http_file_cache_node_t));
1234 if (fcn == NULL) {
1235 ngx_shmtx_unlock(&cache->shpool->mutex);
1236 return NGX_ERROR;
1237 }
1238
1239 ngx_memcpy((u_char *) &fcn->node.key, c->key, sizeof(ngx_rbtree_key_t));
1240
1241 ngx_memcpy(fcn->key, &c->key[sizeof(ngx_rbtree_key_t)],
1242 NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t));
1243
1244 ngx_rbtree_insert(cache->rbtree, &fcn->node);
1245
1246 fcn->uses = 1;
1247 fcn->count = 0;
1248 fcn->valid_msec = c->valid_msec;
1249 fcn->error = 0;
1250 fcn->exists = 1;
1251 fcn->uniq = c->uniq;
1252 fcn->valid_sec = c->valid_sec;
1253 fcn->body_start = c->body_start;
1254 fcn->length = c->length;
1255
1256 *cache->size += (c->length + cache->bsize - 1) / cache->bsize;
1257
1258 } else {
1259 ngx_queue_remove(&fcn->queue);
1260 }
1261
1262 fcn->expire = ngx_time() + cache->inactive;
1263
1264 ngx_queue_insert_head(cache->queue, &fcn->queue);
974 1265
975 ngx_shmtx_unlock(&cache->shpool->mutex); 1266 ngx_shmtx_unlock(&cache->shpool->mutex);
976 1267
977 if (node) { 1268 return NGX_OK;
978 return NGX_OK; 1269 }
979 } 1270
980 1271
981 clean: 1272 static ngx_int_t
982 1273 ngx_http_file_cache_delete_file(ngx_tree_ctx_t *ctx, ngx_str_t *path)
1274 {
983 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ctx->log, 0, 1275 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ctx->log, 0,
984 "http file cache clean: \"%s\"", path->data); 1276 "http file cache delete: \"%s\"", path->data);
985 1277
986 if (ngx_delete_file(path->data) == NGX_FILE_ERROR) { 1278 if (ngx_delete_file(path->data) == NGX_FILE_ERROR) {
987 ngx_log_error(NGX_LOG_CRIT, ctx->log, ngx_errno, 1279 ngx_log_error(NGX_LOG_CRIT, ctx->log, ngx_errno,
988 ngx_delete_file_n " \"%s\" failed", path->data); 1280 ngx_delete_file_n " \"%s\" failed", path->data);
989 return NGX_ERROR;
990 } 1281 }
991 1282
992 return NGX_OK; 1283 return NGX_OK;
993 } 1284 }
994 1285
1016 1307
1017 1308
1018 char * 1309 char *
1019 ngx_http_file_cache_set_slot(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) 1310 ngx_http_file_cache_set_slot(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
1020 { 1311 {
1312 off_t max_size;
1021 u_char *last, *p; 1313 u_char *last, *p;
1022 time_t inactive, clean_time, next; 1314 time_t inactive;
1023 ssize_t size; 1315 ssize_t size;
1024 ngx_str_t s, name, *value; 1316 ngx_str_t s, name, *value;
1025 ngx_uint_t i, n; 1317 ngx_uint_t i, n;
1026 ngx_http_file_cache_t *cache; 1318 ngx_http_file_cache_t *cache;
1027 1319
1034 if (cache->path == NULL) { 1326 if (cache->path == NULL) {
1035 return NGX_CONF_ERROR; 1327 return NGX_CONF_ERROR;
1036 } 1328 }
1037 1329
1038 inactive = 600; 1330 inactive = 600;
1039 clean_time = 5 * 60 * 60;
1040 1331
1041 name.len = 0; 1332 name.len = 0;
1042 size = 0; 1333 size = 0;
1334 max_size = NGX_MAX_OFF_T_VALUE;
1043 1335
1044 value = cf->args->elts; 1336 value = cf->args->elts;
1045 1337
1046 cache->path->name = value[1]; 1338 cache->path->name = value[1];
1047 1339
1140 } 1432 }
1141 1433
1142 continue; 1434 continue;
1143 } 1435 }
1144 1436
1145 if (ngx_strncmp(value[i].data, "clean_time=", 11) == 0) { 1437 if (ngx_strncmp(value[i].data, "max_size=", 9) == 0) {
1146 1438
1147 s.len = value[i].len - 11; 1439 s.len = value[i].len - 9;
1148 s.data = value[i].data + 11; 1440 s.data = value[i].data + 9;
1149 1441
1150 clean_time = ngx_parse_time(&s, 1); 1442 max_size = ngx_parse_offset(&s);
1151 if (clean_time < 0 || clean_time > 24 * 60 * 60) { 1443 if (max_size < 0) {
1152 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 1444 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
1153 "invalid clean_time value \"%V\"", &value[i]); 1445 "invalid max_size value \"%V\"", &value[i]);
1154 return NGX_CONF_ERROR; 1446 return NGX_CONF_ERROR;
1155 } 1447 }
1156 1448
1157 continue; 1449 continue;
1158 } 1450 }
1167 "\"%V\" must have \"keys_zone\" parameter", 1459 "\"%V\" must have \"keys_zone\" parameter",
1168 &cmd->name); 1460 &cmd->name);
1169 return NGX_CONF_ERROR; 1461 return NGX_CONF_ERROR;
1170 } 1462 }
1171 1463
1172 cache->path->cleaner = ngx_http_file_cache_cleaner; 1464 cache->path->manager = ngx_http_file_cache_manager;
1173 cache->path->data = cache; 1465 cache->path->data = cache;
1174 1466
1175 if (ngx_add_path(cf, &cache->path) != NGX_OK) { 1467 if (ngx_add_path(cf, &cache->path) != NGX_OK) {
1176 return NGX_CONF_ERROR; 1468 return NGX_CONF_ERROR;
1177 } 1469 }
1185 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 1477 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
1186 "duplicate zone \"%V\"", &name); 1478 "duplicate zone \"%V\"", &name);
1187 return NGX_CONF_ERROR; 1479 return NGX_CONF_ERROR;
1188 } 1480 }
1189 1481
1190 next = ngx_next_time(clean_time);
1191
1192 if (next == -1) {
1193 ngx_conf_log_error(NGX_LOG_EMERG, cf, ngx_errno,
1194 ngx_next_time_n " failed");
1195 return NGX_CONF_ERROR;
1196 }
1197 1482
1198 cache->shm_zone->init = ngx_http_file_cache_init; 1483 cache->shm_zone->init = ngx_http_file_cache_init;
1199 cache->shm_zone->data = cache; 1484 cache->shm_zone->data = cache;
1200 1485
1201 cache->inactive = inactive; 1486 cache->inactive = inactive;
1202 cache->clean_time = clean_time; 1487 cache->max_size = max_size;
1203 cache->next_clean_time = next;
1204 1488
1205 return NGX_CONF_OK; 1489 return NGX_CONF_OK;
1206 } 1490 }
1207 1491
1208 1492