Mercurial > hg > nginx-quic
comparison src/http/ngx_http_file_cache.c @ 2616:d19979e0d980
introduce cache manager instead of cache cleaner
author | Igor Sysoev <igor@sysoev.ru> |
---|---|
date | Mon, 30 Mar 2009 07:45:55 +0000 |
parents | 2bce3f6416c6 |
children | b0cfe5f66e8d |
comparison
equal
deleted
inserted
replaced
2615:ceef364208c8 | 2616:d19979e0d980 |
---|---|
8 #include <ngx_core.h> | 8 #include <ngx_core.h> |
9 #include <ngx_http.h> | 9 #include <ngx_http.h> |
10 #include <ngx_md5.h> | 10 #include <ngx_md5.h> |
11 | 11 |
12 | 12 |
13 static ngx_int_t ngx_http_file_cache_exists(ngx_http_request_t *r, | 13 static ngx_int_t ngx_http_file_cache_exists(ngx_http_file_cache_t *cache, |
14 ngx_http_file_cache_t *cache); | 14 ngx_http_cache_t *c); |
15 static ngx_http_file_cache_node_t * | 15 static ngx_http_file_cache_node_t * |
16 ngx_http_file_cache_lookup(ngx_http_file_cache_t *cache, u_char *key); | 16 ngx_http_file_cache_lookup(ngx_http_file_cache_t *cache, u_char *key); |
17 static void ngx_http_file_cache_rbtree_insert_value(ngx_rbtree_node_t *temp, | 17 static void ngx_http_file_cache_rbtree_insert_value(ngx_rbtree_node_t *temp, |
18 ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel); | 18 ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel); |
19 static void ngx_http_file_cache_cleanup(void *data); | 19 static void ngx_http_file_cache_cleanup(void *data); |
20 static time_t ngx_http_file_cache_expire(ngx_http_file_cache_t *cache, | 20 static time_t ngx_http_file_cache_forced_expire(ngx_http_file_cache_t *cache); |
21 ngx_uint_t force); | 21 static time_t ngx_http_file_cache_expire(ngx_http_file_cache_t *cache); |
22 static ngx_int_t ngx_http_file_cache_clean_noop(ngx_tree_ctx_t *ctx, | 22 static void ngx_http_file_cache_delete(ngx_http_file_cache_t *cache, |
23 ngx_queue_t *q, u_char *name); | |
24 static ngx_int_t | |
25 ngx_http_file_cache_manager_sleep(ngx_http_file_cache_t *cache); | |
26 static ngx_int_t ngx_http_file_cache_noop(ngx_tree_ctx_t *ctx, | |
23 ngx_str_t *path); | 27 ngx_str_t *path); |
24 static ngx_int_t ngx_http_file_cache_clean_file(ngx_tree_ctx_t *ctx, | 28 static ngx_int_t ngx_http_file_cache_manage_file(ngx_tree_ctx_t *ctx, |
29 ngx_str_t *path); | |
30 static ngx_int_t ngx_http_file_cache_add_file(ngx_tree_ctx_t *ctx, | |
31 ngx_str_t *path); | |
32 static ngx_int_t ngx_http_file_cache_add(ngx_http_file_cache_t *cache, | |
33 ngx_http_cache_t *c); | |
34 static ngx_int_t ngx_http_file_cache_delete_file(ngx_tree_ctx_t *ctx, | |
25 ngx_str_t *path); | 35 ngx_str_t *path); |
26 | 36 |
27 | 37 |
28 static u_char ngx_http_file_cache_key[] = { LF, 'K', 'E', 'Y', ':', ' ' }; | 38 static u_char ngx_http_file_cache_key[] = { LF, 'K', 'E', 'Y', ':', ' ' }; |
29 | 39 |
51 } | 61 } |
52 | 62 |
53 cache->rbtree = ocache->rbtree; | 63 cache->rbtree = ocache->rbtree; |
54 cache->queue = ocache->queue; | 64 cache->queue = ocache->queue; |
55 cache->shpool = ocache->shpool; | 65 cache->shpool = ocache->shpool; |
56 cache->created = ocache->created; | 66 cache->cold = ocache->cold; |
67 cache->size = ocache->size; | |
68 cache->bsize = ocache->bsize; | |
69 | |
70 cache->max_size /= cache->bsize; | |
57 | 71 |
58 return NGX_OK; | 72 return NGX_OK; |
59 } | 73 } |
60 | 74 |
61 cache->shpool = (ngx_slab_pool_t *) shm_zone->shm.addr; | 75 cache->shpool = (ngx_slab_pool_t *) shm_zone->shm.addr; |
78 return NGX_ERROR; | 92 return NGX_ERROR; |
79 } | 93 } |
80 | 94 |
81 ngx_queue_init(cache->queue); | 95 ngx_queue_init(cache->queue); |
82 | 96 |
97 cache->cold = ngx_slab_alloc(cache->shpool, sizeof(ngx_atomic_t)); | |
98 if (cache->cold == NULL) { | |
99 return NGX_ERROR; | |
100 } | |
101 | |
102 *cache->cold = 1; | |
103 | |
104 cache->size = ngx_slab_alloc(cache->shpool, sizeof(off_t)); | |
105 if (cache->size == NULL) { | |
106 return NGX_ERROR; | |
107 } | |
108 | |
109 *cache->size = 0; | |
110 | |
111 cache->bsize = ngx_fs_bsize(cache->path->name.data); | |
112 | |
113 cache->max_size /= cache->bsize; | |
114 | |
83 len = sizeof(" in cache keys zone \"\"") + shm_zone->name.len; | 115 len = sizeof(" in cache keys zone \"\"") + shm_zone->name.len; |
84 | 116 |
85 cache->shpool->log_ctx = ngx_slab_alloc(cache->shpool, len); | 117 cache->shpool->log_ctx = ngx_slab_alloc(cache->shpool, len); |
86 if (cache->shpool->log_ctx == NULL) { | 118 if (cache->shpool->log_ctx == NULL) { |
87 return NGX_ERROR; | 119 return NGX_ERROR; |
88 } | 120 } |
89 | 121 |
90 ngx_sprintf(cache->shpool->log_ctx, " in cache keys zone \"%V\"%Z", | 122 ngx_sprintf(cache->shpool->log_ctx, " in cache keys zone \"%V\"%Z", |
91 &shm_zone->name); | 123 &shm_zone->name); |
92 | |
93 cache->created = ngx_time(); | |
94 | 124 |
95 return NGX_OK; | 125 return NGX_OK; |
96 } | 126 } |
97 | 127 |
98 | 128 |
153 cln = ngx_pool_cleanup_add(r->pool, 0); | 183 cln = ngx_pool_cleanup_add(r->pool, 0); |
154 if (cln == NULL) { | 184 if (cln == NULL) { |
155 return NGX_ERROR; | 185 return NGX_ERROR; |
156 } | 186 } |
157 | 187 |
158 rc = ngx_http_file_cache_exists(r, cache); | 188 rc = ngx_http_file_cache_exists(cache, c); |
159 | 189 |
160 ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, | 190 ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, |
161 "http file cache exists: %i u:%ui e:%d", | 191 "http file cache exists: %i u:%ui e:%d", |
162 rc, c->uses, c->exists); | 192 rc, c->uses, c->exists); |
163 | 193 |
170 | 200 |
171 if (rc == NGX_AGAIN) { | 201 if (rc == NGX_AGAIN) { |
172 return rc; | 202 return rc; |
173 } | 203 } |
174 | 204 |
175 now = ngx_time(); | 205 cold = *cache->cold; |
176 | |
177 cold = (now - cache->created < cache->inactive) ? 1 : 0; | |
178 | 206 |
179 if (rc == NGX_OK) { | 207 if (rc == NGX_OK) { |
180 | 208 |
181 if (c->error) { | 209 if (c->error) { |
182 return c->error; | 210 return c->error; |
302 | 330 |
303 if (cold) { | 331 if (cold) { |
304 | 332 |
305 ngx_shmtx_lock(&cache->shpool->mutex); | 333 ngx_shmtx_lock(&cache->shpool->mutex); |
306 | 334 |
307 c->node->uses = c->min_uses; | 335 if (!c->node->exists) { |
308 c->node->body_start = c->body_start; | 336 c->node->uses = 1; |
309 c->node->exists = 1; | 337 c->node->body_start = c->body_start; |
338 c->node->exists = 1; | |
339 | |
340 *cache->size += (c->length + cache->bsize - 1) / cache->bsize; | |
341 } | |
310 | 342 |
311 ngx_shmtx_unlock(&cache->shpool->mutex); | 343 ngx_shmtx_unlock(&cache->shpool->mutex); |
312 } | 344 } |
345 | |
346 now = ngx_time(); | |
313 | 347 |
314 if (c->valid_sec < now) { | 348 if (c->valid_sec < now) { |
315 | 349 |
316 c->uses = c->min_uses; | 350 c->uses = c->min_uses; |
317 | 351 |
326 return NGX_OK; | 360 return NGX_OK; |
327 } | 361 } |
328 | 362 |
329 | 363 |
330 static ngx_int_t | 364 static ngx_int_t |
331 ngx_http_file_cache_exists(ngx_http_request_t *r, ngx_http_file_cache_t *cache) | 365 ngx_http_file_cache_exists(ngx_http_file_cache_t *cache, ngx_http_cache_t *c) |
332 { | 366 { |
333 ngx_int_t rc; | 367 ngx_int_t rc; |
334 ngx_http_file_cache_node_t *fcn; | 368 ngx_http_file_cache_node_t *fcn; |
335 | 369 |
336 ngx_shmtx_lock(&cache->shpool->mutex); | 370 ngx_shmtx_lock(&cache->shpool->mutex); |
337 | 371 |
338 fcn = ngx_http_file_cache_lookup(cache, r->cache->key); | 372 fcn = ngx_http_file_cache_lookup(cache, c->key); |
339 | 373 |
340 if (fcn) { | 374 if (fcn) { |
341 ngx_queue_remove(&fcn->queue); | 375 ngx_queue_remove(&fcn->queue); |
342 | 376 |
343 if (fcn->error) { | 377 if (fcn->error) { |
354 fcn->uses++; | 388 fcn->uses++; |
355 fcn->count++; | 389 fcn->count++; |
356 | 390 |
357 if (fcn->exists) { | 391 if (fcn->exists) { |
358 | 392 |
359 r->cache->exists = fcn->exists; | 393 c->exists = fcn->exists; |
360 r->cache->body_start = fcn->body_start; | 394 c->body_start = fcn->body_start; |
361 | 395 |
362 rc = NGX_OK; | 396 rc = NGX_OK; |
363 | 397 |
364 goto done; | 398 goto done; |
365 } | 399 } |
366 | 400 |
367 if (fcn->uses >= r->cache->min_uses) { | 401 if (fcn->uses >= c->min_uses) { |
368 | 402 |
369 r->cache->exists = fcn->exists; | 403 c->exists = fcn->exists; |
370 r->cache->body_start = fcn->body_start; | 404 c->body_start = fcn->body_start; |
371 | 405 |
372 rc = NGX_OK; | 406 rc = NGX_OK; |
373 | 407 |
374 } else { | 408 } else { |
375 rc = NGX_AGAIN; | 409 rc = NGX_AGAIN; |
381 fcn = ngx_slab_alloc_locked(cache->shpool, | 415 fcn = ngx_slab_alloc_locked(cache->shpool, |
382 sizeof(ngx_http_file_cache_node_t)); | 416 sizeof(ngx_http_file_cache_node_t)); |
383 if (fcn == NULL) { | 417 if (fcn == NULL) { |
384 ngx_shmtx_unlock(&cache->shpool->mutex); | 418 ngx_shmtx_unlock(&cache->shpool->mutex); |
385 | 419 |
386 (void) ngx_http_file_cache_expire(cache, 1); | 420 ngx_http_file_cache_forced_expire(cache); |
387 | 421 |
388 ngx_shmtx_lock(&cache->shpool->mutex); | 422 ngx_shmtx_lock(&cache->shpool->mutex); |
389 | 423 |
390 fcn = ngx_slab_alloc_locked(cache->shpool, | 424 fcn = ngx_slab_alloc_locked(cache->shpool, |
391 sizeof(ngx_http_file_cache_node_t)); | 425 sizeof(ngx_http_file_cache_node_t)); |
393 rc = NGX_ERROR; | 427 rc = NGX_ERROR; |
394 goto failed; | 428 goto failed; |
395 } | 429 } |
396 } | 430 } |
397 | 431 |
398 ngx_memcpy((u_char *) &fcn->node.key, r->cache->key, | 432 ngx_memcpy((u_char *) &fcn->node.key, c->key, sizeof(ngx_rbtree_key_t)); |
399 sizeof(ngx_rbtree_key_t)); | 433 |
400 | 434 ngx_memcpy(fcn->key, &c->key[sizeof(ngx_rbtree_key_t)], |
401 ngx_memcpy(fcn->key, &r->cache->key[sizeof(ngx_rbtree_key_t)], | |
402 NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t)); | 435 NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t)); |
403 | 436 |
404 ngx_rbtree_insert(cache->rbtree, &fcn->node); | 437 ngx_rbtree_insert(cache->rbtree, &fcn->node); |
405 | 438 |
406 renew: | 439 renew: |
420 | 453 |
421 fcn->expire = ngx_time() + cache->inactive; | 454 fcn->expire = ngx_time() + cache->inactive; |
422 | 455 |
423 ngx_queue_insert_head(cache->queue, &fcn->queue); | 456 ngx_queue_insert_head(cache->queue, &fcn->queue); |
424 | 457 |
425 r->cache->uniq = fcn->uniq; | 458 c->uniq = fcn->uniq; |
426 r->cache->uses = fcn->uses; | 459 c->uses = fcn->uses; |
427 r->cache->error = fcn->error; | 460 c->error = fcn->error; |
428 r->cache->node = fcn; | 461 c->node = fcn; |
429 | 462 |
430 failed: | 463 failed: |
431 | 464 |
432 ngx_shmtx_unlock(&cache->shpool->mutex); | 465 ngx_shmtx_unlock(&cache->shpool->mutex); |
433 | 466 |
565 | 598 |
566 | 599 |
567 void | 600 void |
568 ngx_http_file_cache_update(ngx_http_request_t *r, ngx_temp_file_t *tf) | 601 ngx_http_file_cache_update(ngx_http_request_t *r, ngx_temp_file_t *tf) |
569 { | 602 { |
603 off_t size; | |
570 ngx_int_t rc; | 604 ngx_int_t rc; |
571 ngx_file_uniq_t uniq; | 605 ngx_file_uniq_t uniq; |
572 ngx_file_info_t fi; | 606 ngx_file_info_t fi; |
573 ngx_http_cache_t *c; | 607 ngx_http_cache_t *c; |
574 ngx_ext_rename_file_t ext; | 608 ngx_ext_rename_file_t ext; |
614 } else { | 648 } else { |
615 uniq = ngx_file_uniq(&fi); | 649 uniq = ngx_file_uniq(&fi); |
616 } | 650 } |
617 } | 651 } |
618 | 652 |
653 size = (c->length + cache->bsize - 1) / cache->bsize; | |
654 | |
619 ngx_shmtx_lock(&cache->shpool->mutex); | 655 ngx_shmtx_lock(&cache->shpool->mutex); |
620 | 656 |
621 c->node->count--; | 657 c->node->count--; |
622 c->node->uniq = uniq; | 658 c->node->uniq = uniq; |
623 c->node->body_start = c->body_start; | 659 c->node->body_start = c->body_start; |
660 | |
661 size = size - (c->node->length + cache->bsize - 1) / cache->bsize; | |
662 | |
663 c->node->length = c->length; | |
664 | |
665 *cache->size += size; | |
624 | 666 |
625 if (rc == NGX_OK) { | 667 if (rc == NGX_OK) { |
626 c->node->exists = 1; | 668 c->node->exists = 1; |
627 } | 669 } |
628 | 670 |
755 ngx_shmtx_unlock(&cache->shpool->mutex); | 797 ngx_shmtx_unlock(&cache->shpool->mutex); |
756 } | 798 } |
757 | 799 |
758 | 800 |
759 static time_t | 801 static time_t |
760 ngx_http_file_cache_expire(ngx_http_file_cache_t *cache, ngx_uint_t forced) | 802 ngx_http_file_cache_forced_expire(ngx_http_file_cache_t *cache) |
761 { | 803 { |
762 u_char *name, *p; | 804 u_char *name; |
763 size_t len; | 805 size_t len; |
764 time_t now, wait; | 806 time_t wait; |
765 ngx_uint_t tries; | 807 ngx_uint_t tries; |
766 ngx_path_t *path; | 808 ngx_path_t *path; |
767 ngx_queue_t *q; | 809 ngx_queue_t *q; |
768 ngx_http_file_cache_node_t *fcn; | 810 ngx_http_file_cache_node_t *fcn; |
811 | |
812 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, | |
813 "http file cache forced expire"); | |
814 | |
815 path = cache->path; | |
816 len = path->name.len + 1 + path->len + 2 * NGX_HTTP_CACHE_KEY_LEN; | |
817 | |
818 name = ngx_alloc(len + 1, ngx_cycle->log); | |
819 if (name == NULL) { | |
820 return 60; | |
821 } | |
822 | |
823 ngx_memcpy(name, path->name.data, path->name.len); | |
824 | |
825 wait = 60; | |
826 tries = 0; | |
827 | |
828 ngx_shmtx_lock(&cache->shpool->mutex); | |
829 | |
830 for (q = ngx_queue_last(cache->queue); | |
831 q != ngx_queue_sentinel(cache->queue); | |
832 q = ngx_queue_prev(q)) | |
833 { | |
834 fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue); | |
835 | |
836 ngx_log_debug6(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, | |
837 "http file cache forced expire: #%d %d %02xd%02xd%02xd%02xd", | |
838 fcn->count, fcn->exists, | |
839 fcn->key[0], fcn->key[1], fcn->key[2], fcn->key[3]); | |
840 | |
841 if (fcn->count) { | |
842 | |
843 if (tries++ < 20) { | |
844 continue; | |
845 } | |
846 | |
847 wait = 1; | |
848 | |
849 break; | |
850 } | |
851 | |
852 if (!fcn->exists) { | |
853 | |
854 ngx_queue_remove(q); | |
855 ngx_rbtree_delete(cache->rbtree, &fcn->node); | |
856 ngx_slab_free_locked(cache->shpool, fcn); | |
857 | |
858 break; | |
859 } | |
860 | |
861 ngx_http_file_cache_delete(cache, q, name); | |
862 | |
863 break; | |
864 } | |
865 | |
866 ngx_shmtx_unlock(&cache->shpool->mutex); | |
867 | |
868 ngx_free(name); | |
869 | |
870 return wait; | |
871 } | |
872 | |
873 | |
874 static time_t | |
875 ngx_http_file_cache_expire(ngx_http_file_cache_t *cache) | |
876 { | |
877 u_char *name, *p; | |
878 size_t len; | |
879 time_t now, wait; | |
880 ngx_path_t *path; | |
881 ngx_queue_t *q; | |
882 ngx_http_file_cache_node_t *fcn; | |
769 u_char key[2 * NGX_HTTP_CACHE_KEY_LEN]; | 883 u_char key[2 * NGX_HTTP_CACHE_KEY_LEN]; |
770 | 884 |
771 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, | 885 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, |
772 "http file cache expire"); | 886 "http file cache expire"); |
773 | 887 |
774 path = cache->path; | 888 path = cache->path; |
775 len = path->name.len + 1 + path->len + 2 * NGX_HTTP_CACHE_KEY_LEN; | 889 len = path->name.len + 1 + path->len + 2 * NGX_HTTP_CACHE_KEY_LEN; |
776 | 890 |
891 name = ngx_alloc(len + 1, ngx_cycle->log); | |
892 if (name == NULL) { | |
893 return 60; | |
894 } | |
895 | |
896 ngx_memcpy(name, path->name.data, path->name.len); | |
897 | |
777 now = ngx_time(); | 898 now = ngx_time(); |
778 | 899 |
779 ngx_shmtx_lock(&cache->shpool->mutex); | 900 ngx_shmtx_lock(&cache->shpool->mutex); |
780 | 901 |
781 tries = 0; | |
782 | |
783 for ( ;; ) { | 902 for ( ;; ) { |
784 | 903 |
785 if (ngx_queue_empty(cache->queue)) { | 904 if (ngx_queue_empty(cache->queue)) { |
786 wait = cache->inactive; | 905 wait = 60; |
787 break; | 906 break; |
788 } | 907 } |
789 | 908 |
790 q = ngx_queue_last(cache->queue); | 909 q = ngx_queue_last(cache->queue); |
791 | 910 |
792 fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue); | 911 fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue); |
793 | 912 |
794 if (!forced) { | 913 wait = fcn->expire - now; |
795 wait = fcn->expire - now; | 914 |
796 | 915 if (wait > 0) { |
797 if (wait > 0) { | 916 wait = wait > 60 ? 60 : wait; |
798 break; | 917 break; |
799 } | |
800 } | 918 } |
801 | 919 |
802 ngx_log_debug6(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, | 920 ngx_log_debug6(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, |
803 "http file cache expire: #%d %d %02xd%02xd%02xd%02xd", | 921 "http file cache expire: #%d %d %02xd%02xd%02xd%02xd", |
804 fcn->count, fcn->exists, | 922 fcn->count, fcn->exists, |
805 fcn->key[0], fcn->key[1], fcn->key[2], fcn->key[3]); | 923 fcn->key[0], fcn->key[1], fcn->key[2], fcn->key[3]); |
806 | 924 |
807 if (fcn->count) { | 925 if (fcn->count) { |
808 | 926 |
809 if (!forced) { | 927 p = ngx_hex_dump(key, (u_char *) &fcn->node.key, |
810 | 928 sizeof(ngx_rbtree_key_t)); |
811 p = ngx_hex_dump(key, (u_char *) &fcn->node.key, | 929 |
812 sizeof(ngx_rbtree_key_t)); | 930 len = NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t); |
813 (void) ngx_hex_dump(p, fcn->key, | 931 (void) ngx_hex_dump(p, fcn->key, len); |
814 NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t)); | 932 |
815 | 933 /* |
816 /* | 934 * abnormally exited workers may leave locked cache entries, |
817 * abnormally exited workers may leave locked cache entries, | 935 * and although it may be safe to remove them completely, |
818 * and although it may be safe to remove them completely, | 936 * we prefer to remove them from inactive queue and rbtree |
819 * we prefer to remove them from inactive queue and rbtree | 937 * only, and to allow other leaks |
820 * only, and to allow other leaks | 938 */ |
821 */ | 939 |
822 | 940 ngx_queue_remove(q); |
823 ngx_queue_remove(q); | 941 ngx_rbtree_delete(cache->rbtree, &fcn->node); |
824 | 942 |
825 ngx_rbtree_delete(cache->rbtree, &fcn->node); | 943 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0, |
826 | |
827 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0, | |
828 "ignore long locked inactive cache entry %*s, count:%d", | 944 "ignore long locked inactive cache entry %*s, count:%d", |
829 2 * NGX_HTTP_CACHE_KEY_LEN, key, fcn->count); | 945 2 * NGX_HTTP_CACHE_KEY_LEN, key, fcn->count); |
830 } | 946 |
831 | 947 continue; |
832 if (tries++ < 10) { | 948 } |
833 continue; | |
834 } | |
835 | |
836 wait = 1; | |
837 break; | |
838 } | |
839 | |
840 forced = 0; | |
841 | 949 |
842 if (!fcn->exists) { | 950 if (!fcn->exists) { |
843 | 951 |
844 ngx_queue_remove(q); | 952 ngx_queue_remove(q); |
845 | |
846 ngx_rbtree_delete(cache->rbtree, &fcn->node); | 953 ngx_rbtree_delete(cache->rbtree, &fcn->node); |
847 | |
848 ngx_slab_free_locked(cache->shpool, fcn); | 954 ngx_slab_free_locked(cache->shpool, fcn); |
849 | 955 |
850 continue; | 956 continue; |
851 } | 957 } |
852 | 958 |
853 name = ngx_alloc(len + 1, ngx_cycle->log); | 959 ngx_http_file_cache_delete(cache, q, name); |
854 if (name == NULL) { | |
855 wait = 60; | |
856 break; | |
857 } | |
858 | |
859 ngx_memcpy(name, path->name.data, path->name.len); | |
860 | |
861 p = name + path->name.len + 1 + path->len; | |
862 p = ngx_hex_dump(p, (u_char *) &fcn->node.key, | |
863 sizeof(ngx_rbtree_key_t)); | |
864 p = ngx_hex_dump(p, fcn->key, | |
865 NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t)); | |
866 *p = '\0'; | |
867 | |
868 ngx_queue_remove(q); | |
869 | |
870 ngx_rbtree_delete(cache->rbtree, &fcn->node); | |
871 | |
872 ngx_slab_free_locked(cache->shpool, fcn); | |
873 | |
874 ngx_shmtx_unlock(&cache->shpool->mutex); | |
875 | |
876 ngx_create_hashed_filename(path, name, len); | |
877 | |
878 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, | |
879 "http file cache expire: \"%s\"", name); | |
880 | |
881 if (ngx_delete_file(name) == NGX_FILE_ERROR) { | |
882 ngx_log_error(NGX_LOG_CRIT, ngx_cycle->log, ngx_errno, | |
883 ngx_delete_file_n " \"%s\" failed", name); | |
884 } | |
885 | |
886 ngx_free(name); | |
887 | |
888 ngx_shmtx_lock(&cache->shpool->mutex); | |
889 } | 960 } |
890 | 961 |
891 ngx_shmtx_unlock(&cache->shpool->mutex); | 962 ngx_shmtx_unlock(&cache->shpool->mutex); |
892 | 963 |
964 ngx_free(name); | |
965 | |
893 return wait; | 966 return wait; |
894 } | 967 } |
895 | 968 |
896 | 969 |
897 time_t | 970 static void |
898 ngx_http_file_cache_cleaner(void *data) | 971 ngx_http_file_cache_delete(ngx_http_file_cache_t *cache, ngx_queue_t *q, |
972 u_char *name) | |
973 { | |
974 u_char *p; | |
975 size_t len; | |
976 ngx_path_t *path; | |
977 ngx_http_file_cache_node_t *fcn; | |
978 | |
979 fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue); | |
980 | |
981 *cache->size -= (fcn->length + cache->bsize - 1) / cache->bsize; | |
982 | |
983 path = cache->path; | |
984 | |
985 p = name + path->name.len + 1 + path->len; | |
986 | |
987 p = ngx_hex_dump(p, (u_char *) &fcn->node.key, sizeof(ngx_rbtree_key_t)); | |
988 | |
989 len = NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t); | |
990 p = ngx_hex_dump(p, fcn->key, len); | |
991 *p = '\0'; | |
992 | |
993 ngx_queue_remove(q); | |
994 | |
995 ngx_rbtree_delete(cache->rbtree, &fcn->node); | |
996 | |
997 ngx_slab_free_locked(cache->shpool, fcn); | |
998 | |
999 ngx_shmtx_unlock(&cache->shpool->mutex); | |
1000 | |
1001 len = path->name.len + 1 + path->len + 2 * NGX_HTTP_CACHE_KEY_LEN; | |
1002 | |
1003 ngx_create_hashed_filename(path, name, len); | |
1004 | |
1005 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, | |
1006 "http file cache expire: \"%s\"", name); | |
1007 | |
1008 if (ngx_delete_file(name) == NGX_FILE_ERROR) { | |
1009 ngx_log_error(NGX_LOG_CRIT, ngx_cycle->log, ngx_errno, | |
1010 ngx_delete_file_n " \"%s\" failed", name); | |
1011 } | |
1012 | |
1013 ngx_shmtx_lock(&cache->shpool->mutex); | |
1014 } | |
1015 | |
1016 | |
1017 static time_t | |
1018 ngx_http_file_cache_manager(void *data) | |
899 { | 1019 { |
900 ngx_http_file_cache_t *cache = data; | 1020 ngx_http_file_cache_t *cache = data; |
901 | 1021 |
902 time_t now, next; | 1022 off_t size; |
1023 time_t next; | |
903 ngx_tree_ctx_t tree; | 1024 ngx_tree_ctx_t tree; |
904 | 1025 |
905 now = ngx_time(); | 1026 if (*cache->cold) { |
906 | 1027 |
907 if (now >= cache->next_clean_time | 1028 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, |
908 && now >= cache->created + cache->inactive) | 1029 "http file cache manager update"); |
909 { | |
910 ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0, | |
911 "clean unused cache files"); | |
912 | 1030 |
913 tree.init_handler = NULL; | 1031 tree.init_handler = NULL; |
914 tree.file_handler = ngx_http_file_cache_clean_file; | 1032 tree.file_handler = ngx_http_file_cache_manage_file; |
915 tree.pre_tree_handler = ngx_http_file_cache_clean_noop; | 1033 tree.pre_tree_handler = ngx_http_file_cache_noop; |
916 tree.post_tree_handler = ngx_http_file_cache_clean_noop; | 1034 tree.post_tree_handler = ngx_http_file_cache_noop; |
917 tree.spec_handler = ngx_http_file_cache_clean_file; | 1035 tree.spec_handler = ngx_http_file_cache_delete_file; |
918 tree.data = cache; | 1036 tree.data = cache; |
919 tree.alloc = 0; | 1037 tree.alloc = 0; |
920 tree.log = ngx_cycle->log; | 1038 tree.log = ngx_cycle->log; |
921 | 1039 |
922 (void) ngx_walk_tree(&tree, &cache->path->name); | 1040 cache->last = ngx_current_msec; |
1041 cache->files = 0; | |
1042 | |
1043 if (ngx_walk_tree(&tree, &cache->path->name) == NGX_ABORT) { | |
1044 return 60; | |
1045 } | |
1046 | |
1047 *cache->cold = 0; | |
1048 | |
1049 ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0, | |
1050 "http file cache: %V %.3fM, bsize: %uz", | |
1051 &cache->path->name, | |
1052 ((double) *cache->size * cache->bsize) / (1024 * 1024), | |
1053 cache->bsize); | |
1054 } | |
1055 | |
1056 next = ngx_http_file_cache_expire(cache); | |
1057 | |
1058 cache->last = ngx_current_msec; | |
1059 cache->files = 0; | |
1060 | |
1061 for ( ;; ) { | |
1062 ngx_shmtx_lock(&cache->shpool->mutex); | |
1063 | |
1064 size = *cache->size; | |
1065 | |
1066 ngx_shmtx_unlock(&cache->shpool->mutex); | |
1067 | |
1068 if (size < cache->max_size) { | |
1069 return next; | |
1070 } | |
1071 | |
1072 next = ngx_http_file_cache_forced_expire(cache); | |
1073 | |
1074 if (ngx_http_file_cache_manager_sleep(cache) != NGX_OK) { | |
1075 return next; | |
1076 } | |
1077 } | |
1078 } | |
1079 | |
1080 | |
1081 static ngx_int_t | |
1082 ngx_http_file_cache_manager_sleep(ngx_http_file_cache_t *cache) | |
1083 { | |
1084 ngx_msec_t elapsed; | |
1085 | |
1086 if (cache->files++ > 100) { | |
923 | 1087 |
924 ngx_time_update(0, 0); | 1088 ngx_time_update(0, 0); |
925 | 1089 |
926 next = ngx_next_time(cache->clean_time); | 1090 elapsed = ngx_abs((ngx_msec_int_t) (ngx_current_msec - cache->last)); |
927 | 1091 |
928 if (next == -1) { | 1092 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, |
929 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, ngx_errno, | 1093 "http file cache manager time: %M", elapsed); |
930 ngx_next_time_n " failed"); | 1094 |
931 return 60; | 1095 if (elapsed > 200) { |
932 } | 1096 |
933 | 1097 /* |
934 cache->next_clean_time = next; | 1098 * if processing 100 files takes more than 200ms, |
935 } | 1099 * it seems that many operations require disk i/o, |
936 | 1100 * therefore sleep 200ms |
937 next = ngx_http_file_cache_expire(cache, 0); | 1101 */ |
938 | 1102 |
939 now = ngx_time(); | 1103 ngx_msleep(200); |
940 | 1104 |
941 return (now + next < cache->next_clean_time) ? next: | 1105 ngx_time_update(0, 0); |
942 cache->next_clean_time - now; | 1106 } |
1107 | |
1108 cache->last = ngx_current_msec; | |
1109 cache->files = 0; | |
1110 } | |
1111 | |
1112 return (ngx_quit || ngx_terminate) ? NGX_ABORT : NGX_OK; | |
943 } | 1113 } |
944 | 1114 |
945 | 1115 |
946 static ngx_int_t | 1116 static ngx_int_t |
947 ngx_http_file_cache_clean_noop(ngx_tree_ctx_t *ctx, ngx_str_t *path) | 1117 ngx_http_file_cache_noop(ngx_tree_ctx_t *ctx, ngx_str_t *path) |
948 { | 1118 { |
949 return NGX_OK; | 1119 return NGX_OK; |
950 } | 1120 } |
951 | 1121 |
952 | 1122 |
953 static ngx_int_t | 1123 static ngx_int_t |
954 ngx_http_file_cache_clean_file(ngx_tree_ctx_t *ctx, ngx_str_t *path) | 1124 ngx_http_file_cache_manage_file(ngx_tree_ctx_t *ctx, ngx_str_t *path) |
955 { | 1125 { |
956 u_char *p, key[2 * NGX_HTTP_CACHE_KEY_LEN]; | 1126 ngx_http_file_cache_t *cache; |
957 ngx_int_t n; | 1127 |
958 ngx_uint_t i; | 1128 cache = ctx->data; |
959 ngx_http_file_cache_t *cache; | 1129 |
960 ngx_http_file_cache_node_t *node; | 1130 if (ngx_http_file_cache_add_file(ctx, path) != NGX_OK) { |
961 | 1131 (void) ngx_http_file_cache_delete_file(ctx, path); |
962 if (path->len < 2 * NGX_HTTP_CACHE_KEY_LEN) { | 1132 } |
963 goto clean; | 1133 |
964 } | 1134 return ngx_http_file_cache_manager_sleep(cache); |
965 | 1135 } |
966 p = &path->data[path->len - 2 * NGX_HTTP_CACHE_KEY_LEN]; | 1136 |
1137 | |
1138 static ngx_int_t | |
1139 ngx_http_file_cache_add_file(ngx_tree_ctx_t *ctx, ngx_str_t *name) | |
1140 { | |
1141 u_char *p; | |
1142 ngx_fd_t fd; | |
1143 ngx_int_t n; | |
1144 ngx_uint_t i; | |
1145 ngx_file_info_t fi; | |
1146 ngx_http_cache_t c; | |
1147 ngx_http_file_cache_t *cache; | |
1148 ngx_http_file_cache_header_t h; | |
1149 | |
1150 if (name->len < 2 * NGX_HTTP_CACHE_KEY_LEN) { | |
1151 return NGX_ERROR; | |
1152 } | |
1153 | |
1154 ngx_memzero(&c, sizeof(ngx_http_cache_t)); | |
1155 | |
1156 fd = ngx_open_file(name->data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0); | |
1157 | |
1158 if (fd == NGX_INVALID_FILE) { | |
1159 ngx_log_error(NGX_LOG_CRIT, ctx->log, ngx_errno, | |
1160 ngx_open_file_n " \"%s\" failed", name->data); | |
1161 return NGX_ERROR; | |
1162 } | |
1163 | |
1164 c.file.fd = fd; | |
1165 c.file.name = *name; | |
1166 c.file.log = ctx->log; | |
1167 | |
1168 n = ngx_read_file(&c.file, (u_char *) &h, | |
1169 sizeof(ngx_http_file_cache_header_t), 0); | |
1170 if (n == NGX_ERROR) { | |
1171 return NGX_ERROR; | |
1172 } | |
1173 | |
1174 if ((size_t) n < sizeof(ngx_http_file_cache_header_t)) { | |
1175 ngx_log_error(NGX_LOG_CRIT, ctx->log, 0, | |
1176 "cache file \"%s\" is too small", name->data); | |
1177 return NGX_ERROR; | |
1178 } | |
1179 | |
1180 if (ngx_fd_info(fd, &fi) == NGX_FILE_ERROR) { | |
1181 ngx_log_error(NGX_LOG_CRIT, ctx->log, ngx_errno, | |
1182 ngx_fd_info_n " \"%s\" failed", name->data); | |
1183 | |
1184 } else { | |
1185 c.uniq = ngx_file_uniq(&fi); | |
1186 c.valid_sec = h.valid_sec; | |
1187 c.valid_msec = h.valid_msec; | |
1188 c.body_start = h.body_start; | |
1189 c.length = ngx_file_size(&fi); | |
1190 } | |
1191 | |
1192 if (ngx_close_file(fd) == NGX_FILE_ERROR) { | |
1193 ngx_log_error(NGX_LOG_ALERT, ctx->log, ngx_errno, | |
1194 ngx_close_file_n " \"%s\" failed", name->data); | |
1195 } | |
1196 | |
1197 if (c.body_start == 0) { | |
1198 return NGX_ERROR; | |
1199 } | |
1200 | |
1201 p = &name->data[name->len - 2 * NGX_HTTP_CACHE_KEY_LEN]; | |
967 | 1202 |
968 for (i = 0; i < NGX_HTTP_CACHE_KEY_LEN; i++) { | 1203 for (i = 0; i < NGX_HTTP_CACHE_KEY_LEN; i++) { |
969 n = ngx_hextoi(p, 2); | 1204 n = ngx_hextoi(p, 2); |
970 | 1205 |
971 if (n == NGX_ERROR) { | 1206 if (n == NGX_ERROR) { |
972 goto clean; | 1207 return NGX_ERROR; |
973 } | 1208 } |
974 | 1209 |
975 p += 2; | 1210 p += 2; |
976 | 1211 |
977 key[i] = (u_char) n; | 1212 c.key[i] = (u_char) n; |
978 } | 1213 } |
979 | 1214 |
980 cache = ctx->data; | 1215 cache = ctx->data; |
981 | 1216 |
1217 return ngx_http_file_cache_add(cache, &c); | |
1218 } | |
1219 | |
1220 | |
1221 static ngx_int_t | |
1222 ngx_http_file_cache_add(ngx_http_file_cache_t *cache, ngx_http_cache_t *c) | |
1223 { | |
1224 ngx_http_file_cache_node_t *fcn; | |
1225 | |
982 ngx_shmtx_lock(&cache->shpool->mutex); | 1226 ngx_shmtx_lock(&cache->shpool->mutex); |
983 | 1227 |
984 node = ngx_http_file_cache_lookup(cache, key); | 1228 fcn = ngx_http_file_cache_lookup(cache, c->key); |
1229 | |
1230 if (fcn == NULL) { | |
1231 | |
1232 fcn = ngx_slab_alloc_locked(cache->shpool, | |
1233 sizeof(ngx_http_file_cache_node_t)); | |
1234 if (fcn == NULL) { | |
1235 ngx_shmtx_unlock(&cache->shpool->mutex); | |
1236 return NGX_ERROR; | |
1237 } | |
1238 | |
1239 ngx_memcpy((u_char *) &fcn->node.key, c->key, sizeof(ngx_rbtree_key_t)); | |
1240 | |
1241 ngx_memcpy(fcn->key, &c->key[sizeof(ngx_rbtree_key_t)], | |
1242 NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t)); | |
1243 | |
1244 ngx_rbtree_insert(cache->rbtree, &fcn->node); | |
1245 | |
1246 fcn->uses = 1; | |
1247 fcn->count = 0; | |
1248 fcn->valid_msec = c->valid_msec; | |
1249 fcn->error = 0; | |
1250 fcn->exists = 1; | |
1251 fcn->uniq = c->uniq; | |
1252 fcn->valid_sec = c->valid_sec; | |
1253 fcn->body_start = c->body_start; | |
1254 fcn->length = c->length; | |
1255 | |
1256 *cache->size += (c->length + cache->bsize - 1) / cache->bsize; | |
1257 | |
1258 } else { | |
1259 ngx_queue_remove(&fcn->queue); | |
1260 } | |
1261 | |
1262 fcn->expire = ngx_time() + cache->inactive; | |
1263 | |
1264 ngx_queue_insert_head(cache->queue, &fcn->queue); | |
985 | 1265 |
986 ngx_shmtx_unlock(&cache->shpool->mutex); | 1266 ngx_shmtx_unlock(&cache->shpool->mutex); |
987 | 1267 |
988 if (node) { | 1268 return NGX_OK; |
989 return NGX_OK; | 1269 } |
990 } | 1270 |
991 | 1271 |
992 clean: | 1272 static ngx_int_t |
993 | 1273 ngx_http_file_cache_delete_file(ngx_tree_ctx_t *ctx, ngx_str_t *path) |
1274 { | |
994 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ctx->log, 0, | 1275 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ctx->log, 0, |
995 "http file cache clean: \"%s\"", path->data); | 1276 "http file cache delete: \"%s\"", path->data); |
996 | 1277 |
997 if (ngx_delete_file(path->data) == NGX_FILE_ERROR) { | 1278 if (ngx_delete_file(path->data) == NGX_FILE_ERROR) { |
998 ngx_log_error(NGX_LOG_CRIT, ctx->log, ngx_errno, | 1279 ngx_log_error(NGX_LOG_CRIT, ctx->log, ngx_errno, |
999 ngx_delete_file_n " \"%s\" failed", path->data); | 1280 ngx_delete_file_n " \"%s\" failed", path->data); |
1000 return NGX_ERROR; | |
1001 } | 1281 } |
1002 | 1282 |
1003 return NGX_OK; | 1283 return NGX_OK; |
1004 } | 1284 } |
1005 | 1285 |
1027 | 1307 |
1028 | 1308 |
1029 char * | 1309 char * |
1030 ngx_http_file_cache_set_slot(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) | 1310 ngx_http_file_cache_set_slot(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) |
1031 { | 1311 { |
1312 off_t max_size; | |
1032 u_char *last, *p; | 1313 u_char *last, *p; |
1033 time_t inactive, clean_time, next; | 1314 time_t inactive; |
1034 ssize_t size; | 1315 ssize_t size; |
1035 ngx_str_t s, name, *value; | 1316 ngx_str_t s, name, *value; |
1036 ngx_uint_t i, n; | 1317 ngx_uint_t i, n; |
1037 ngx_http_file_cache_t *cache; | 1318 ngx_http_file_cache_t *cache; |
1038 | 1319 |
1045 if (cache->path == NULL) { | 1326 if (cache->path == NULL) { |
1046 return NGX_CONF_ERROR; | 1327 return NGX_CONF_ERROR; |
1047 } | 1328 } |
1048 | 1329 |
1049 inactive = 600; | 1330 inactive = 600; |
1050 clean_time = 5 * 60 * 60; | |
1051 | 1331 |
1052 name.len = 0; | 1332 name.len = 0; |
1053 size = 0; | 1333 size = 0; |
1334 max_size = NGX_MAX_OFF_T_VALUE; | |
1054 | 1335 |
1055 value = cf->args->elts; | 1336 value = cf->args->elts; |
1056 | 1337 |
1057 cache->path->name = value[1]; | 1338 cache->path->name = value[1]; |
1058 | 1339 |
1151 } | 1432 } |
1152 | 1433 |
1153 continue; | 1434 continue; |
1154 } | 1435 } |
1155 | 1436 |
1156 if (ngx_strncmp(value[i].data, "clean_time=", 11) == 0) { | 1437 if (ngx_strncmp(value[i].data, "max_size=", 9) == 0) { |
1157 | 1438 |
1158 s.len = value[i].len - 11; | 1439 s.len = value[i].len - 9; |
1159 s.data = value[i].data + 11; | 1440 s.data = value[i].data + 9; |
1160 | 1441 |
1161 clean_time = ngx_parse_time(&s, 1); | 1442 max_size = ngx_parse_offset(&s); |
1162 if (clean_time < 0 || clean_time > 24 * 60 * 60) { | 1443 if (max_size < 0) { |
1163 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, | 1444 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, |
1164 "invalid clean_time value \"%V\"", &value[i]); | 1445 "invalid max_size value \"%V\"", &value[i]); |
1165 return NGX_CONF_ERROR; | 1446 return NGX_CONF_ERROR; |
1166 } | 1447 } |
1167 | 1448 |
1168 continue; | 1449 continue; |
1169 } | 1450 } |
1178 "\"%V\" must have \"keys_zone\" parameter", | 1459 "\"%V\" must have \"keys_zone\" parameter", |
1179 &cmd->name); | 1460 &cmd->name); |
1180 return NGX_CONF_ERROR; | 1461 return NGX_CONF_ERROR; |
1181 } | 1462 } |
1182 | 1463 |
1183 cache->path->cleaner = ngx_http_file_cache_cleaner; | 1464 cache->path->manager = ngx_http_file_cache_manager; |
1184 cache->path->data = cache; | 1465 cache->path->data = cache; |
1185 | 1466 |
1186 if (ngx_add_path(cf, &cache->path) != NGX_OK) { | 1467 if (ngx_add_path(cf, &cache->path) != NGX_OK) { |
1187 return NGX_CONF_ERROR; | 1468 return NGX_CONF_ERROR; |
1188 } | 1469 } |
1196 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, | 1477 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, |
1197 "duplicate zone \"%V\"", &name); | 1478 "duplicate zone \"%V\"", &name); |
1198 return NGX_CONF_ERROR; | 1479 return NGX_CONF_ERROR; |
1199 } | 1480 } |
1200 | 1481 |
1201 next = ngx_next_time(clean_time); | |
1202 | |
1203 if (next == -1) { | |
1204 ngx_conf_log_error(NGX_LOG_EMERG, cf, ngx_errno, | |
1205 ngx_next_time_n " failed"); | |
1206 return NGX_CONF_ERROR; | |
1207 } | |
1208 | 1482 |
1209 cache->shm_zone->init = ngx_http_file_cache_init; | 1483 cache->shm_zone->init = ngx_http_file_cache_init; |
1210 cache->shm_zone->data = cache; | 1484 cache->shm_zone->data = cache; |
1211 | 1485 |
1212 cache->inactive = inactive; | 1486 cache->inactive = inactive; |
1213 cache->clean_time = clean_time; | 1487 cache->max_size = max_size; |
1214 cache->next_clean_time = next; | |
1215 | 1488 |
1216 return NGX_CONF_OK; | 1489 return NGX_CONF_OK; |
1217 } | 1490 } |
1218 | 1491 |
1219 | 1492 |