Mercurial > hg > nginx
comparison src/http/ngx_http_file_cache.c @ 7002:ab199f0eb8e8
Cache: ignore long locked entries during forced expire.
Abnormally exited workers may leave locked cache entries, this can
result in the cache size on disk exceeding max_size and shared memory
exhaustion.
This change mitigates the issue by ignoring locked entries during forced
expire. It also increases the visibility of the problem by logging such
entries.
author | Dmitry Volyntsev <xeioex@nginx.com> |
---|---|
date | Thu, 18 May 2017 18:39:16 +0300 |
parents | 99934aade555 |
children | 72d3aefc2993 |
comparison
equal
deleted
inserted
replaced
7001:08537eab4f23 | 7002:ab199f0eb8e8 |
---|---|
1698 | 1698 |
1699 | 1699 |
1700 static time_t | 1700 static time_t |
1701 ngx_http_file_cache_forced_expire(ngx_http_file_cache_t *cache) | 1701 ngx_http_file_cache_forced_expire(ngx_http_file_cache_t *cache) |
1702 { | 1702 { |
1703 u_char *name; | 1703 u_char *name, *p; |
1704 size_t len; | 1704 size_t len; |
1705 time_t wait; | 1705 time_t wait; |
1706 ngx_uint_t tries; | 1706 ngx_uint_t tries; |
1707 ngx_path_t *path; | 1707 ngx_path_t *path; |
1708 ngx_queue_t *q; | 1708 ngx_queue_t *q, *sentinel; |
1709 ngx_http_file_cache_node_t *fcn; | 1709 ngx_http_file_cache_node_t *fcn; |
1710 u_char key[2 * NGX_HTTP_CACHE_KEY_LEN]; | |
1710 | 1711 |
1711 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, | 1712 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, |
1712 "http file cache forced expire"); | 1713 "http file cache forced expire"); |
1713 | 1714 |
1714 path = cache->path; | 1715 path = cache->path; |
1721 | 1722 |
1722 ngx_memcpy(name, path->name.data, path->name.len); | 1723 ngx_memcpy(name, path->name.data, path->name.len); |
1723 | 1724 |
1724 wait = 10; | 1725 wait = 10; |
1725 tries = 20; | 1726 tries = 20; |
1727 sentinel = NULL; | |
1726 | 1728 |
1727 ngx_shmtx_lock(&cache->shpool->mutex); | 1729 ngx_shmtx_lock(&cache->shpool->mutex); |
1728 | 1730 |
1729 for (q = ngx_queue_last(&cache->sh->queue); | 1731 for ( ;; ) { |
1730 q != ngx_queue_sentinel(&cache->sh->queue); | 1732 if (ngx_queue_empty(&cache->sh->queue)) { |
1731 q = ngx_queue_prev(q)) | 1733 break; |
1732 { | 1734 } |
1735 | |
1736 q = ngx_queue_last(&cache->sh->queue); | |
1737 | |
1738 if (q == sentinel) { | |
1739 break; | |
1740 } | |
1741 | |
1733 fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue); | 1742 fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue); |
1734 | 1743 |
1735 ngx_log_debug6(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, | 1744 ngx_log_debug6(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, |
1736 "http file cache forced expire: #%d %d %02xd%02xd%02xd%02xd", | 1745 "http file cache forced expire: #%d %d %02xd%02xd%02xd%02xd", |
1737 fcn->count, fcn->exists, | 1746 fcn->count, fcn->exists, |
1738 fcn->key[0], fcn->key[1], fcn->key[2], fcn->key[3]); | 1747 fcn->key[0], fcn->key[1], fcn->key[2], fcn->key[3]); |
1739 | 1748 |
1740 if (fcn->count == 0) { | 1749 if (fcn->count == 0) { |
1741 ngx_http_file_cache_delete(cache, q, name); | 1750 ngx_http_file_cache_delete(cache, q, name); |
1742 wait = 0; | 1751 wait = 0; |
1743 | 1752 break; |
1744 } else { | 1753 } |
1745 if (--tries) { | 1754 |
1746 continue; | 1755 p = ngx_hex_dump(key, (u_char *) &fcn->node.key, |
1747 } | 1756 sizeof(ngx_rbtree_key_t)); |
1748 | 1757 len = NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t); |
1749 wait = 1; | 1758 (void) ngx_hex_dump(p, fcn->key, len); |
1750 } | 1759 |
1751 | 1760 /* |
1761 * abnormally exited workers may leave locked cache entries, | |
1762 * and although it may be safe to remove them completely, | |
1763 * we prefer to just move them to the top of the inactive queue | |
1764 */ | |
1765 | |
1766 ngx_queue_remove(q); | |
1767 fcn->expire = ngx_time() + cache->inactive; | |
1768 ngx_queue_insert_head(&cache->sh->queue, &fcn->queue); | |
1769 | |
1770 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0, | |
1771 "ignore long locked inactive cache entry %*s, count:%d", | |
1772 (size_t) 2 * NGX_HTTP_CACHE_KEY_LEN, key, fcn->count); | |
1773 | |
1774 if (sentinel == NULL) { | |
1775 sentinel = q; | |
1776 } | |
1777 | |
1778 if (--tries) { | |
1779 continue; | |
1780 } | |
1781 | |
1782 wait = 1; | |
1752 break; | 1783 break; |
1753 } | 1784 } |
1754 | 1785 |
1755 ngx_shmtx_unlock(&cache->shpool->mutex); | 1786 ngx_shmtx_unlock(&cache->shpool->mutex); |
1756 | 1787 |