Mercurial > hg > nginx

comparison src/http/ngx_http_file_cache.c @ 7002:ab199f0eb8e8

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Cache: ignore long locked entries during forced expire. Abnormally exited workers may leave locked cache entries, this can result in the cache size on disk exceeding max_size and shared memory exhaustion. This change mitigates the issue by ignoring locked entries during forced expire. It also increases the visibility of the problem by logging such entries.
author Dmitry Volyntsev <xeioex@nginx.com>
date Thu, 18 May 2017 18:39:16 +0300
parents 99934aade555
children 72d3aefc2993
comparison
equal deleted inserted replaced
7001:08537eab4f23 7002:ab199f0eb8e8
1698 1698
1699 1699
1700 static time_t 1700 static time_t
1701 ngx_http_file_cache_forced_expire(ngx_http_file_cache_t *cache) 1701 ngx_http_file_cache_forced_expire(ngx_http_file_cache_t *cache)
1702 { 1702 {
1703 u_char *name; 1703 u_char *name, *p;
1704 size_t len; 1704 size_t len;
1705 time_t wait; 1705 time_t wait;
1706 ngx_uint_t tries; 1706 ngx_uint_t tries;
1707 ngx_path_t *path; 1707 ngx_path_t *path;
1708 ngx_queue_t *q; 1708 ngx_queue_t *q, *sentinel;
1709 ngx_http_file_cache_node_t *fcn; 1709 ngx_http_file_cache_node_t *fcn;
1710 u_char key[2 * NGX_HTTP_CACHE_KEY_LEN];
1710 1711
1711 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, 1712 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
1712 "http file cache forced expire"); 1713 "http file cache forced expire");
1713 1714
1714 path = cache->path; 1715 path = cache->path;
1721 1722
1722 ngx_memcpy(name, path->name.data, path->name.len); 1723 ngx_memcpy(name, path->name.data, path->name.len);
1723 1724
1724 wait = 10; 1725 wait = 10;
1725 tries = 20; 1726 tries = 20;
1727 sentinel = NULL;
1726 1728
1727 ngx_shmtx_lock(&cache->shpool->mutex); 1729 ngx_shmtx_lock(&cache->shpool->mutex);
1728 1730
1729 for (q = ngx_queue_last(&cache->sh->queue); 1731 for ( ;; ) {
1730 q != ngx_queue_sentinel(&cache->sh->queue); 1732 if (ngx_queue_empty(&cache->sh->queue)) {
1731 q = ngx_queue_prev(q)) 1733 break;
1732 { 1734 }
1735
1736 q = ngx_queue_last(&cache->sh->queue);
1737
1738 if (q == sentinel) {
1739 break;
1740 }
1741
1733 fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue); 1742 fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue);
1734 1743
1735 ngx_log_debug6(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, 1744 ngx_log_debug6(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
1736 "http file cache forced expire: #%d %d %02xd%02xd%02xd%02xd", 1745 "http file cache forced expire: #%d %d %02xd%02xd%02xd%02xd",
1737 fcn->count, fcn->exists, 1746 fcn->count, fcn->exists,
1738 fcn->key[0], fcn->key[1], fcn->key[2], fcn->key[3]); 1747 fcn->key[0], fcn->key[1], fcn->key[2], fcn->key[3]);
1739 1748
1740 if (fcn->count == 0) { 1749 if (fcn->count == 0) {
1741 ngx_http_file_cache_delete(cache, q, name); 1750 ngx_http_file_cache_delete(cache, q, name);
1742 wait = 0; 1751 wait = 0;
1743 1752 break;
1744 } else { 1753 }
1745 if (--tries) { 1754
1746 continue; 1755 p = ngx_hex_dump(key, (u_char *) &fcn->node.key,
1747 } 1756 sizeof(ngx_rbtree_key_t));
1748 1757 len = NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t);
1749 wait = 1; 1758 (void) ngx_hex_dump(p, fcn->key, len);
1750 } 1759
1751 1760 /*
1761 * abnormally exited workers may leave locked cache entries,
1762 * and although it may be safe to remove them completely,
1763 * we prefer to just move them to the top of the inactive queue
1764 */
1765
1766 ngx_queue_remove(q);
1767 fcn->expire = ngx_time() + cache->inactive;
1768 ngx_queue_insert_head(&cache->sh->queue, &fcn->queue);
1769
1770 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
1771 "ignore long locked inactive cache entry %*s, count:%d",
1772 (size_t) 2 * NGX_HTTP_CACHE_KEY_LEN, key, fcn->count);
1773
1774 if (sentinel == NULL) {
1775 sentinel = q;
1776 }
1777
1778 if (--tries) {
1779 continue;
1780 }
1781
1782 wait = 1;
1752 break; 1783 break;
1753 } 1784 }
1754 1785
1755 ngx_shmtx_unlock(&cache->shpool->mutex); 1786 ngx_shmtx_unlock(&cache->shpool->mutex);
1756 1787